diff options
Diffstat (limited to 'config-model/src/main/java')
518 files changed, 47126 insertions, 0 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java new file mode 100644 index 00000000000..1ef0ed6ec0a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -0,0 +1,279 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +import com.yahoo.cloud.config.*; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Version; +import com.yahoo.vespa.config.content.LoadTypeConfig; +import com.yahoo.cloud.config.ModelConfig.Hosts; +import com.yahoo.cloud.config.ModelConfig.Hosts.Services; +import com.yahoo.cloud.config.ModelConfig.Hosts.Services.Ports; +import com.yahoo.cloud.config.log.LogdConfig; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.document.DocumenttypesConfig; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; +import com.yahoo.messagebus.MessagebusConfig; +import com.yahoo.vespa.configmodel.producers.DocumentManager; +import com.yahoo.vespa.configmodel.producers.DocumentTypes; +import com.yahoo.vespa.documentmodel.DocumentModel; +import com.yahoo.vespa.model.*; +import com.yahoo.vespa.model.admin.Admin; +import com.yahoo.vespa.model.clients.Clients; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; +import com.yahoo.vespa.model.routing.Routing; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + + +/** + * This is the parent of all ConfigProducers in the system resulting from configuring an application. + * + * @author gjoranv + */ +public class ApplicationConfigProducerRoot extends AbstractConfigProducer<AbstractConfigProducer<?>> implements CommonConfigsProducer { + + private final DocumentModel documentModel; + private Routing routing = null; + // The ConfigProducers contained in this Vespa. (configId->producer) + Map<String, ConfigProducer> id2producer = new LinkedHashMap<>(); + private Admin admin = null; + private HostSystem hostSystem = null; + private final Version vespaVersion; + private final ApplicationId applicationId; + + /** + * Creates and initializes a new Vespa from the service config file + * in the given application directory. + * + * @param parent The parent, usually VespaModel + * @param name The name, used as configId + * @param documentModel DocumentModel to serve global document config from. + */ + public ApplicationConfigProducerRoot(AbstractConfigProducer parent, String name, DocumentModel documentModel, Version vespaVersion, ApplicationId applicationId) { + super(parent, name); + this.documentModel = documentModel; + this.vespaVersion = vespaVersion; + this.applicationId = applicationId; + } + + /** + * @return an unmodifiable copy of the set of configIds in this VespaModel. + */ + public Set<String> getConfigIds() { + return Collections.unmodifiableSet(id2producer.keySet()); + } + + /** + * Returns the ConfigProducer with the given id, or null if no such + * configId exists. + * + * @param configId The configId, e.g. "search.0/tld.0" + * @return ConfigProducer with the given configId + */ + public ConfigProducer getConfigProducer(String configId) { + return id2producer.get(configId); + } + + /** + * Returns the Service with the given id, or null if no such + * configId exists or if it belongs to a non-Service ConfigProducer. + * + * @param configId The configId, e.g. "search.0/tld.0" + * @return Service with the given configId + */ + public Service getService(String configId) { + ConfigProducer cp = getConfigProducer(configId); + if (cp == null || !(cp instanceof Service)) { + return null; + } + return (Service) cp; + } + + /** + * Adds the descendant (at any depth level), so it can be looked up + * on configId in the Map. + * + * @param descendant The configProducer descendant to add + */ + // TODO: Make protected if this moves to the same package as AbstractConfigProducer + public void addDescendant(AbstractConfigProducer descendant) { + id2producer.put(descendant.getConfigId(), descendant); + } + + /** + * Prepares the model for start. The {@link VespaModel} calls + * this methods after it has loaded this and all plugins have been loaded and + * their initialize() methods have been called. + * + * @param plugins All initialized plugins of the vespa model. + */ + public void prepare(ConfigModelRepo plugins) { + if (routing != null) { + routing.deriveCommonSettings(plugins); + } + } + + public void setupAdmin(Admin admin) { + this.admin = admin; + } + + public void setupRouting(ConfigModelRepo configModels) { + if (admin != null) { + Routing routing = configModels.getRouting(); + if (routing == null) { + routing = new Routing(ConfigModelContext.createFromParentAndId(configModels, this, "routing")); + configModels.add(routing); + } + this.routing = routing; + } + } + + @Override + public void getConfig(DocumentmanagerConfig.Builder builder) { + new DocumentManager().produce(documentModel, builder); + } + + @Override + public void getConfig(DocumenttypesConfig.Builder builder) { + new DocumentTypes().produce(documentModel, builder); + } + + @Override + public void getConfig(DocumentrouteselectorpolicyConfig.Builder builder) { + if (routing != null) { + routing.getConfig(builder); + } + } + + @Override + public void getConfig(MessagebusConfig.Builder builder) { + if (routing != null) { + routing.getConfig(builder); + } + } + + @Override + public void getConfig(LogdConfig.Builder builder) { + if (admin != null) { + admin.getConfig(builder); + } + } + + @Override + public void getConfig(SlobroksConfig.Builder builder) { + if (admin != null) { + admin.getConfig(builder); + } + } + + @Override + public void getConfig(ZookeepersConfig.Builder builder) { + if (admin != null) { + admin.getConfig(builder); + } + } + + @Override + public void getConfig(LoadTypeConfig.Builder builder) { + VespaModel model = (VespaModel) getRoot(); + Clients clients = model.getClients(); + if (clients != null) { + clients.getConfig(builder); + } + } + + @Override + public void getConfig(ClusterListConfig.Builder builder) { + VespaModel model = (VespaModel) getRoot(); + for (ContentCluster cluster : model.getContentClusters().values()) { + ClusterListConfig.Storage.Builder storage = new ClusterListConfig.Storage.Builder(); + storage.name(cluster.getName()); + storage.configid(cluster.getConfigId()); + builder.storage(storage); + } + } + + @Override + public void getConfig(ModelConfig.Builder builder) { + builder.vespaVersion(vespaVersion.toSerializedForm()); + for (HostResource modelHost : getHostSystem().getHosts()) { + builder.hosts(new Hosts.Builder() + .name(modelHost.getHostName()) + .services(getServices(modelHost)) + ); + } + } + + private List<Services.Builder> getServices(HostResource modelHost) { + List<Services.Builder> ret = new ArrayList<>(); + for (Service modelService : modelHost.getServices()) { + ret.add(new Services.Builder() + .name(modelService.getServiceName()) + .type(modelService.getServiceType()) + .configid(modelService.getConfigId()) + .clustertype(modelService.getServicePropertyString("clustertype", "")) + .clustername(modelService.getServicePropertyString("clustername", "")) + .index(Integer.parseInt(modelService.getServicePropertyString("index", "999999"))) + .ports(getPorts(modelService)) + ); + } + return ret; + } + + private List<Ports.Builder> getPorts(Service modelService) { + List<Ports.Builder> ret = new ArrayList<>(); + PortsMeta portsMeta = modelService.getPortsMeta(); + for (int i = 0; i < portsMeta.getNumPorts(); i++) { + ret.add(new Ports.Builder() + .number(modelService.getRelativePort(i)) + .tags(getPortTags(portsMeta, i)) + ); + } + return ret; + } + + public static String getPortTags(PortsMeta portsMeta, int portNumber) { + StringBuilder sb = new StringBuilder(); + boolean firstTag = true; + for (String s : portsMeta.getTagsAt(portNumber)) { + if (!firstTag) { + sb.append(" "); + } else { + firstTag = false; + } + sb.append(s); + } + return sb.toString(); + } + + public void setHostSystem(HostSystem hostSystem) { + this.hostSystem = hostSystem; + } + + @Override + public HostSystem getHostSystem() { + return hostSystem; + } + + public FileDistributionConfigProducer getFileDistributionConfigProducer() { + return admin.getFileDistributionConfigProducer(); + } + + public Admin getAdmin() { + return admin; + } + + @Override + public void getConfig(ApplicationIdConfig.Builder builder) { + builder.tenant(applicationId.tenant().value()); + builder.application(applicationId.application().value()); + builder.instance(applicationId.instance().value()); + } +} diff --git a/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java b/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java new file mode 100644 index 00000000000..228c74480c4 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +import com.yahoo.cloud.config.ApplicationIdConfig; +import com.yahoo.vespa.config.content.LoadTypeConfig; +import com.yahoo.cloud.config.log.LogdConfig; +import com.yahoo.cloud.config.SlobroksConfig; +import com.yahoo.cloud.config.ClusterListConfig; +import com.yahoo.cloud.config.ZookeepersConfig; +import com.yahoo.cloud.config.ModelConfig; +import com.yahoo.document.DocumenttypesConfig; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; +import com.yahoo.messagebus.MessagebusConfig; + + +/** + * This interface describes the configs that are produced by the model producer root. + * + * @author lulf + * @since 5.1 + */ +public interface CommonConfigsProducer extends DocumentmanagerConfig.Producer, + DocumenttypesConfig.Producer, + MessagebusConfig.Producer, + DocumentrouteselectorpolicyConfig.Producer, + LogdConfig.Producer, + SlobroksConfig.Producer, + ZookeepersConfig.Producer, + LoadTypeConfig.Producer, + ClusterListConfig.Producer, + ModelConfig.Producer, + ApplicationIdConfig.Producer { +} diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java new file mode 100644 index 00000000000..10342d44914 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +/** + * A config model is an abstract representation of a subsystem, which is used + * by the builder of that subsystem to derive a config producer tree for the subsystem, and by other + * builders to access information about the subsystem for config production at a suitable abstraction level. + * + * @author gjoranv + * @author bratseth + * @author lulf + */ +public abstract class ConfigModel { + + private final String id; + + /** + * Constructs a new config model given a context. + * + * @param modelContext The model context. + */ + public ConfigModel(ConfigModelContext modelContext) { + super(); + this.id = modelContext.getProducerId(); + } + + /** Returns the id of this model */ + public final String getId() { return id; } + + /** + * Initializes this model. All inter-model independent initialization + * is done by implementing this method. + * The model will be made available to dependent models by the framework when this returns. + * <p> + * TODO: Remove this method, as this is now done by the model builders. + * + * This default implementation does nothing. + * + * @param configModelRepo The ConfigModelRepo of the VespaModel + * @deprecated This will go away in the next Vespa major release. Instead, inject the models you depend on + * in your config model constructor. + */ + public void initialize(ConfigModelRepo configModelRepo) { return; } + + /** + * Prepares this model to start serving config requests, possibly using properties of other models. + * The framework will call this method after models have been built. The model + * should finalize its configurations that depend on other models in this step. + * + * This default implementation does nothing. + * + * @param configModelRepo The ConfigModelRepo of the system model + */ + public void prepare(ConfigModelRepo configModelRepo) { return; } + + /** + * <p>Returns whether this model must be maintained in memory for serving config requests. + * Models which are used to amend other models at build time should override this to return false.</p> + * + * <p>This default implementation returns true.</p> + */ + public boolean isServing() { return true; } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java new file mode 100644 index 00000000000..c375d8e28fa --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.producer.AbstractConfigProducer; + +/** + * This class contains a context that is passed to a model builder, and can be used to retrieve the application package, + * logger etc. + * + * @author lulf + * @since 5.1 + */ +public class ConfigModelContext { + + private final AbstractConfigProducer producer; + private final String producerId; + private final DeployState deployState; + private final ConfigModelRepoAdder configModelRepoAdder; + + private ConfigModelContext(DeployState deployState, ConfigModelRepoAdder configModelRepoAdder, + AbstractConfigProducer producer, String producerId) { + this.deployState = deployState; + this.configModelRepoAdder = configModelRepoAdder; + this.producer = producer; + this.producerId = producerId; + } + + public ApplicationPackage getApplicationPackage() { return deployState.getApplicationPackage(); } + public String getProducerId() { return producerId; } + public AbstractConfigProducer getParentProducer() { return producer; } + public DeployLogger getDeployLogger() { return deployState.getDeployLogger(); } + public DeployState getDeployState() { return deployState; } + + /** Returns write access to the config model repo, or null (only) if this is improperly initialized during testing */ + public ConfigModelRepoAdder getConfigModelRepoAdder() { return configModelRepoAdder; } + + /** + * Create a new context with a different parent, but with the same id and application package. + * + * @param newParent The parent to use for the new context. + * @return A new context. + */ + public ConfigModelContext modifyParent(AbstractConfigProducer newParent) { + return ConfigModelContext.create(deployState, configModelRepoAdder, newParent, producerId); + } + + /** + * Create an application context from a parent producer and an id. + * @param deployState The global deploy state for this model. + * @param parent The parent to be used for the config model. + * @param id The id to be used for the config model. + * @return An model context that can be passed to a model. + */ + public static ConfigModelContext create(DeployState deployState, ConfigModelRepoAdder configModelRepoAdder, + AbstractConfigProducer parent, String id) { + return new ConfigModelContext(deployState, configModelRepoAdder, parent, id); + } + + /** + * Create an application context from a parent producer and an id. + * @param parent The parent to be used for the config model. + * @param id The id to be used for the config model. + * @return An model context that can be passed to a model. + */ + public static ConfigModelContext createFromParentAndId(ConfigModelRepoAdder configModelRepoAdder, AbstractConfigProducer parent, String id) { + return create(parent.getRoot().getDeployState(), configModelRepoAdder, parent, id); + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelInstanceFactory.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelInstanceFactory.java new file mode 100644 index 00000000000..a9b3fc73eb3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelInstanceFactory.java @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +/** + * Interface for factories of config models. + * + * @author lulf + * @since 5.1 + */ +public interface ConfigModelInstanceFactory<MODEL extends ConfigModel> { + + /** + * Create an instance of {@link com.yahoo.config.model.ConfigModel} given the input context. + * + * @param context The {@link com.yahoo.config.model.ConfigModelContext} to use. + * @return an instance of {@link com.yahoo.config.model.ConfigModel} + */ + MODEL createModel(ConfigModelContext context); + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRegistry.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRegistry.java new file mode 100644 index 00000000000..2735b9f7fdf --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRegistry.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.config.model.builder.xml.ConfigModelId; + +import java.util.Collection; +import java.util.Collections; + +/** + * A resolver of implementations of named config models. + * Registries may be chained in a chain of command. + * + * @author bratseth + */ +public abstract class ConfigModelRegistry { + + private final ConfigModelRegistry chained; + + public ConfigModelRegistry() { + this(new EmptyTerminalRegistry()); + } + + /** Creates a config model class registry which forwards unresolved requests to the argument instance */ + public ConfigModelRegistry(ConfigModelRegistry chained) { + this.chained=chained; + } + + /** + * Returns the builders this id resolves to both in this and any chained registry. + * + * @return the resolved config model builders, or an empty list (never null) if none + */ + public abstract Collection<ConfigModelBuilder> resolve(ConfigModelId id); + + public ConfigModelRegistry chained() { return chained; } + + /** An empty registry which does not support chaining */ + private static class EmptyTerminalRegistry extends ConfigModelRegistry { + + public EmptyTerminalRegistry() { + super(null); + } + + @Override + public Collection<ConfigModelBuilder> resolve(ConfigModelId id) { + return Collections.emptyList(); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java new file mode 100644 index 00000000000..fa46c33a97f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java @@ -0,0 +1,256 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.config.model.builder.xml.ConfigModelId; +import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.config.model.graph.ModelGraphBuilder; +import com.yahoo.config.model.graph.ModelNode; +import com.yahoo.config.model.provision.HostsXmlProvisioner; +import com.yahoo.log.LogLevel; +import com.yahoo.path.Path; +import com.yahoo.text.XML; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.builder.VespaModelBuilder; +import com.yahoo.vespa.model.clients.Clients; +import com.yahoo.vespa.model.content.Content; +import com.yahoo.vespa.model.routing.Routing; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.io.Reader; +import java.io.Serializable; +import java.io.StringReader; +import java.util.*; +import java.util.logging.Logger; + +/** + * A collection of config model instances owned by a system model + * + * @author gjoranv + */ +public class ConfigModelRepo implements ConfigModelRepoAdder, Serializable, Iterable<ConfigModel> { + + private static final long serialVersionUID = 1L; + + private static final Logger log = Logger.getLogger(ConfigModelRepo.class.getPackage().toString()); + + private final Map<String,ConfigModel> configModelMap = new TreeMap<>(); + private final List<ConfigModel> configModels = new ArrayList<>(); + + /** + * Returns a config model for a given id + * + * @param id the id of the model to return + * @return the model, or none if a model with this id is not present in this + */ + public ConfigModel get(String id) { + return configModelMap.get(id); + } + + /** Adds a new config model instance in this */ + @Override + public void add(ConfigModel model) { + configModelMap.put(model.getId(), model); + configModels.add(model); + } + + /** Returns the models in this as an iterator */ + public Iterator<ConfigModel> iterator() { + return configModels.iterator(); + } + + /** Returns a read-only view of the config model instances of this */ + public Map<String,ConfigModel> asMap() { return Collections.unmodifiableMap(configModelMap); } + + /** Initialize part 1.: Reads the config models used in the application package. */ + public void readConfigModels(DeployState deployState, VespaModelBuilder builder, + ApplicationConfigProducerRoot root, ConfigModelRegistry configModelRegistry) throws IOException, SAXException { + Element userServicesElement = getServicesFromApp(deployState.getApplicationPackage()); + readConfigModels(root, userServicesElement, deployState, configModelRegistry); + builder.postProc(root, this); + } + + private Element getServicesFromApp(ApplicationPackage applicationPackage) throws IOException, SAXException { + try (Reader servicesFile = applicationPackage.getServices()) { + return getServicesFromReader(servicesFile); + } + } + + /** + * If the top level is <services>, it contains a list of services elements, + * otherwise, the top level tag is a single service. + */ + private List<Element> getServiceElements(Element servicesRoot) { + if (servicesRoot.getTagName().equals("services")) + return XML.getChildren(servicesRoot); + + List<Element> singleServiceList = new ArrayList<>(1); + singleServiceList.add(servicesRoot); + return singleServiceList; + } + + /** + * Creates all the config models specified in the given XML element and + * passes their respective XML node as parameter. + * + * @param root The Root to set as parent for all plugins + * @param servicesRoot XML root node of the services file + */ + private void readConfigModels(ApplicationConfigProducerRoot root, Element servicesRoot, DeployState deployState, ConfigModelRegistry configModelRegistry) throws IOException, SAXException { + final Map<ConfigModelBuilder, List<Element>> model2Element = new LinkedHashMap<>(); + ModelGraphBuilder graphBuilder = new ModelGraphBuilder(); + + final List<Element> children = getServiceElements(servicesRoot); + + if (XML.getChild(servicesRoot, "admin") == null) + children.add(getImplicitAdmin(deployState)); + + children.addAll(getPermanentServices(deployState)); + + for (Element servicesElement : children) { + String tagName = servicesElement.getTagName(); + if (tagName.equals("config")) continue; // TODO: Remove on Vespa 6 + if (tagName.equals("cluster")) continue; // TODO: Remove on Vespa 6 + if ((tagName.equals("clients")) && deployState.isHostedVespa()) + throw new IllegalArgumentException("<" + tagName + "> is not allowed when running Vespa in a hosted environment"); + + String tagVersion = servicesElement.getAttribute("version"); + ConfigModelId xmlId = ConfigModelId.fromNameAndVersion(tagName, tagVersion); + + Collection<ConfigModelBuilder> builders = configModelRegistry.resolve(xmlId); + + if (builders.isEmpty()) + throw new RuntimeException("Could not resolve tag <" + tagName + " version=\"" + tagVersion + "\"> to a config model component"); + + for (ConfigModelBuilder builder : builders) { + if ( ! model2Element.containsKey(builder)) { + model2Element.put(builder, new ArrayList<>()); + graphBuilder.addBuilder(builder); + } + model2Element.get(builder).add(servicesElement); + } + } + + for (ModelNode node : graphBuilder.build().topologicalSort()) + buildModels(node, deployState, root, model2Element.get(node.builder)); + for (ConfigModel model : configModels) + model.initialize(ConfigModelRepo.this); + } + + private Collection<Element> getPermanentServices(DeployState deployState) throws IOException, SAXException { + List<Element> permanentServices = new ArrayList<>(); + Optional<ApplicationPackage> applicationPackage = deployState.getPermanentApplicationPackage(); + if (applicationPackage.isPresent()) { + ApplicationFile file = applicationPackage.get().getFile(Path.fromString(ApplicationPackage.PERMANENT_SERVICES)); + if (file.exists()) { + try (Reader reader = file.createReader()) { + Element permanentServicesRoot = getServicesFromReader(reader); + permanentServices.addAll(getServiceElements(permanentServicesRoot)); + } + } + } + return permanentServices; + } + + private Element getServicesFromReader(Reader reader) throws IOException, SAXException { + Document doc = XmlHelper.getDocumentBuilder().parse(new InputSource(reader)); + return doc.getDocumentElement(); + } + + private void buildModels(ModelNode node, DeployState deployState, AbstractConfigProducer parent, List<Element> elements) { + for (Element servicesElement : elements) { + ConfigModel model = buildModel(node, deployState, parent, servicesElement); + if (model.isServing()) + add(model); + } + } + + private ConfigModel buildModel(ModelNode node, DeployState deployState, AbstractConfigProducer parent, Element servicesElement) { + ConfigModelBuilder builder = node.builder; + ConfigModelContext context = ConfigModelContext.create(deployState, this, parent, getIdString(servicesElement)); + return builder.build(node, servicesElement, context); + } + + private static String getIdString(Element spec) { + String idString = XmlHelper.getIdString(spec); + if (idString == null || idString.isEmpty()) { + idString = spec.getTagName(); + } + return idString; + } + + /** + * Initialize part 2.: + * Prepare all config models for starting. Must be called after plugins are loaded and frozen. + */ + public void prepareConfigModels() { + for (ConfigModel model : configModels) { + model.prepare(ConfigModelRepo.this); + } + } + + @SuppressWarnings("unchecked") + public <T extends ConfigModel> List<T> getModels(Class<T> modelClass) { + List<T> modelsOfModelClass = new ArrayList<>(); + + for (ConfigModel model : asMap().values()) { + if (modelClass.isInstance(model)) + modelsOfModelClass.add((T)model); + } + return modelsOfModelClass; + } + + public Clients getClients() { + for (ConfigModel m : configModels) { + if (m instanceof Clients) { + return (Clients)m; + } + } + return null; + } + + public Routing getRouting() { + for (ConfigModel m : configModels) { + if (m instanceof Routing) { + return (Routing)m; + } + } + return null; + } + + public Content getContent() { + for (ConfigModel m : configModels) { + if (m instanceof Content) { + return (Content)m; + } + } + return null; + } + + // TODO: Doctoring on the XML is the wrong level for this. We should be able to mark a model as default instead -Jon + private static Element getImplicitAdmin(DeployState deployState) throws IOException, SAXException { + final boolean hostedVespa = deployState.isHostedVespa(); + String defaultAdminElement = hostedVespa ? getImplicitAdminV4() : getImplicitAdminV2(); + log.log(LogLevel.DEBUG, "No <admin> defined, using " + defaultAdminElement); + return XmlHelper.getDocumentBuilder().parse(new InputSource(new StringReader(defaultAdminElement))).getDocumentElement(); + } + + private static String getImplicitAdminV2() { + return "<admin version='2.0'>\n" + + " <adminserver hostalias='" + HostsXmlProvisioner.IMPLICIT_ADMIN_HOSTALIAS + "'/>\n" + + "</admin>\n"; + } + + private static String getImplicitAdminV4() { + return "<admin version='4.0'>\n" + + " <nodes count='1' />\n" + + "</admin>\n"; + } +} diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepoAdder.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepoAdder.java new file mode 100644 index 00000000000..93c591b3a8e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepoAdder.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +/** + * An interface which provides addition of new config models. + * This exists because some models need to add additional models during the build phase so *write* access + * to the config model repo is needed. *Read* access, on the other hand needs to happen through config model dependency + * inkection to avoid circular dependencies or undeclared dependencies working by accident. + * + * @author bratseth + */ +public interface ConfigModelRepoAdder { + + void add(ConfigModel model); + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelUtils.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelUtils.java new file mode 100644 index 00000000000..f64eee6e863 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelUtils.java @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; +import java.io.Serializable; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static javax.xml.xpath.XPathConstants.BOOLEAN; + +import static com.yahoo.text.Lowercase.toLowerCase; + +/** + * Utilities for config models + * + * @author gjoranv + */ +// TODO: Split this into appropriate classes, or move to ConfigModel superclass +public class ConfigModelUtils implements Serializable { + + private static final long serialVersionUID = 1L; + + public static Pattern hourNmin = Pattern.compile("(\\d\\d):(\\d\\d)"); + + public static Map<String, Integer> day2int; + static { + day2int = new HashMap<>(); + day2int.put("sunday", 0); + day2int.put("monday", 1); + day2int.put("tuesday", 2); + day2int.put("wednesday", 3); + day2int.put("thursday", 4); + day2int.put("friday", 5); + day2int.put("saturday", 6); + } + + /** Parses a 24 hour clock that must be the five characters ##:## to an int stating minutes after midnight. */ + public static int getTimeOfDay(String time) { + Matcher m = ConfigModelUtils.hourNmin.matcher(time); + if (m.matches()) { + return Integer.parseInt(m.group(1)) * 60 + Integer.parseInt(m.group(2)); + } + throw new IllegalArgumentException("The string '" + time + "' is not in ##:## format."); + } + + /** Parses a day of week name in english to an int, where 0 is sunday, 6 saturday. */ + public static int getDayOfWeek(String day) { + return ConfigModelUtils.day2int.get(toLowerCase(day)); + } + + /** + * Create a string with link to documentation for latest release. + * + * @param filePath Relative path of the file to link to, e.g. reference/services-jdisc.html + * @return a String with link to documentation + */ + public static String createDocLink(String filePath) { + return filePath; + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/MapConfigModelRegistry.java b/config-model/src/main/java/com/yahoo/config/model/MapConfigModelRegistry.java new file mode 100644 index 00000000000..d47a009928d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/MapConfigModelRegistry.java @@ -0,0 +1,63 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +import com.google.inject.Inject; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.config.model.builder.xml.ConfigModelId; +import com.yahoo.log.LogLevel; +import org.osgi.framework.Bundle; +import org.osgi.framework.FrameworkUtil; + +import java.util.*; +import java.util.logging.Logger; + +/** + * @author lulf + * @since 5.1 + */ +public class MapConfigModelRegistry extends ConfigModelRegistry { + + private static final Logger log = Logger.getLogger(MapConfigModelRegistry.class.getPackage().getName()); + private final List<ConfigModelBuilder> builders; + + /** + * Constructs a registry of config models, where the components are injected. + * + * @param registry a component registry + */ + @Inject + public MapConfigModelRegistry(ComponentRegistry<? extends ConfigModelBuilder> registry) { + this(registry.allComponents()); + } + + /** + * Constructs a registry of config models. + * + * @param builderCollection A collection of builders used to populate the registry. + */ + public MapConfigModelRegistry(Collection<? extends ConfigModelBuilder> builderCollection) { + super(); + builders = new ArrayList<>(builderCollection); + } + + @Override + public Collection<ConfigModelBuilder> resolve(ConfigModelId id) { + Set<ConfigModelBuilder> matchingBuilders = new HashSet<>(chained().resolve(id)); + for (ConfigModelBuilder builder : builders) + if (builder.handlesElements().contains(id)) + matchingBuilders.add(builder); + return matchingBuilders; + } + + /** + * Create a registry from a variable argument list of builders. + * + * @param builders A variable argument list of builders to use in this map + * @return a ConfigModelRegistry instance. + */ + public static ConfigModelRegistry createFromList(ConfigModelBuilder<? extends ConfigModel> ... builders) { + return new MapConfigModelRegistry(Arrays.asList(builders)); + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/NullConfigModelRegistry.java b/config-model/src/main/java/com/yahoo/config/model/NullConfigModelRegistry.java new file mode 100644 index 00000000000..6930c8c792b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/NullConfigModelRegistry.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.config.model.builder.xml.ConfigModelId; + +import java.util.Collection; + +/** + * A config model class registry that only forwards to the chained registry. + * + * @author bratseth + */ +public class NullConfigModelRegistry extends ConfigModelRegistry { + + @Override + public Collection<ConfigModelBuilder> resolve(ConfigModelId id) { + return chained().resolve(id); + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java new file mode 100644 index 00000000000..8b6ea27ccab --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java @@ -0,0 +1,113 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.admin; + +import com.google.common.collect.ImmutableList; +import com.yahoo.config.model.ConfigModel; +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.config.model.ApplicationConfigProducerRoot; +import com.yahoo.config.model.deploy.DeployProperties; +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.config.model.builder.xml.ConfigModelId; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.admin.Admin; +import com.yahoo.vespa.model.builder.xml.dom.DomAdminV2Builder; +import com.yahoo.vespa.model.builder.xml.dom.DomAdminV4Builder; +import com.yahoo.vespa.model.container.ContainerModel; +import org.w3c.dom.Element; + +import java.util.*; + +/** + * Config model adaptor of the Admin class. + * + * @author lulf + * @since 5.1 + */ +public class AdminModel extends ConfigModel { + + private Admin admin = null; + private final Collection<ContainerModel> containerModels; + + /** + * Constructs a new config model given a context. + * + * @param modelContext the model context. + */ + public AdminModel(ConfigModelContext modelContext, Collection<ContainerModel> containerModels) { + super(modelContext); + this.containerModels = containerModels; + } + + public Admin getAdmin() { return admin; } + + private Collection<ContainerModel> getContainerModels() { return containerModels; } + + @Override + public void prepare(ConfigModelRepo configModelRepo) { + verifyClusterControllersOnlyDefinedForContent(configModelRepo); + if (admin == null || admin.getClusterControllers() == null) return; + admin.getClusterControllers().prepare(); + } + + private void verifyClusterControllersOnlyDefinedForContent(ConfigModelRepo configModelRepo) { + Admin admin = getAdmin(); + if (admin == null || admin.getClusterControllers() == null) return; + if (configModelRepo.getContent() == null) { + throw new IllegalArgumentException("Declaring <clustercontrollers> in <admin> in services.xml will not work when <content> is not defined"); + } + } + + public static class BuilderV2 extends ConfigModelBuilder<AdminModel> { + + public static final List<ConfigModelId> configModelIds = + ImmutableList.of(ConfigModelId.fromNameAndVersion("admin", "2.0"), + ConfigModelId.fromNameAndVersion("admin", "1.0")); + + public BuilderV2() { + super(AdminModel.class); + } + + @Override + public List<ConfigModelId> handlesElements() { return configModelIds; } + + @Override + public void doBuild(AdminModel model, Element adminElement, ConfigModelContext modelContext) { + AbstractConfigProducer parent = modelContext.getParentProducer(); + DeployProperties properties = modelContext.getDeployState().getProperties(); + DomAdminV2Builder domBuilder = new DomAdminV2Builder(modelContext.getDeployState().getFileRegistry(), properties.multitenant(), properties.configServerSpecs()); + model.admin = domBuilder.build(parent, adminElement); + // TODO: Is required since other models depend on admin. + if (parent instanceof ApplicationConfigProducerRoot) { + ((ApplicationConfigProducerRoot)parent).setupAdmin(model.admin); + } + } + } + + public static class BuilderV4 extends ConfigModelBuilder<AdminModel> { + + public static final List<ConfigModelId> configModelIds = + ImmutableList.of(ConfigModelId.fromNameAndVersion("admin", "3.0"), + ConfigModelId.fromNameAndVersion("admin", "4.0")); + + public BuilderV4() { + super(AdminModel.class); + } + + @Override + public List<ConfigModelId> handlesElements() { return configModelIds; } + + @Override + public void doBuild(AdminModel model, Element adminElement, ConfigModelContext modelContext) { + AbstractConfigProducer parent = modelContext.getParentProducer(); + DeployProperties properties = modelContext.getDeployState().getProperties(); + DomAdminV4Builder domBuilder = new DomAdminV4Builder(modelContext, properties.multitenant(), properties.configServerSpecs(), model.getContainerModels()); + model.admin = domBuilder.build(parent, adminElement); + // TODO: Is required since other models depend on admin. + if (parent instanceof ApplicationConfigProducerRoot) { + ((ApplicationConfigProducerRoot)parent).setupAdmin(model.admin); + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java new file mode 100644 index 00000000000..a0b1be20df6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java @@ -0,0 +1,119 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.builder.xml; + +import com.yahoo.component.AbstractComponent; +import com.yahoo.config.model.ConfigModel; +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.ConfigModelInstanceFactory; +import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.config.model.api.ConfigModelPlugin; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import org.w3c.dom.Element; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.List; + +/** + * Builds a config model using DOM parsers + * + * @author vegardh + * @since 5.1.10 + */ +public abstract class ConfigModelBuilder<MODEL extends ConfigModel> extends AbstractComponent implements ConfigModelPlugin { + + private Class<MODEL> configModelClass; + + public ConfigModelBuilder(Class<MODEL> configModelClass) { + this.configModelClass = configModelClass; + } + + /** + * Method that must return the XML elements this builder handles. Subclasses must implement this in order to + * get called when one of the elements have been encountered when parsing. + * + * @return A list of elements that this builder handles. + */ + public abstract List<ConfigModelId> handlesElements(); + + /** + * Convenience hook called from {@link #build}. Implement this method to build a config model. + * + * @param spec The XML element that this builder should handle. + * @param modelContext A model context that contains the application package and other data needed by the + * config model constructor. + */ + public abstract void doBuild(MODEL model, Element spec, ConfigModelContext modelContext); + + /** + * Builds an instance of this component model. + * This calls instantiate(...), instance.setUp(...), doBuild(instance, ...). + * + * @param deployState a global deployment state used for this model. + * @param parent the root config producer this should be added to + * @param spec the XML element this is constructed from + */ + public final MODEL build(DeployState deployState, ConfigModelRepo configModelRepo, AbstractConfigProducer parent, Element spec) { + ConfigModelContext context = ConfigModelContext.create(deployState, configModelRepo, parent, getIdString(spec)); + return build(new DefaultModelInstanceFactory(), spec, context); + } + + /** + * Builds an instance of this component model. + * This calls instantiate(...), instance.setUp(...), doBuild(instance, ...). + * + * @param factory A factory capable of creating models. + * @param spec the XML element this is constructed from + * @param context A context object containing various data used by builders. + */ + public MODEL build(ConfigModelInstanceFactory<MODEL> factory, Element spec, ConfigModelContext context) { + MODEL model = factory.createModel(context); + doBuild(model, spec, context); + return model; + } + + public Class<MODEL> getModelClass() { + return configModelClass; + } + + private static String getIdString(Element spec) { + String idString = XmlHelper.getIdString(spec); + if (idString == null || idString.isEmpty()) { + idString = spec.getTagName(); + } + return idString; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof ConfigModelBuilder)) { + return false; + } + ConfigModelBuilder otherBuilder = (ConfigModelBuilder) other; + List<ConfigModelId> thisIds = this.handlesElements(); + List<ConfigModelId> otherIds = otherBuilder.handlesElements(); + if (thisIds.size() != otherIds.size()) { + return false; + } + for (int i = 0; i < thisIds.size(); i++) { + if (!thisIds.get(i).equals(otherIds.get(i))) { + return false; + } + } + return true; + } + + + private class DefaultModelInstanceFactory implements ConfigModelInstanceFactory<MODEL> { + @Override + public MODEL createModel(ConfigModelContext context) { + try { + Constructor<MODEL> constructor = configModelClass.getConstructor(ConfigModelContext.class); + return constructor.newInstance(context); + } catch (NoSuchMethodException | InvocationTargetException | InstantiationException | IllegalAccessException e) { + throw new RuntimeException("Error constructing model '" + configModelClass.getName() + "'", e); + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java new file mode 100644 index 00000000000..fca114757ec --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java @@ -0,0 +1,94 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.builder.xml; + +import com.yahoo.component.Version; + +/** + * A {@link ConfigModelId} describes an element handled by a {@link ConfigModelBuilder}. + * + * @author lulf + * @since 5.1 + */ +public class ConfigModelId implements Comparable<ConfigModelId> { + + private final String name; + private final Version version; + private final String stringValue; + + private ConfigModelId(String name, Version version) { + this.name = name; + this.version = version; + this.stringValue = toStringValue(); + } + + /** + * Create id with a name and version + * @param tagName Name of the id + * @param tagVersion Version of the id + * @return A ConfigModelId instance + */ + public static ConfigModelId fromNameAndVersion(String tagName, String tagVersion) { + return new ConfigModelId(tagName, Version.fromString(tagVersion)); + } + + /** + * Create id with given name, using default version 1. + * + * @param tagName Name of the id + * @return A ConfigModelId instance + */ + public static ConfigModelId fromName(String tagName) { + return new ConfigModelId(tagName, new Version(1)); + } + + @Override + public boolean equals(Object object) { + if (!(object instanceof ConfigModelId)) return false; + ConfigModelId other = (ConfigModelId)object; + return this.name.equals(other.name) && this.version.equals(other.version); + } + + @Override + public int compareTo(ConfigModelId other) { + if (other == this) return 0; + int cmp = this.name.compareTo(other.name); + if (cmp == 0) { + cmp = this.version.compareTo(other.version); + } + return cmp; + } + + @Override + public String toString() { + return stringValue; + } + + @Override + public int hashCode() { + return stringValue.hashCode(); + } + + /** + * Return the XML element name. + * @return the name of the config model + */ + public String getName() { + return name; + } + + /** + * Return the XML element version. + * @return the version of the config model + */ + Version getVersion() { + return version; + } + + private String toStringValue() { + StringBuilder sb = new StringBuilder(); + sb.append(name); + sb.append("."); + sb.append(version); + return sb.toString(); + } +} diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java new file mode 100644 index 00000000000..924d888b0d0 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java @@ -0,0 +1,135 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.builder.xml; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.log.LogLevel; +import com.yahoo.text.XML; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import java.io.IOException; +import java.io.Reader; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.logging.Logger; + + +/** + * Static methods for helping dom building + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public final class XmlHelper { + private static final Logger log = Logger.getLogger(XmlHelper.class.getPackage().toString()); + + + private static final String idReference = "idref"; + // Access to this needs to be synchronized (as it is in getDocumentBuilder() below) + public static final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + + static { + XmlHelper.factory.setNamespaceAware(true); + // if use jdom and jaxen this will fail badly: + XmlHelper.factory.setXIncludeAware(true); + } + + private XmlHelper() {} + + public static String nullIfEmpty(String attribute) { + if (attribute.isEmpty()) + return null; + else + return attribute; + } + + /** + * For searchers inside search chains, the id may be both a reference and an id at once, or just a reference. + * In other cases, it is clear which one it is from context, so I think the difference is not worth bothering users + * with, unless they are XML purists in which case they will have the option of writing this correctly. + * - Jon + */ + public static String getIdString(Element element) { + String idString = element.getAttribute("id"); + if (idString == null || idString.trim().equals("")) + idString = element.getAttribute(idReference); + if (idString == null || idString.trim().equals("")) + idString = element.getAttribute("ident"); + return idString; + } + + public static ComponentId getId(Element element) { + return new ComponentId(getIdString(element)); + } + + public static ComponentSpecification getIdRef(Element element) { + return new ComponentSpecification(getIdString(element)); + } + + public static Document getDocument(Reader reader) { + Document doc; + try { + doc = getDocumentBuilder().parse(new InputSource(reader)); + } catch (SAXException | IOException e) { + throw new IllegalArgumentException(e); + } + return doc; + } + + public static List<String> splitAndDiscardEmpty(String field, String regex) { + List<String> ret = new ArrayList<>(); + for (String t : field.split(regex)) { + if (!t.isEmpty()) { + ret.add(t); + } + } + return ret; + } + + public static List<String> spaceSeparatedSymbols(String field) { + return splitAndDiscardEmpty(field, " "); + } + + public static Collection<String> spaceSeparatedSymbolsFromAttribute(Element spec, String name) { + return spaceSeparatedSymbols(spec.getAttribute(name)); + } + + public static Collection<String> valuesFromElements(Element parent, String elementName) { + List<String> symbols = new ArrayList<>(); + for (Element symbol : XML.getChildren(parent, elementName)) { + symbols.add(XML.getValue(symbol).trim()); + } + return symbols; + } + + public static boolean isReference(Element element) { + return element.hasAttribute(idReference); + } + + /** + * Creates a new XML document builder. + * + * @return A new DocumentBuilder instance, or null if we fail to get one. + */ + public static synchronized DocumentBuilder getDocumentBuilder() { + try { + DocumentBuilder docBuilder = factory.newDocumentBuilder(); + log.log(LogLevel.DEBUG, "XML parser now operational!"); + return docBuilder; + } catch (ParserConfigurationException e) { + log.log(LogLevel.WARNING, "No XML parser available - " + e); + return null; + } + } + + public static Optional<String> getOptionalAttribute(Element element, String name) { + return Optional.ofNullable(element.getAttribute(name)).filter(s -> !s.isEmpty()); + } +} diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/package-info.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/package-info.java new file mode 100644 index 00000000000..6ce400dc922 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.config.model.builder.xml; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java b/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java new file mode 100644 index 00000000000..22ea054a27d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.deploy; + +import com.yahoo.vespa.config.ConfigDefinition; +import com.yahoo.vespa.config.ConfigDefinitionKey; + +/** + * @author lulf + * @since 5.1 + */ +public interface ConfigDefinitionStore { + + ConfigDefinition getConfigDefinition(ConfigDefinitionKey defKey); + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java new file mode 100644 index 00000000000..3f6ef494a27 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java @@ -0,0 +1,115 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.deploy; + +import com.yahoo.config.model.api.ConfigServerSpec; +import com.yahoo.config.provision.*; + +import java.util.ArrayList; +import java.util.List; + +/** + * Collection of properties for a deployment. + * + * @author lulf + * @since 5.17 + */ +public class DeployProperties { + private final boolean multitenant; + private final ApplicationId applicationId; + private final List<ConfigServerSpec> serverSpecs = new ArrayList<>(); + private final boolean hostedVespa; + private final Version vespaVersion; + private final Zone zone; + + private DeployProperties(boolean multitenant, + ApplicationId applicationId, + List<ConfigServerSpec> configServerSpecs, + boolean hostedVespa, Version vespaVersion, Zone zone) { + this.vespaVersion = vespaVersion; + this.zone = zone; + this.multitenant = multitenant || hostedVespa || Boolean.getBoolean("multitenant"); + this.applicationId = applicationId; + this.serverSpecs.addAll(configServerSpecs); + this.hostedVespa = hostedVespa; + } + + + public boolean multitenant() { + return multitenant; + } + + public ApplicationId applicationId() { + return applicationId; + } + + public List<ConfigServerSpec> configServerSpecs() { + return serverSpecs; + } + + public Quota quota() { + return new Quota(Integer.MAX_VALUE); + } + + public boolean hostedVespa() { + return hostedVespa; + } + + public Version vespaVersion() { + return vespaVersion; + } + + public Zone zone() { + return zone; + } + + public static class Builder { + + private ApplicationId applicationId = ApplicationId.defaultId(); + private boolean multitenant = false; + private List<ConfigServerSpec> configServerSpecs = new ArrayList<>(); + private Quota quota = new Quota(Integer.MAX_VALUE); + private boolean hostedVespa = false; + private Version vespaVersion = Version.fromIntValues(1, 0, 0); + private Zone zone = Zone.defaultZone(); + + public Builder applicationId(ApplicationId applicationId) { + this.applicationId = applicationId; + return this; + } + + public Builder multitenant(boolean multitenant) { + this.multitenant = multitenant; + return this; + } + + public Builder configServerSpecs(List<ConfigServerSpec> configServerSpecs) { + this.configServerSpecs = configServerSpecs; + return this; + } + + public Builder quota(Quota quota) { + this.quota = quota; + return this; + } + + public Builder vespaVersion(Version version) { + this.vespaVersion = version; + return this; + } + + public Builder hostedVespa(boolean hostedVespa) { + this.hostedVespa = hostedVespa; + return this; + } + + public Builder zone(Zone zone) { + this.zone = zone; + return this; + } + + public DeployProperties build() { + return new DeployProperties(multitenant, applicationId, configServerSpecs, hostedVespa, vespaVersion, zone); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java new file mode 100644 index 00000000000..5896dc59df2 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -0,0 +1,381 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.deploy; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.application.api.UnparsedConfigDefinition; +import com.yahoo.config.codegen.CNode; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.provision.HostsXmlProvisioner; +import com.yahoo.config.model.provision.SingleNodeProvisioner; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.Rotation; +import com.yahoo.config.provision.Zone; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.log.LogLevel; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.config.ConfigDefinition; +import com.yahoo.vespa.config.ConfigDefinitionBuilder; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.documentmodel.DocumentModel; +import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.vespa.model.application.validation.xml.ValidationOverridesXMLReader; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import com.yahoo.vespa.model.container.search.QueryProfilesBuilder; +import com.yahoo.vespa.model.container.search.SemanticRuleBuilder; +import com.yahoo.vespa.model.container.search.SemanticRules; +import com.yahoo.vespa.model.search.SearchDefinition; + +import java.io.File; +import java.io.IOException; +import java.io.Reader; +import java.time.Instant; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.logging.Logger; + +/** + * Contains various state during deploy that should be reachable to all builders of a {@link com.yahoo.config.model.ConfigModel} + * + * @author lulf + * @since 5.8 + */ +public class DeployState implements ConfigDefinitionStore { + + private static final Logger log = Logger.getLogger(DeployState.class.getName()); + + private final DeployLogger logger; + private final FileRegistry fileRegistry; + private final DocumentModel documentModel; + private final List<SearchDefinition> searchDefinitions; + private final ApplicationPackage applicationPackage; + private final Optional<ConfigDefinitionRepo> configDefinitionRepo; + private final Optional<ApplicationPackage> permanentApplicationPackage; + private final Optional<Model> previousModel; + private final DeployProperties properties; + private final Set<Rotation> rotations; + private final Zone zone; + private final QueryProfiles queryProfiles; + private final SemanticRules semanticRules; + private final ValidationOverrides validationOverrides; + + private final HostProvisioner provisioner; + + public static DeployState createTestState() { + return new Builder().build(); + } + + public static DeployState createTestState(ApplicationPackage applicationPackage) { + return new Builder().applicationPackage(applicationPackage).build(); + } + + private DeployState(ApplicationPackage applicationPackage, SearchDocumentModel searchDocumentModel, RankProfileRegistry rankProfileRegistry, + FileRegistry fileRegistry, DeployLogger deployLogger, Optional<HostProvisioner> hostProvisioner, DeployProperties properties, + Optional<ApplicationPackage> permanentApplicationPackage, Optional<ConfigDefinitionRepo> configDefinitionRepo, + java.util.Optional<Model> previousModel, Set<Rotation> rotations, Zone zone, QueryProfiles queryProfiles, SemanticRules semanticRules, Instant now) { + this.logger = deployLogger; + this.fileRegistry = fileRegistry; + this.rankProfileRegistry = rankProfileRegistry; + this.applicationPackage = applicationPackage; + this.properties = properties; + this.previousModel = previousModel; + if (hostProvisioner.isPresent()) { + this.provisioner = hostProvisioner.get(); + } else { + this.provisioner = getDefaultModelHostProvisioner(applicationPackage); + } + this.searchDefinitions = searchDocumentModel.getSearchDefinitions(); + this.documentModel = searchDocumentModel.getDocumentModel(); + this.permanentApplicationPackage = permanentApplicationPackage; + this.configDefinitionRepo = configDefinitionRepo; + this.rotations = rotations; + this.zone = zone; + this.queryProfiles = queryProfiles; // TODO: Remove this by seeing how pagetemplates are propagated + this.semanticRules = semanticRules; // TODO: Remove this by seeing how pagetemplates are propagated + this.validationOverrides = new ValidationOverridesXMLReader().read(applicationPackage.getValidationOverrides(), now); + + } + + public static HostProvisioner getDefaultModelHostProvisioner(ApplicationPackage applicationPackage) { + if (applicationPackage.getHosts() == null) { + return new SingleNodeProvisioner(); + } else { + return new HostsXmlProvisioner(applicationPackage.getHosts()); + } + } + + /** Get the global rank profile registry for this application. */ + public final RankProfileRegistry rankProfileRegistry() { return rankProfileRegistry; } + + /** Returns the validation overrides of this. This is never null */ + public ValidationOverrides validationOverrides() { return validationOverrides; } + + /** + * Returns the config def with the given name and namespace. + * + * @param defKey The {@link ConfigDefinitionKey} that will uniquely identify a config definition. + * @return The definition with a matching name and namespace + * @throws java.lang.IllegalArgumentException if def is not found. + */ + public final ConfigDefinition getConfigDefinition(ConfigDefinitionKey defKey) { + if (existingConfigDefs == null) { + existingConfigDefs = new LinkedHashMap<>(); + if (configDefinitionRepo.isPresent()) { + existingConfigDefs.putAll(createLazyMapping(configDefinitionRepo.get())); + } + existingConfigDefs.putAll(applicationPackage.getAllExistingConfigDefs()); + } + log.log(LogLevel.DEBUG, "Getting config definition " + defKey); + // Fall back to default namespace if not found. + if (defKey.getNamespace() == null || defKey.getNamespace().isEmpty()) { + defKey = new ConfigDefinitionKey(defKey.getName(), CNode.DEFAULT_NAMESPACE); + } + ConfigDefinitionKey lookupKey = defKey; + // Fall back to just using name + if (!existingConfigDefs.containsKey(lookupKey)) { + + int count = 0; + for (ConfigDefinitionKey entry : existingConfigDefs.keySet()) { + if (entry.getName().equals(defKey.getName())) { + count++; + } + } + if (count > 1) { + throw new IllegalArgumentException("Using config definition '" + defKey.getName() + "' is ambiguous, there are more than one config definitions with this name, please specify namespace"); + } + + lookupKey = null; + log.log(LogLevel.DEBUG, "Could not find config definition '" + defKey + "', trying with same name in all namespaces"); + for (ConfigDefinitionKey entry : existingConfigDefs.keySet()) { + if (entry.getName().equals(defKey.getName()) && defKey.getNamespace().equals(CNode.DEFAULT_NAMESPACE)) { + log.log(LogLevel.INFO, "Could not find config definition '" + defKey + "'" + + ", using config definition '" + entry + "' with same name instead (please use new namespace when specifying this config)"); + lookupKey = entry; + break; + } + } + } + + if (lookupKey == null) { + throw new IllegalArgumentException("Could not find a config definition with name '" + defKey + "'."); + } + if (defArchive.get(defKey) != null) { + log.log(LogLevel.DEBUG, "Found in archive: " + defKey); + return defArchive.get(defKey); + } + + log.log(LogLevel.DEBUG, "Retrieving config definition: " + defKey); + ConfigDefinition def = existingConfigDefs.get(lookupKey).parse(); + + log.log(LogLevel.DEBUG, "Adding " + def + " to archive"); + defArchive.put(defKey, def); + return def; + } + + private static Map<ConfigDefinitionKey, UnparsedConfigDefinition> createLazyMapping(final ConfigDefinitionRepo configDefinitionRepo) { + Map<ConfigDefinitionKey, UnparsedConfigDefinition> keyToRepo = new LinkedHashMap<>(); + for (final Map.Entry<ConfigDefinitionKey, com.yahoo.vespa.config.buildergen.ConfigDefinition> defEntry : configDefinitionRepo.getConfigDefinitions().entrySet()) { + keyToRepo.put(defEntry.getKey(), new UnparsedConfigDefinition() { + @Override + public ConfigDefinition parse() { + return ConfigDefinitionBuilder.createConfigDefinition(configDefinitionRepo.getConfigDefinitions().get(defEntry.getKey()).getCNode()); + } + + @Override + public String getUnparsedContent() { + throw new UnsupportedOperationException("Cannot get unparsed content from " + defEntry.getKey()); + } + }); + } + return keyToRepo; + } + + // Global registry of rank profiles. + // TODO: I think this can be removed when we remove "<search version=2.0>" and only support content. + private final RankProfileRegistry rankProfileRegistry; + + // Mapping from key to something that can create a config definition. + private Map<ConfigDefinitionKey, UnparsedConfigDefinition> existingConfigDefs = null; + + // Cache of config defs for all [def,version] combinations looked up so far. + private final Map<ConfigDefinitionKey, ConfigDefinition> defArchive = new LinkedHashMap<>(); + + public ApplicationPackage getApplicationPackage() { + return applicationPackage; + } + public List<SearchDefinition> getSearchDefinitions() { + return searchDefinitions; + } + + public DocumentModel getDocumentModel() { + return documentModel; + } + + public DeployLogger getDeployLogger() { + return logger; + } + + public FileRegistry getFileRegistry() { + return fileRegistry; + } + + public HostProvisioner getProvisioner() { return provisioner; } + + public Optional<ApplicationPackage> getPermanentApplicationPackage() { + return permanentApplicationPackage; + } + + public DeployProperties getProperties() { return properties; } + + public Optional<Model> getPreviousModel() { return previousModel; } + + public boolean isHostedVespa() { + return properties.hostedVespa(); + } + + public Set<Rotation> getRotations() { + return this.rotations; // todo: consider returning a copy or immutable view + } + + /** Returns the zone in which this is currently running */ + public Zone zone() { return zone; } + + public QueryProfiles getQueryProfiles() { return queryProfiles; } + + public SemanticRules getSemanticRules() { return semanticRules; } + + public static class Builder { + + private ApplicationPackage applicationPackage = MockApplicationPackage.createEmpty(); + private FileRegistry fileRegistry = new MockFileRegistry(); + private DeployLogger logger = new BaseDeployLogger(); + private Optional<HostProvisioner> hostProvisioner = Optional.empty(); + private Optional<ApplicationPackage> permanentApplicationPackage = Optional.empty(); + private DeployProperties properties = new DeployProperties.Builder().build(); + private Optional<ConfigDefinitionRepo> configDefinitionRepo = Optional.empty(); + private Optional<Model> previousModel = Optional.empty(); + private Set<Rotation> rotations = new HashSet<>(); + private Zone zone = Zone.defaultZone(); + private Instant now = Instant.now(); + + public Builder applicationPackage(ApplicationPackage applicationPackage) { + this.applicationPackage = applicationPackage; + return this; + } + + public Builder fileRegistry(FileRegistry fileRegistry) { + this.fileRegistry = fileRegistry; + return this; + } + + public Builder deployLogger(DeployLogger logger) { + this.logger = logger; + return this; + } + + public Builder modelHostProvisioner(HostProvisioner modelProvisioner) { + this.hostProvisioner = Optional.of(modelProvisioner); + return this; + } + + public Builder permanentApplicationPackage(Optional<ApplicationPackage> permanentApplicationPackage) { + this.permanentApplicationPackage = permanentApplicationPackage; + return this; + } + + public Builder properties(DeployProperties properties) { + this.properties = properties; + return this; + } + + public Builder configDefinitionRepo(ConfigDefinitionRepo configDefinitionRepo) { + this.configDefinitionRepo = Optional.of(configDefinitionRepo); + return this; + } + + public Builder previousModel(Model previousModel) { + this.previousModel = Optional.of(previousModel); + return this; + } + + public Builder rotations(Set<Rotation> rotations) { + this.rotations = rotations; + return this; + } + + public Builder zone(Zone zone) { + this.zone = zone; + return this; + } + + public Builder now(Instant now) { + this.now = now; + return this; + } + + public DeployState build() { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + QueryProfiles queryProfiles = new QueryProfilesBuilder().build(applicationPackage); + SemanticRules semanticRules = new SemanticRuleBuilder().build(applicationPackage); + SearchDocumentModel searchDocumentModel = createSearchDocumentModel(rankProfileRegistry, logger, queryProfiles); + return new DeployState(applicationPackage, searchDocumentModel, rankProfileRegistry, fileRegistry, logger, hostProvisioner, + properties, permanentApplicationPackage, configDefinitionRepo, previousModel, rotations, zone, queryProfiles, semanticRules, now); + } + + private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, DeployLogger logger, QueryProfiles queryProfiles) { + Collection<NamedReader> readers = applicationPackage.getSearchDefinitions(); + Map<String, String> names = new LinkedHashMap<>(); + SearchBuilder builder = new SearchBuilder(applicationPackage, rankProfileRegistry); + for (NamedReader reader : readers) { + try { + String readerName = reader.getName(); + String searchName = builder.importReader(reader, readerName, logger); + String sdName = stripSuffix(readerName, ApplicationPackage.SD_NAME_SUFFIX); + names.put(searchName, sdName); + if (!sdName.equals(searchName)) { + throw new IllegalArgumentException("Search definition file name ('" + sdName + "') and name of " + + "search element ('" + searchName + "') are not equal for file '" + readerName + "'"); + } + } catch (ParseException e) { + throw new IllegalArgumentException("Could not parse search definition file '" + getSearchDefinitionRelativePath(reader.getName()) + "': " + e.getMessage(), e); + } catch (IOException e) { + throw new IllegalArgumentException("Could not read search definition file '" + getSearchDefinitionRelativePath(reader.getName()) + "': " + e.getMessage(), e); + } finally { + closeIgnoreException(reader.getReader()); + } + } + builder.build(logger, queryProfiles); + return SearchDocumentModel.fromBuilderAndNames(builder, names); + } + + private String getSearchDefinitionRelativePath(String name) { + return ApplicationPackage.SEARCH_DEFINITIONS_DIR + File.separator + name; + } + + private static String stripSuffix(String nodeName, String postfix) { + assert (nodeName.endsWith(postfix)); + return nodeName.substring(0, nodeName.length() - postfix.length()); + } + + @SuppressWarnings("EmptyCatchBlock") + private static void closeIgnoreException(Reader reader) { + try { + reader.close(); + } catch(Exception e) {} + } + } + +} + diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java b/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java new file mode 100644 index 00000000000..1fbd72599fe --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.deploy; + +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.vespa.documentmodel.DocumentModel; +import com.yahoo.vespa.model.search.SearchDefinition; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Internal helper class to retrieve document model and search definitions. + * + * @author lulf + * @since 5.1 + */ +public class SearchDocumentModel { + + private final DocumentModel documentModel; + private final List<SearchDefinition> searchDefinitions; + + public SearchDocumentModel(DocumentModel documentModel, List<SearchDefinition> searchDefinitions) { + this.documentModel = documentModel; + this.searchDefinitions = searchDefinitions; + + } + + public DocumentModel getDocumentModel() { + return documentModel; + } + + public List<SearchDefinition> getSearchDefinitions() { + return searchDefinitions; + } + + public static SearchDocumentModel fromBuilderAndNames(SearchBuilder builder, Map<String, String> names) { + List<SearchDefinition> ret = new ArrayList<>(); + for (com.yahoo.searchdefinition.Search search : builder.getSearchList()) { + ret.add(new SearchDefinition(names.get(search.getName()), search)); + } + return new SearchDocumentModel(builder.getModel(), ret); + } + + public static SearchDocumentModel fromBuilder(SearchBuilder builder) { + List<SearchDefinition> ret = new ArrayList<>(); + for (com.yahoo.searchdefinition.Search search : builder.getSearchList()) { + ret.add(new SearchDefinition(search.getName(), search)); + } + return new SearchDocumentModel(builder.getModel(), ret); + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/package-info.java b/config-model/src/main/java/com/yahoo/config/model/deploy/package-info.java new file mode 100644 index 00000000000..3feb934615f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.config.model.deploy; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/config/model/graph/ModelGraph.java b/config-model/src/main/java/com/yahoo/config/model/graph/ModelGraph.java new file mode 100644 index 00000000000..a435733e60f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/graph/ModelGraph.java @@ -0,0 +1,80 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.graph; + +import com.yahoo.component.ComponentId; + +import java.util.*; + +/** + * A model graph contains the dependency graph of config models. + * + * @author lulf + * @since 5.1 + */ +public class ModelGraph { + + private final List<ModelNode> modelNodes; + private final List<ModelNode> roots; + public ModelGraph(List<ModelNode> modelNodes, List<ModelNode> roots) { + this.modelNodes = modelNodes; + this.roots = roots; + } + + /** + * Performs a topological sort ot the models stored in this graph. The algorithm is based on Kahn topological sort. + * + * @return a sorted list of {@link com.yahoo.config.model.graph.ModelNode} in dependency order. + */ + public List<ModelNode> topologicalSort() { + DependencyMap dependencyMap = new DependencyMap(modelNodes); + Queue<ModelNode> unprocessed = new LinkedList<>(); + unprocessed.addAll(roots); + List<ModelNode> sortedList = new ArrayList<>(); + while (!unprocessed.isEmpty()) { + ModelNode sortedNode = unprocessed.remove(); + sortedList.add(sortedNode); + for (ModelNode node : modelNodes) { + if (dependencyMap.dependsOn(node, sortedNode)) { + dependencyMap.removeDependency(node, sortedNode); + if (!dependencyMap.hasDependencies(node)) { + unprocessed.add(node); + } + } + } + } + for (ModelNode node : modelNodes) { + if (dependencyMap.hasDependencies(node)) { + throw new IllegalArgumentException("Unable to sort graph because it contains cycles"); + } + } + return sortedList; + } + + List<ModelNode> getNodes() { + return modelNodes; + } + + private static class DependencyMap { + private final Map<ComponentId, Set<ComponentId>> map = new LinkedHashMap<>(); + DependencyMap(List<ModelNode> modelNodes) { + for (ModelNode node : modelNodes) { + Set<ComponentId> ids = new LinkedHashSet<>(); + ids.addAll(node.listDependencyIds()); + map.put(node.id, ids); + } + } + + public boolean dependsOn(ModelNode node, ModelNode sortedNode) { + return map.get(node.id).contains(sortedNode.id); + } + + public void removeDependency(ModelNode node, ModelNode sortedNode) { + map.get(node.id).remove(sortedNode.id); + } + + public boolean hasDependencies(ModelNode node) { + return !map.get(node.id).isEmpty(); + } + } +} + diff --git a/config-model/src/main/java/com/yahoo/config/model/graph/ModelGraphBuilder.java b/config-model/src/main/java/com/yahoo/config/model/graph/ModelGraphBuilder.java new file mode 100644 index 00000000000..97d914c6012 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/graph/ModelGraphBuilder.java @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.graph; + +import com.yahoo.config.model.ConfigModel; +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; + +import java.util.ArrayList; +import java.util.List; + +/** + * Used to add builders and elements in addBuilder, and then build a dependency graph based on the + * constructor arguments. + * + * @author lulf + * @since 5.1 + */ +public class ModelGraphBuilder { + + private final List<ConfigModelBuilder<? extends ConfigModel>> builders = new ArrayList<>(); + + /** + * Add a {@link com.yahoo.config.model.builder.xml.ConfigModelBuilder} to this graph. + * + * @param builder The {@link com.yahoo.config.model.builder.xml.ConfigModelBuilder} to add. + * @return this for convenience + */ + public ModelGraphBuilder addBuilder(ConfigModelBuilder<? extends ConfigModel> builder) { + builders.add(builder); + return this; + } + + /** + * Build a {@link com.yahoo.config.model.graph.ModelGraph} based on the {@link com.yahoo.config.model.builder.xml.ConfigModelBuilder}s + * added to this. + * + * @return A {@link com.yahoo.config.model.graph.ModelGraph} representing the dependency graph. + */ + public ModelGraph build() { + List<ModelNode> modelNodes = new ArrayList<>(); + for (ConfigModelBuilder<? extends ConfigModel> builder : builders) { + modelNodes.add(new ModelNode(builder)); + } + List<ModelNode> roots = new ArrayList<>(); + for (ModelNode modelNode : modelNodes) { + int numDependencies = modelNode.addDependenciesFrom(modelNodes); + if (numDependencies == 0) { + roots.add(modelNode); + } + } + return new ModelGraph(modelNodes, roots); + } +} diff --git a/config-model/src/main/java/com/yahoo/config/model/graph/ModelNode.java b/config-model/src/main/java/com/yahoo/config/model/graph/ModelNode.java new file mode 100644 index 00000000000..0cd045b09c8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/graph/ModelNode.java @@ -0,0 +1,135 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.graph; + +import com.google.inject.Inject; +import com.yahoo.component.ComponentId; +import com.yahoo.config.model.ConfigModel; +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.config.model.ConfigModelInstanceFactory; + +import java.lang.reflect.*; +import java.util.*; + +/** + * Represents a node in the dependency graph, and contains information about a builders dependencies. + * Constructor signatures of model classes must have ConfigModelContext as the first argument and + * ConfigModel subclasses or Collection of ConfigModels as subsequent arguments. + * Only Collection, not Collection subtypes can be used. + * + * @author lulf + * @since 5.1 + */ +public class ModelNode<MODEL extends ConfigModel> implements ConfigModelInstanceFactory<MODEL> { + + final ComponentId id; + public final ConfigModelBuilder<MODEL> builder; + final Class<MODEL> clazz; + final Constructor<MODEL> constructor; + final List<MODEL> instances = new ArrayList<>(); + private final Map<ComponentId, ModelNode> dependencies = new HashMap<>(); + + public ModelNode(ConfigModelBuilder<MODEL> builder) { + this.id = builder.getId(); + this.builder = builder; + this.clazz = builder.getModelClass(); + this.constructor = findConstructor(clazz); + } + + private Constructor<MODEL> findConstructor(Class<MODEL> clazz) { + for (Constructor<?> ctor : clazz.getDeclaredConstructors()) { + if (ctor.getAnnotation(Inject.class) != null) { + return (Constructor<MODEL>) ctor; + } + } + return (Constructor<MODEL>) clazz.getDeclaredConstructors()[0]; + } + + boolean hasDependencies() { + return !dependencies.isEmpty(); + } + + boolean dependsOn(ModelNode node) { + return dependencies.containsKey(node.id); + } + + /** + * This adds dependencies base on constructor arguments in the model classes themselves. + * These then have to be created by this mini-di framework and then handed to the builders + * that will fill them. + * + * TODO: This should be changed to model dependencies between model builders instead, such + * that they can create their model objects, and eventually make them immutable. + */ + int addDependenciesFrom(List<ModelNode> modelNodes) { + int numDependencies = 0; + for (Type param : constructor.getGenericParameterTypes()) { + for (ModelNode node : modelNodes) { + if (param.equals(node.clazz) || isCollectionOf(param, node.clazz)) { + addDependency(node); + numDependencies++; + } + } + } + return numDependencies; + } + + private boolean isCollectionOf(Type type, Class<?> nodeClazz) { + if (type instanceof ParameterizedType) { + ParameterizedType t = (ParameterizedType) type; + // Note: IntelliJ says the following cannot be equal but that is wrong + return (t.getRawType().equals(java.util.Collection.class) && t.getActualTypeArguments().length == 1 && t.getActualTypeArguments()[0].equals(nodeClazz)); + } + return false; + } + + private boolean isCollection(Type type) { + if (type instanceof ParameterizedType) { + ParameterizedType t = (ParameterizedType) type; + // Note: IntelliJ says the following cannot be equal but that is wrong + return (t.getRawType().equals(java.util.Collection.class) && t.getActualTypeArguments().length == 1); + } + return false; + } + + private void addDependency(ModelNode node) { + dependencies.put(node.id, node); + } + + Collection<ComponentId> listDependencyIds() { + return dependencies.keySet(); + } + + @Override + public MODEL createModel(ConfigModelContext context) { + try { + Type [] params = constructor.getGenericParameterTypes(); + if (params.length < 1 || ! params[0].equals(ConfigModelContext.class)) { + throw new IllegalArgumentException("Constructor for " + clazz.getName() + " must have as its first argument a " + ConfigModelContext.class.getName()); + } + Object arguments[] = new Object[params.length]; + arguments[0] = context; + for (int i = 1; i < params.length; i++) + arguments[i] = findArgument(params[i]); + MODEL instance = constructor.newInstance(arguments); + instances.add(instance); + return instance; + } catch (InvocationTargetException | InstantiationException | IllegalAccessException e) { + throw new RuntimeException("Error constructing model '" + clazz.getName() + "'", e); + } + } + + private Object findArgument(Type param) { + for (ModelNode dependency : dependencies.values()) { + if (param.equals(dependency.clazz)) + return dependency.instances.get(0); + if (isCollectionOf(param, dependency.clazz)) + return Collections.unmodifiableCollection(dependency.instances); + } + // For collections, we don't require that dependency has been added, we just give an empty collection + if (isCollection(param)) + return Collections.emptyList(); + throw new IllegalArgumentException("Unable to find constructor argument " + param + " for " + clazz.getName()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/package-info.java b/config-model/src/main/java/com/yahoo/config/model/package-info.java new file mode 100644 index 00000000000..594f52c0d14 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.config.model; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java new file mode 100644 index 00000000000..41927bc09a9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java @@ -0,0 +1,448 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.producer; + +import com.google.common.annotations.Beta; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.ConfigInstanceUtil; +import com.yahoo.config.model.ApplicationConfigProducerRoot; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.log.LogLevel; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.*; +import com.yahoo.vespa.model.*; +import com.yahoo.vespa.model.admin.Admin; +import com.yahoo.vespa.model.admin.MonitoringSystem; +import com.yahoo.vespa.model.utils.FreezableMap; + +import java.io.*; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Type; +import java.util.*; +import java.util.logging.Logger; + +/** + * Superclass for all config producers. + * Config producers constructs and returns config instances on request. + * + * @author gjoranv + */ +public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProducer<?>> + implements ConfigProducer, ConfigInstance.Producer, Serializable { + + private static final long serialVersionUID = 1L; + public static final Logger log = Logger.getLogger(AbstractConfigProducer.class.getPackage().toString()); + private final String subId; + private final boolean hostedVespa; + private String configId = null; + + private List<Service> descendantServices = new ArrayList<>(); + + private AbstractConfigProducer parent = null; + + private UserConfigRepo userConfigs = new UserConfigRepo(); + + private final FreezableMap<String, CHILD> childrenBySubId = new FreezableMap<>(LinkedHashMap.class); + + private static boolean isHostedVespa(AbstractConfigProducer parent) { + return (parent != null) + && (parent.getRoot() != null) + && (parent.getRoot().getDeployState() != null) + && parent.getRoot().getDeployState().isHostedVespa(); + } + + /** + * Creates a new AbstractConfigProducer with the given parent and subId. + * This constructor will add the resulting producer to the children of parent. + * + * @param parent The parent of this ConfigProducer + * @param subId The fragment of the config id for the producer + */ + public AbstractConfigProducer(AbstractConfigProducer parent, String subId) { + this(subId, isHostedVespa(parent)); + + if (parent != null) { + parent.addChild(this); + } + } + + protected final void setParent(AbstractConfigProducer parent) { this.parent = parent; } + public final String getSubId() { return subId; } + public final boolean isHostedVespa() { return hostedVespa; } + + /** + * Create an config producer with a configId only. Used e.g. to create root nodes, and producers + * that are given children after construction using {@link #addChild(AbstractConfigProducer)}. + * + * @param subId The sub configId. Note that this can be prefixed when calling addChild with this producer as arg. + */ + public AbstractConfigProducer(String subId) { + this(subId, false); + } + + private AbstractConfigProducer(String subId, boolean hostedVespa) { + if (subId.indexOf('/') != -1) { + throw new IllegalArgumentException("A subId might not contain '/' : '" + subId + "'"); + } + this.subId = subId; + this.hostedVespa = hostedVespa; + } + + /** + * Adds a child to this config producer. + * + * @param child The child config producer to add. + */ + protected void addChild(CHILD child) { + if (child == null) { + throw new IllegalArgumentException("Trying to add null child for: " + this); + } + if (child instanceof AbstractConfigProducerRoot) { + throw new IllegalArgumentException("Child cannot be a root node: " + child); + } + + child.setParent(this); + if (childrenBySubId.get(child.getSubId()) != null) { + throw new IllegalArgumentException("Multiple services/instances of the id '" + child.getSubId() + "' under the service/instance " + + errorMsgClassName() + " '" + subId + "'. (This is commonly caused by service/node index " + + "collisions in the config.)." + + "\nExisting instance: " + childrenBySubId.get(child.getSubId()) + + "\nAttempted to add: " + child); + } + childrenBySubId.put(child.getSubId(), child); + + if (child instanceof Service) { + addDescendantService((Service)child); + } + } + + public void removeChild(CHILD child) { + if (child.getParent() != this) + throw new IllegalArgumentException("Could not remove " + child + ": Expected its parent to be " + + this + ", but was " + child.getParent()); + + if (child instanceof Service) + descendantServices.remove(child); + + childrenBySubId.remove(child.getSubId()); + child.setParent(null); + } + + /** + * Helper to provide an error message on collisions of sub ids (ignore SimpleConfigProducer, use the parent in that case) + */ + private String errorMsgClassName() { + if (getClass().equals(SimpleConfigProducer.class)) return parent.getClass().getSimpleName(); + return getClass().getSimpleName(); + } + + /** + * Sets the user configs for this producer. + * + * @param repo User configs repo. + */ + public void setUserConfigs(UserConfigRepo repo) { this.userConfigs = repo; } + + /** Returns the user configs of this */ + @Override + public UserConfigRepo getUserConfigs() { return userConfigs; } + + /** + * ConfigProducers that must have a special config id should use + * setConfigId() instead of overloading this method. This is + * because config IDs must be registered through setConfigId(). + */ + public final String getConfigId() { + if (configId == null) throw new RuntimeException("The system topology must be frozen first."); + return configId; + } + + /** + * Sets the config id for this producer. Will also add this + * service to the root node, so the new config id will be picked + * up. Note that this producer will be known with both the old + * and the new config id in the root node after using this method. + */ + protected void addConfigId(String id) { + if (id == null) throw new NullPointerException("Config ID cannot be null."); + getRoot().addDescendant(id, this); + if (!isVespa() && (getVespa() != null)) + getVespa().addDescendant(this); + } + + /** Returns this ConfigProducer's children (only 1st level) */ + public Map<String, CHILD> getChildren() { return Collections.unmodifiableMap(childrenBySubId); } + + @Beta + public <J extends AbstractConfigProducer<?>> List<J> getChildrenByTypeRecursive(Class<J> type) { + List<J> validChildren = new ArrayList<>(); + + if (this.getClass().equals(type)) { + validChildren.add(type.cast(this)); + } + + Map<String, ? extends AbstractConfigProducer<?>> children = this.getChildren(); + for (AbstractConfigProducer<?> child : children.values()) { + validChildren.addAll(child.getChildrenByTypeRecursive(type)); + } + + return Collections.unmodifiableList(validChildren); + } + + /** Returns a list of all the children of this who are instances of Service */ + public List<Service> getDescendantServices() { return Collections.unmodifiableList(descendantServices); } + + protected void addDescendantService(Service s) { descendantServices.add(s); } + + @Override + public final boolean cascadeConfig(ConfigInstance.Builder builder) { + boolean found=false; + if (parent != null) + found = parent.cascadeConfig(builder); + + boolean foundHere = builder.dispatchGetConfig(this); + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "cascadeconfig in " + this + ", getting config " + + builder.getClass().getDeclaringClass().getName() + " for config id '" + configId + "' found here=" + foundHere); + } + found = found || foundHere; + return found; + } + + @Override + public final boolean addUserConfig(ConfigInstance.Builder builder) { + boolean didApply = false; + if (parent != null) { + didApply = parent.addUserConfig(builder); + } + + if (log.isLoggable(LogLevel.SPAM)) { + log.log(LogLevel.SPAM, "User configs is: " + userConfigs.toString()); + } + // TODO: What do we do with md5. Currently ignored for user configs? + ConfigDefinitionKey key = new ConfigDefinitionKey(builder.getDefName(), builder.getDefNamespace()); + if (userConfigs.get(key) != null) { + if (log.isLoggable(LogLevel.SPAM)) { + log.log(LogLevel.SPAM, "Apply in " + configId); + } + applyUserConfig(builder, userConfigs.get(key)); + didApply = true; + } + return didApply; + } + + private void applyUserConfig(ConfigInstance.Builder builder, ConfigPayloadBuilder payloadBuilder) { + ConfigInstance.Builder override; + if (builder instanceof GenericConfig.GenericConfigBuilder) { + // Means that the builder is unknown and that we should try to apply the payload without + // the real builder + override = getGenericConfigBuilderOverride((GenericConfig.GenericConfigBuilder) builder, payloadBuilder); + } else { + override = getConfigInstanceBuilderOverride(builder, ConfigPayload.fromBuilder(payloadBuilder)); + } + ConfigInstanceUtil.setValues(builder, override); + } + + private ConfigInstance.Builder getGenericConfigBuilderOverride(GenericConfig.GenericConfigBuilder builder, ConfigPayloadBuilder payloadBuilder) { + ConfigDefinitionKey key = new ConfigDefinitionKey(builder.getDefName(), builder.getDefNamespace()); + return new GenericConfig.GenericConfigBuilder(key, payloadBuilder); + } + + private ConfigInstance.Builder getConfigInstanceBuilderOverride(ConfigInstance.Builder builder, ConfigPayload payload) { + try { + ConfigTransformer transformer = new ConfigTransformer(builder.getClass().getEnclosingClass()); + return transformer.toConfigBuilder(payload); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Error applying override to builder", e); + } + } + + /** + * Returns the one and only HostSystem of the root node + * Must be overridden by root node. + */ + public HostSystem getHostSystem() { + return getRoot().getHostSystem(); + } + + public AbstractConfigProducerRoot getRoot() { + return parent == null ? null : parent.getRoot(); + } + + /** + * Returns the {@link ApplicationConfigProducerRoot} that is the parent of this sub-tree, or null + * if this sub-tree has no Vespa parent. + */ + private ApplicationConfigProducerRoot getVespa() { + if (isRoot()) return null; + return isVespa() ? (ApplicationConfigProducerRoot)this : parent.getVespa(); + } + + private boolean isRoot() { + return parent == null; + } + + private boolean isVespa() { + return ((this instanceof ApplicationConfigProducerRoot) && parent.isRoot()); + } + + public AbstractConfigProducer getParent() { return parent; } + + /** + * Writes files that need to be written. The files will usually only be + * written when the Vespa model is generated through the deploy-application + * script. + * + * TODO: Make sure all implemented ConfigProducers call createConfig() + * instead of getConfig() when implementing this method. + */ + public void writeFiles(File directory) throws java.io.IOException { + if (!directory.isDirectory() && !directory.mkdirs()) { + throw new java.io.IOException("Cannot create directory: "+ directory); + } + for (Method m : getClass().getMethods()) { + try { + ConfigInstance.Builder builder = getBuilderIfIsGetConfig(m); + if (builder!=null) { + writeBuilder(directory, m, builder); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + + private void writeBuilder(File directory, Method m, + ConfigInstance.Builder builder) throws IllegalAccessException, + InvocationTargetException, InstantiationException, + NoSuchMethodException, IOException { + m.invoke(this, builder); + Class<?> configInstClass = builder.getClass().getEnclosingClass(); + ConfigInstance inst = (ConfigInstance) configInstClass.getConstructor(builder.getClass()).newInstance(builder); + List<String> payloadList = ConfigInstance.serialize(inst); + File outfn = new File(directory, ConfigInstance.getDefName(inst.getClass()) + ".MODEL.cfg"); + FileOutputStream out = new FileOutputStream(outfn); + for (String s : payloadList) { + out.write(Utf8.toBytes(s)); + out.write('\n'); + } + } + + /** + * New Builder instance if m is getConfig(SomeConfig.Builder), or null + */ + private ConfigInstance.Builder getBuilderIfIsGetConfig(Method m) throws InstantiationException, IllegalAccessException { + if (!"getConfig".equals(m.getName())) return null; + Type[] params = m.getParameterTypes(); + if (params.length!=1) return null; + Type param = params[0]; + if (!(param instanceof Class)) return null; + Class<?> paramClass = (Class<?>) param; + if (!(ConfigInstance.Builder.class.isAssignableFrom(paramClass))) return null; + return (ConfigInstance.Builder) paramClass.newInstance(); + } + + public void dump(PrintStream out) { + for (ConfigProducer c : getChildren().values()) { + out.println("id: " + c.getConfigId()); + if (c.getChildren().size() > 0) { + c.dump(out); + } + } + } + + void setupConfigId(String parentConfigId) { + if (this instanceof AbstractConfigProducerRoot) { + configId = ""; + } else { + configId = parentConfigId + subId; + addConfigId(configId); + } + + if (this instanceof AbstractConfigProducerRoot || this instanceof ApplicationConfigProducerRoot) { + setupChildConfigIds(""); + } else { + setupChildConfigIds(configId + '/'); + } + } + + private static ClassLoader findInheritedClassLoader(Class clazz, String producerName) { + Class<?>[] interfazes = clazz.getInterfaces(); + for (Class interfaze : interfazes) { + if (producerName.equals(interfaze.getName())) { + return interfaze.getClassLoader(); + } + } + if (clazz.getSuperclass() == null) + return null; + return findInheritedClassLoader(clazz.getSuperclass(), producerName); + } + + public ClassLoader getConfigClassLoader(String producerName) { + ClassLoader classLoader = findInheritedClassLoader(getClass(), producerName); + if (classLoader != null) + return classLoader; + + // TODO: Make logic correct, so that the deepest child will be the one winning. + for (AbstractConfigProducer child : childrenBySubId.values()) { + ClassLoader loader = child.getConfigClassLoader(producerName); + if (loader != null) { + return loader; + } + } + return null; + } + + private void setupChildConfigIds(String currentConfigId) { + for (AbstractConfigProducer child : childrenBySubId.values()) { + child.setupConfigId(currentConfigId); + } + } + + void aggregateDescendantServices() { + for (AbstractConfigProducer child : childrenBySubId.values()) { + child.aggregateDescendantServices(); + descendantServices.addAll(child.descendantServices); + } + } + + void freeze() { + childrenBySubId.freeze(); + for (AbstractConfigProducer child : childrenBySubId.values()) { + child.freeze(); + } + } + + public void mergeUserConfigs(UserConfigRepo newRepo) { + userConfigs.merge(newRepo); + } + + @Override + public void validate() throws Exception { + assert (childrenBySubId.isFrozen()); + + for (AbstractConfigProducer child : childrenBySubId.values()) { + child.validate(); + } + } + + /** Returns a logger to be used for warnings and messages during initialization, never null */ + public DeployLogger deployLogger() { + return parent.deployLogger(); + } + + // TODO: Make producers depend on AdminModel instead + /** Returns a monitoring service (yamas if that is configured, null otherwise) */ + protected MonitoringSystem getMonitoringService() { + AbstractConfigProducerRoot root = getRoot(); + Admin admin = (root == null? null : root.getAdmin()); + if (admin == null) { + return null; + } + if (admin.getYamas() != null) { + return admin.getYamas(); + } + return null; + } +} diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducerRoot.java new file mode 100644 index 00000000000..b85e2b8fa97 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducerRoot.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.producer; + +import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.vespa.model.ConfigProducer; +import com.yahoo.vespa.model.ConfigProducerRoot; +import com.yahoo.vespa.model.Service; +import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; + +/** + * The parent class of classes having the role as the root of a config producer tree. + * + * @author tonytv + */ +public abstract class AbstractConfigProducerRoot extends AbstractConfigProducer<AbstractConfigProducer<?>> + implements ConfigProducerRoot { + + /** The ConfigProducers contained in this model indexed by config id */ + protected final Map<String, ConfigProducer> id2producer = new LinkedHashMap<>(); + + public AbstractConfigProducerRoot(String rootConfigId) { + super(rootConfigId); + } + + public AbstractConfigProducerRoot getRoot() { + return this; + } + + public abstract FileDistributionConfigProducer getFileDistributionConfigProducer(); + + /** + * Freezes the parent - child connections of the model + * and sets information derived from the topology. + */ + public void freezeModelTopology() { + freeze(); + setupConfigId(""); + aggregateDescendantServices(); + } + + public abstract ConfigModelRepo configModelRepo(); + + /** + * Returns the ConfigProducer with the given id if such configId exists. + * + * @param configId The configId, e.g. "search.0/tld.0" + * @return ConfigProducer with the given configId + */ + public Optional<ConfigProducer> getConfigProducer(String configId) { + return Optional.ofNullable(id2producer.get(configId)); + } + + /** + * Returns the Service with the given id if such configId exists and it belongs to a Service ConfigProducer. + * + * @param configId The configId, e.g. "search.0/tld.0" + * @return Service with the given configId + */ + public Optional<Service> getService(String configId) { + return getConfigProducer(configId) + .filter(Service.class::isInstance) + .map(Service.class::cast); + } +} diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/UserConfigRepo.java b/config-model/src/main/java/com/yahoo/config/model/producer/UserConfigRepo.java new file mode 100644 index 00000000000..d32b0c75e69 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/producer/UserConfigRepo.java @@ -0,0 +1,109 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.producer; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.*; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +/** + * A UserConfigRepo is a repository for user configs, typically for a particular config producer. The repo encapsulates + * how the user configs are stored, and defines the methods to retrieve user configs and merge the repo with others. + * + * @author lulf + * @since 5.1 + */ +public class UserConfigRepo { + private final Map<ConfigDefinitionKey, ConfigPayloadBuilder> userConfigsMap; + + public UserConfigRepo() { + this.userConfigsMap = new LinkedHashMap<>(); + } + + @Override + public UserConfigRepo clone() { + return new UserConfigRepo(copyBuilders(userConfigsMap)); + } + + /** + * Must copy the builder, because the merge method on {@link AbstractConfigProducer} might override the row's builders otherwise + */ + private Map<ConfigDefinitionKey, ConfigPayloadBuilder> copyBuilders(Map<ConfigDefinitionKey, ConfigPayloadBuilder> source) { + Map<ConfigDefinitionKey, ConfigPayloadBuilder> ret = new LinkedHashMap<>(); + for (Map.Entry<ConfigDefinitionKey, ConfigPayloadBuilder> e : source.entrySet()) { + ConfigDefinitionKey key = e.getKey(); + ConfigPayloadBuilder sourceVal = e.getValue(); + ConfigPayloadBuilder destVal = new ConfigPayloadBuilder(ConfigPayload.fromBuilder(sourceVal)); + ret.put(key, destVal); + } + return ret; + } + + public UserConfigRepo(Map<ConfigDefinitionKey, ConfigPayloadBuilder> map) { + this.userConfigsMap = map; + } + + public UserConfigRepo(UserConfigRepo userConfigRepo) { + this.userConfigsMap = userConfigRepo.userConfigsMap; + } + + public ConfigPayloadBuilder get(ConfigDefinitionKey key) { + return userConfigsMap.get(key); + } + + public void merge(UserConfigRepo newRepo) { + for (Map.Entry<ConfigDefinitionKey, ConfigPayloadBuilder> entry : newRepo.userConfigsMap.entrySet()) { + if (entry.getValue() == null) continue; + + ConfigDefinitionKey key = entry.getKey(); + if (userConfigsMap.containsKey(key)) { + ConfigPayloadBuilder lhsBuilder = userConfigsMap.get(key); + ConfigPayloadBuilder rhsBuilder = entry.getValue(); + lhsBuilder.override(rhsBuilder); + } else { + userConfigsMap.put(key, entry.getValue()); + } + } + } + + public boolean isEmpty() { + return userConfigsMap.isEmpty(); + } + + public int size() { + return userConfigsMap.size(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (ConfigDefinitionKey key : userConfigsMap.keySet()) { + sb.append(key.toString()); + } + return sb.toString(); + } + + /** + * The keys of all the configs contained in this. + * @return a set of ConfigDefinitionsKey + */ + public Set<ConfigDefinitionKey> configsProduced() { + return userConfigsMap.keySet(); + } + + /** + * Will take the warning messages stored on the payload builders, and apply them to the producer's {@link DeployLogger} + * @param producer the producer to apply warnings to + */ + public void applyWarnings(AbstractConfigProducer<?> producer) { + for (ConfigPayloadBuilder b : userConfigsMap.values()) { + for (String warning : b.warnings()) { + producer.deployLogger().log(LogLevel.WARNING, warning); + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/package-info.java b/config-model/src/main/java/com/yahoo/config/model/producer/package-info.java new file mode 100644 index 00000000000..c58a116b196 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/producer/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.config.model.producer; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/Host.java b/config-model/src/main/java/com/yahoo/config/model/provision/Host.java new file mode 100644 index 00000000000..9d97308fc76 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/provision/Host.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.provision; + +import java.util.ArrayList; +import java.util.List; + +/** + * A hostname with zero or more aliases. + * + * @author musum + */ +public class Host { + + private final String hostname; + private final List<String> hostAliases; + + public Host(String hostname) { + this.hostname = hostname; + this.hostAliases = new ArrayList<>(); + } + + public Host(String hostname, List<String> hostAliases) { + this.hostname = hostname; + this.hostAliases = hostAliases; + } + + public String getHostname() { + return hostname; + } + + public List<String> getHostAliases() { + return hostAliases; + } + + @Override + public String toString() { + return hostname + " (aliases: " + hostAliases + ")"; + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java new file mode 100644 index 00000000000..ece781dd5bd --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java @@ -0,0 +1,120 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.provision; + +import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.log.LogLevel; +import com.yahoo.net.HostName; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.io.Reader; +import java.util.*; +import java.util.logging.Logger; + +/** + * TODO: What is this? + * + * @author musum + */ +public class Hosts { + + public static final Logger log = Logger.getLogger(Hosts.class.getPackage().toString()); + + private final HashMap<String, Host> hosts = new LinkedHashMap<>(); + private final Map<String, String> alias2hostname = new LinkedHashMap<>(); + private final Map<String, Host> alias2host = new LinkedHashMap<>(); + + /** + * Builds host system from a hosts.xml file + * + * @param hostsFile a reader for host from application package + * @return the HostSystem for this application package + */ + public static Hosts getHosts(Reader hostsFile) { + Hosts hosts = new Hosts(); + Document doc; + try { + doc = XmlHelper.getDocumentBuilder().parse(new InputSource(hostsFile)); + } catch (SAXException | IOException e) { + throw new IllegalArgumentException(e); + } + for (Element hostE : XML.getChildren(doc.getDocumentElement(), "host")) { + String name = hostE.getAttribute("name"); + if (name.equals("")) { + throw new RuntimeException("Missing 'name' attribute for host."); + } + if ("localhost".equals(name)) { + name = HostName.getLocalhost(); + } + final List<String> hostAliases = VespaDomBuilder.getHostAliases(hostE.getChildNodes()); + if (hostAliases.isEmpty()) { + throw new IllegalArgumentException("No host aliases defined for host '" + name + "'"); + } + Host host = new Host(name, hostAliases); + hosts.addHost(host, hostAliases); + } + log.log(LogLevel.DEBUG, "Created hosts:" + hosts); + return hosts; + } + + public Collection<Host> getHosts() { + return hosts.values(); + } + + /** + * Adds one host to this host system. + * + * @param host The host to add + * @param aliases The aliases for this host. + */ + public void addHost(Host host, List<String> aliases) { + hosts.put(host.getHostname(), host); + if ((aliases != null) && (aliases.size() > 0)) { + addHostAliases(aliases, host); + } + } + + /** + * Add all aliases for one host + * + * @param hostAliases A list of host aliases + * @param host The Host instance to add the alias for + */ + private void addHostAliases(List<String> hostAliases, Host host) { + if (hostAliases.size() < 1) { + throw new RuntimeException("Host '" + host.getHostname() + "' must have at least one <alias> tag."); + } + for (String alias : hostAliases) { + addHostAlias(alias, host); + } + } + + /** + * Adds an alias for the given host + * + * @param alias alias (string) for a Host + * @param host the {@link Host} to add the alias for + */ + protected void addHostAlias(String alias, Host host) { + if (alias2hostname.containsKey(alias)) { + throw new RuntimeException("Alias '" + alias + "' must be used for only one host!"); + } + alias2hostname.put(alias, host.getHostname()); + alias2host.put(alias, host); + } + + public Map<String, Host> getAlias2host() { + return alias2host; + } + + @Override + public String toString() { + return "Hosts: " + hosts.keySet(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java new file mode 100644 index 00000000000..20dc190d8e7 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.provision; + +import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.provision.*; +import com.yahoo.vespa.model.container.Container; + +import java.io.Reader; +import java.util.List; + +/** + * A host provisioner based on a hosts.xml file. + * No state in this provisioner, i.e it does not know anything about the active + * application if one exists. Pre-condition: A valid hosts file. + * + * @author musum + * @since 5.11 + */ +public class HostsXmlProvisioner implements HostProvisioner { + + private final Hosts hosts; + public static final String IMPLICIT_ADMIN_HOSTALIAS = "INTERNAL_VESPA_IMPLICIT_ADMIN"; + + public HostsXmlProvisioner(Reader hosts) { + this.hosts = Hosts.getHosts(hosts); + } + + @Override + public HostSpec allocateHost(String alias) { + /** + * Some special rules to allow no admin elements as well + * as jdisc element without nodes. + */ + if (alias.equals(IMPLICIT_ADMIN_HOSTALIAS)) { + if (hosts.getHosts().size() > 1) { + throw new IllegalArgumentException("More than 1 host specified (" + hosts.getHosts().size() + ") and <admin> not specified"); + } else { + return host2HostSpec(getFirstHost()); + } + } else if (alias.equals(Container.SINGLENODE_CONTAINER_SERVICESPEC)) { + return host2HostSpec(getFirstHost()); + } + for (Host host : hosts.getHosts()) { + if (host.getHostAliases().contains(alias)) { + return new HostSpec(host.getHostname(), host.getHostAliases()); + } + } + throw new IllegalArgumentException("Unable to find host for alias '" + alias + "'"); + } + + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity quantity, int groups, ProvisionLogger logger) { + throw new UnsupportedOperationException(); + } + + private HostSpec host2HostSpec(Host host) { + return new HostSpec(host.getHostname(), host.getHostAliases()); + } + + private Host getFirstHost() { + return hosts.getHosts().iterator().next(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java new file mode 100644 index 00000000000..c06efece329 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -0,0 +1,164 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.provision; + +import com.yahoo.collections.ListMap; +import com.yahoo.collections.Pair; +import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.provision.*; + +import java.util.*; + +/** + * In memory host provisioner. NB! ATM cannot be reused after allocate has been called. + * + * @author musum + * @author bratseth + */ +public class InMemoryProvisioner implements HostProvisioner { + + /** + * If this is true an exception is thrown when all nodes are used. + * If false this will simply return nodes best effort, preferring to satisfy the + * number of groups requested when possible. + */ + private final boolean failOnOutOfCapacity; + + /** Hosts which should be returned as retired */ + private final Set<String> retiredHostNames; + + /** Free hosts of each flavor */ + private final ListMap<String, Host> freeNodes = new ListMap<>(); + private final Map<String, HostSpec> legacyMapping = new LinkedHashMap<>(); + private final Map<ClusterSpec, List<HostSpec>> allocations = new LinkedHashMap<>(); + + /** Indexes must be unique across all groups in a cluster */ + private final Map<Pair<ClusterSpec.Type,ClusterSpec.Id>, Integer> nextIndexInCluster = new HashMap<>(); + + /** Use this index as start index for all clusters */ + private final int startIndexForClusters; + + /** Creates this with a number of nodes of the flavor 'default' */ + public InMemoryProvisioner(int nodeCount) { + this(Collections.singletonMap("default", createHostInstances(nodeCount)), true, 0); + } + + /** Creates this with a set of host names of the flavor 'default' */ + public InMemoryProvisioner(boolean failOnOutOfCapacity, String... hosts) { + this(Collections.singletonMap("default", toHostInstances(hosts)), failOnOutOfCapacity, 0); + } + + /** Creates this with a set of hosts of the flavor 'default' */ + public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, String ... retiredHostNames) { + this(Collections.singletonMap("default", hosts.getHosts()), failOnOutOfCapacity, 0, retiredHostNames); + } + + /** Creates this with a set of hosts of the flavor 'default' */ + public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) { + this(Collections.singletonMap("default", hosts.getHosts()), failOnOutOfCapacity, startIndexForClusters, retiredHostNames); + } + + public InMemoryProvisioner(Map<String, Collection<Host>> hosts, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) { + this.failOnOutOfCapacity = failOnOutOfCapacity; + for (Map.Entry<String, Collection<Host>> hostsOfFlavor : hosts.entrySet()) + for (Host host : hostsOfFlavor.getValue()) + freeNodes.put(hostsOfFlavor.getKey(), host); + this.retiredHostNames = new HashSet<>(Arrays.asList(retiredHostNames)); + this.startIndexForClusters = startIndexForClusters; + } + + private static Collection<Host> toHostInstances(String[] hostnames) { + List<Host> hosts = new ArrayList<>(); + for (String hostname : hostnames) { + hosts.add(new Host(hostname)); + } + return hosts; + } + + private static Collection<Host> createHostInstances(int hostCount) { + List<Host> hosts = new ArrayList<>(); + for (int i = 1; i <= hostCount; i++) { + hosts.add(new Host("host" + i)); + } + return hosts; + } + + @Override + public HostSpec allocateHost(String alias) { + if (legacyMapping.containsKey(alias)) return legacyMapping.get(alias); + List<Host> defaultHosts = freeNodes.get("default"); + if (defaultHosts.isEmpty()) throw new IllegalArgumentException("No more hosts of default flavor available"); + Host newHost = freeNodes.removeValue("default", 0); + HostSpec hostSpec = new HostSpec(newHost.getHostname(), newHost.getHostAliases()); + legacyMapping.put(alias, hostSpec); + return hostSpec; + } + + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity requestedCapacity, int groups, ProvisionLogger logger) { + if (cluster.group().isPresent() && groups > 1) + throw new IllegalArgumentException("Cannot both be specifying a group and ask for groups to be created"); + if (requestedCapacity.nodeCount() % groups != 0) + throw new IllegalArgumentException("Requested " + requestedCapacity.nodeCount() + " nodes in " + + groups + " groups, but the node count is not divisible into this number of groups"); + + int capacity = failOnOutOfCapacity ? requestedCapacity.nodeCount() : + Math.min(requestedCapacity.nodeCount(), freeNodes.get("default").size() + totalAllocatedTo(cluster)); + if (groups > capacity) + groups = capacity; + + String flavor = requestedCapacity.flavor().orElse("default"); + + List<HostSpec> allocation = new ArrayList<>(); + if (groups == 1) { + allocation.addAll(allocateHostGroup(cluster, flavor, capacity, startIndexForClusters)); + } + else { + for (int i = 0; i < groups; i++) { + allocation.addAll(allocateHostGroup(cluster.changeGroup(Optional.of(ClusterSpec.Group.from(String.valueOf(i)))), + flavor, + capacity / groups, + allocation.size())); + } + } + for (ListIterator<HostSpec> i = allocation.listIterator(); i.hasNext(); ) { + HostSpec host = i.next(); + if (retiredHostNames.contains(host.hostname())) + i.set(retire(host)); + } + return allocation; + } + + private HostSpec retire(HostSpec host) { + return new HostSpec(host.hostname(), host.aliases(), host.membership().get().retire()); + } + + private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, String flavor, int nodesInGroup, int startIndex) { + List<HostSpec> allocation = allocations.getOrDefault(clusterGroup, new ArrayList<>()); + allocations.put(clusterGroup, allocation); + + int nextIndex = nextIndexInCluster.getOrDefault(new Pair<>(clusterGroup.type(), clusterGroup.id()), startIndex); + while (allocation.size() < nodesInGroup) { + if (freeNodes.get(flavor).isEmpty()) throw new IllegalArgumentException("No nodes of flavor '" + flavor + "' available"); + Host newHost = freeNodes.removeValue(flavor, 0); + ClusterMembership membership = ClusterMembership.from(clusterGroup, nextIndex++); + allocation.add(new HostSpec(newHost.getHostname(), newHost.getHostAliases(), membership)); + } + nextIndexInCluster.put(new Pair<>(clusterGroup.type(), clusterGroup.id()), nextIndex); + + while (allocation.size() > nodesInGroup) + allocation.remove(0); + + return allocation; + } + + private int totalAllocatedTo(ClusterSpec cluster) { + int count = 0; + for (Map.Entry<ClusterSpec, List<HostSpec>> allocation : allocations.entrySet()) { + if ( ! allocation.getKey().type().equals(cluster.type())) continue; + if ( ! allocation.getKey().id().equals(cluster.id())) continue; + count += allocation.getValue().size(); + } + return count; + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java new file mode 100644 index 00000000000..38ed728e4e8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.provision; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.provision.*; +import com.yahoo.net.HostName; +import com.yahoo.vespa.model.HostSystem; + +import java.util.ArrayList; +import java.util.List; +import java.net.UnknownHostException; + +/** + * A host provisioner used when there is no hosts.xml file (using localhost as the only host) + * No state in this provisioner, i.e it does not know anything about the active + * application if one exists. + * + * @author musum + * @since 5.11 + */ +public class SingleNodeProvisioner implements HostProvisioner { + + private final Host host; // the only host in this system + private final HostSpec hostSpec; + private int counter = 0; + + public SingleNodeProvisioner() { + try { + host = new Host(HostSystem.lookupCanonicalHostname(HostName.getLocalhost())); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } + this.hostSpec = new HostSpec(host.getHostname(), host.getHostAliases()); + } + + @Override + public HostSpec allocateHost(String alias) { + return hostSpec; + } + + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { // TODO: This should fail if capacity requested is more than 1 + List<HostSpec> hosts = new ArrayList<>(); + hosts.add(new HostSpec(host.getHostname(), host.getHostAliases(), ClusterMembership.from(cluster, counter++))); + return hosts; + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/package-info.java b/config-model/src/main/java/com/yahoo/config/model/provision/package-info.java new file mode 100644 index 00000000000..a39e4025efa --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/provision/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.config.model.provision; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/config/model/test/ConfigModelTestUtil.java b/config-model/src/main/java/com/yahoo/config/model/test/ConfigModelTestUtil.java new file mode 100644 index 00000000000..b75147774ab --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/test/ConfigModelTestUtil.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.test; + +import com.yahoo.collections.CollectionUtil; +import com.yahoo.config.model.builder.xml.XmlHelper; +import org.w3c.dom.Element; +import org.xml.sax.InputSource; + +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * @author lulf + * @since 5.1 + */ +public class ConfigModelTestUtil { + /** + * @param xmlLines XML with " replaced with ' + */ + public static Element parse(String... xmlLines) { + List<String> lines = new ArrayList<>(); + lines.add("<?xml version='1.0' encoding='utf-8' ?>"); + lines.addAll(Arrays.asList(xmlLines)); + + try { + return XmlHelper.getDocumentBuilder().parse( + inputSource((CollectionUtil.mkString(lines, "\n").replace("'", "\"")))) + .getDocumentElement(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static InputSource inputSource(String str) { + return new InputSource(new StringReader(str)); + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java new file mode 100644 index 00000000000..731410c9bf3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -0,0 +1,243 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.test; + +import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.UnparsedConfigDefinition; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.provision.Version; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.searchdefinition.*; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.config.application.api.ApplicationPackage; + +import java.io.File; +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.util.*; + +/** + * For testing purposes only + * + * @author tonytv + */ +public class MockApplicationPackage implements ApplicationPackage { + public static final String MUSIC_SEARCHDEFINITION = createSearchDefinition("music", "foo"); + public static final String BOOK_SEARCHDEFINITION = createSearchDefinition("book", "bar"); + + private final String hostsS; + private final String servicesS; + private final List<String> searchDefinitions; + private final String searchDefinitionDir; + private final Optional<String> deploymentInfo; + private final Optional<String> validationOverrides; + private final boolean failOnValidateXml; + + private MockApplicationPackage(String hosts, String services, List<String> searchDefinitions, String searchDefinitionDir, + String deploymentInfo, String validationOverrides, boolean failOnValidateXml) { + this.hostsS = hosts; + this.servicesS = services; + this.searchDefinitions = searchDefinitions; + this.searchDefinitionDir = searchDefinitionDir; + this.deploymentInfo = Optional.ofNullable(deploymentInfo); + this.validationOverrides = Optional.ofNullable(validationOverrides); + this.failOnValidateXml = failOnValidateXml; + } + + @Override + public String getApplicationName() { + return "mock application"; + } + + @Override + public Reader getServices() { + return new StringReader(servicesS); + } + + @Override + public Reader getHosts() { + if (hostsS==null) return null; + return new StringReader(hostsS); + } + + @Override + public List<NamedReader> getSearchDefinitions() { + ArrayList<NamedReader> readers = new ArrayList<>(); + SearchBuilder searchBuilder = new SearchBuilder(this, new RankProfileRegistry()); + for (String sd : searchDefinitions) { + try { + String name = searchBuilder.importString(sd); + readers.add(new NamedReader(name + ApplicationPackage.SD_NAME_SUFFIX, new StringReader(sd))); + } catch (ParseException e) { + throw new RuntimeException(e); + } + } + return readers; + } + + @Override + public List<NamedReader> searchDefinitionContents() { + return new ArrayList<>(); + } + + @Override + public Map<ConfigDefinitionKey, UnparsedConfigDefinition> getAllExistingConfigDefs() { + return Collections.emptyMap(); + } + + @Override + public List<NamedReader> getFiles(Path dir,String fileSuffix,boolean recurse) { + return new ArrayList<>(); + } + + @Override + public ApplicationFile getFile(Path file) { + throw new UnsupportedOperationException(); + } + + @Override + public String getHostSource() { + return "mock source"; + } + + @Override + public String getServicesSource() { + return "mock source"; + } + + @Override + public Optional<Reader> getDeployment() { + return deploymentInfo.map(StringReader::new); + } + + @Override + public Optional<Reader> getValidationOverrides() { + return validationOverrides.map(StringReader::new); + } + + public List<ComponentInfo> getComponentsInfo(Version vespaVersion) { + return Collections.emptyList(); + } + + @Override + public Reader getRankingExpression(String name) { + File expressionFile = new File(searchDefinitionDir, name); + try { + return IOUtils.createReader(expressionFile, "utf-8"); + } + catch (IOException e) { + throw new IllegalArgumentException("Could not read ranking expression file '" + + expressionFile.getAbsolutePath() + "'", e); + } + } + + public static ApplicationPackage createEmpty() { + return new MockApplicationPackage.Builder().withHosts(emptyHosts).withServices(emptyServices).build(); + } + + public static ApplicationPackage fromSearchDefinitionDirectory(String dir) { + return new MockApplicationPackage.Builder() + .withEmptyHosts() + .withEmptyServices() + .withSearchDefinitionDir(dir).build(); + } + + public static class Builder { + private String hosts = null; + private String services = null; + private List<String> searchDefinitions = Collections.emptyList(); + private String searchDefinitionDir = null; + private String deploymentInfo = null; + private String validationOverrides = null; + private boolean failOnValidateXml = false; + + public Builder() { + } + + public Builder withEmptyHosts() { + return this.withHosts(emptyHosts); + } + + public Builder withHosts(String hosts) { + this.hosts = hosts; + return this; + } + + public Builder withEmptyServices() { + return this.withServices(emptyServices); + } + + public Builder withServices(String services) { + this.services = services; + return this; + } + + public Builder withSearchDefinition(String searchDefinition) { + this.searchDefinitions = Collections.singletonList(searchDefinition); + return this; + } + + public Builder withSearchDefinitions(List<String> searchDefinition) { + this.searchDefinitions = Collections.unmodifiableList(searchDefinition); + return this; + } + + public Builder withSearchDefinitionDir(String searchDefinitionDir) { + this.searchDefinitionDir = searchDefinitionDir; + return this; + } + + public Builder withDeploymentInfo(String deploymentInfo) { + this.deploymentInfo = deploymentInfo; + return this; + } + + public Builder withValidationOverrides(String validationOverrides) { + this.validationOverrides = validationOverrides; + return this; + } + + public Builder failOnValidateXml() { + this.failOnValidateXml = true; + return this; + } + + public ApplicationPackage build() { + return new MockApplicationPackage(hosts, services, searchDefinitions, searchDefinitionDir, + deploymentInfo, validationOverrides, failOnValidateXml); + } + } + + public static String createSearchDefinition(String name, String fieldName) { + return "search " + name + " {" + + " document " + name + " {" + + " field " + fieldName + " type string {}" + + " }" + + "}"; + } + + private static final String emptyServices = "<services version=\"1.0\">" + + " <admin version=\"2.0\">" + + " <adminserver hostalias=\"node1\" />" + + " </admin>" + + "</services>"; + + private static final String emptyHosts = "<hosts>" + + " <host name=\"localhost\">" + + " <alias>node1</alias>" + + " </host>" + + "</hosts>"; + + @Override + public void validateXML(DeployLogger logger) throws IOException { + if (failOnValidateXml) { + throw new IllegalArgumentException("Error in application package"); + } else { + throw new UnsupportedOperationException("This application package cannot validate XML"); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java new file mode 100644 index 00000000000..fa84cf1c7eb --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java @@ -0,0 +1,169 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.test; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.model.producer.AbstractConfigProducerRoot; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.ConfigProducer; +import com.yahoo.vespa.model.HostSystem; +import com.yahoo.vespa.model.admin.Admin; +import com.yahoo.vespa.model.builder.xml.dom.DomAdminV2Builder; +import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; +import com.yahoo.vespa.model.filedistribution.FileDistributor; +import org.w3c.dom.Document; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.io.StringReader; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Set; + + +/** + * Use for testing. Use as parent for the config producer(s) you want to test, to test + * only a subtree of the producers. + * + * @author gjoranv + */ +public class MockRoot extends AbstractConfigProducerRoot { + private static final long serialVersionUID = 1L; + public static final String MOCKHOST = "mockhost"; + + private HostSystem hostSystem; + + private final DeployState deployState; + private FileDistributor fileDistributor; + private Admin admin; + + public MockRoot() { + this(""); + } + + public MockRoot(String rootConfigId) { + this(rootConfigId, new MockApplicationPackage.Builder().build()); + } + + public MockRoot(String rootConfigId, ApplicationPackage applicationPackage) { + this(rootConfigId, new DeployState.Builder().applicationPackage(applicationPackage).build()); + } + + public MockRoot(String rootConfigId, DeployState deployState) { + super(rootConfigId); + hostSystem = new HostSystem(this, "hostsystem", deployState.getProvisioner()); + this.deployState = deployState; + fileDistributor = new FileDistributor(deployState.getFileRegistry()); + } + + public FileDistributionConfigProducer getFileDistributionConfigProducer() { + return null; + } + + @Override + public ConfigModelRepo configModelRepo() { + return new ConfigModelRepo(); + } + + public Set<String> getConfigIds() { + return Collections.unmodifiableSet(id2producer.keySet()); + } + + @Override + public ConfigInstance.Builder getConfig(ConfigInstance.Builder builder, String configId) { + ConfigProducer cp = id2producer.get(configId); + if (cp == null) return null; + + cp.cascadeConfig(builder); + cp.addUserConfig(builder); + return builder; + } + + @SuppressWarnings("unchecked") + public <T extends ConfigInstance> T getConfig(Class<T> configClass, String configId) { + try { + ConfigInstance.Builder builder = getConfig(getBuilder(configClass).newInstance(), configId); + return configClass.getConstructor(builder.getClass()).newInstance(builder); + } catch (InstantiationException | NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + public ConfigProducer getProducer(String configId) { + return id2producer.get(configId); + } + + @SuppressWarnings("unchecked") + public static <T extends ConfigInstance> Class<? extends ConfigInstance.Builder> getBuilder(Class<T> configClass) { + for (Class<?> memberClass : configClass.getClasses()) { + if (memberClass.getSimpleName().equals("Builder")) + return (Class<? extends ConfigInstance.Builder>) memberClass; + } + throw new RuntimeException("Missing builder"); + } + + @Override + public DeployState getDeployState() { + return deployState; + } + + public FileDistributor getFileDistributor() { + return fileDistributor; + } + + public HostSystem getHostSystem() { + return hostSystem; + } + + public void addDescendant(String configId, AbstractConfigProducer descendant) { + if (id2producer.containsKey(configId)) { + throw new RuntimeException + ("Config ID '" + configId + "' cannot be reserved by an instance of class '" + + descendant.getClass().getName() + "' since it is already used by an instance of class '" + + id2producer.get(configId).getClass().getName() + "'"); + } + id2producer.put(configId, descendant); + } + + @Override + public void addChild(AbstractConfigProducer abstractConfigProducer) { + super.addChild(abstractConfigProducer); + } + + public final void setAdmin(String xml) { + String servicesXml = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + xml + "</services>"; + + try { + Document doc = XmlHelper.getDocumentBuilder().parse(new InputSource(new StringReader(servicesXml))); + setAdmin(new DomAdminV2Builder(deployState.getFileRegistry(), false, new ArrayList<>()). + build(this, XML.getChildren(doc.getDocumentElement(), "admin").get(0))); + } catch (SAXException | IOException e) { + throw new RuntimeException(e); + } + } + + public final void setAdmin(Admin admin) { + this.admin = admin; + } + + @Override + public final Admin getAdmin() { + return admin; + } + + @Override + public DeployLogger deployLogger() { + return new BaseDeployLogger(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java b/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java new file mode 100644 index 00000000000..4518e44d441 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java @@ -0,0 +1,111 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.test; + +import com.google.common.annotations.Beta; +import com.yahoo.config.model.MapConfigModelRegistry; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.application.provider.SchemaValidator; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.vespa.model.VespaModel; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Test driver for testing config models. Add custom builders for plugins to be tested. Builds a model from the + * xml string and returns a config producer that can be use to test getConfig. + * + * @author lulf + * @since 5.1.20 + */ +@Beta +public class TestDriver { + + private final List<ConfigModelBuilder> builders = new ArrayList<>(); + private final boolean validate; + + public TestDriver(boolean validate) { + this.validate = validate; + } + + public TestDriver() { + this(false); + } + + /** + * Add a new builder to the tester. + * + * @param builder builder to add. + * @return this for chaining + */ + public TestDriver addBuilder(ConfigModelBuilder builder) { + builders.add(builder); + return this; + } + + /** + * Build a model from an XML string. The hosts referenced in services must be set to 'mockhost' when using + * this method, as it automatically generates a hosts file for you. + * + * @param servicesXml The xml for services.xml + * @return a producer root capable of answering getConfig requests. + */ + public TestRoot buildModel(String servicesXml) { + return buildModel(servicesXml, "<hosts><host name='localhost'><alias>mockhost</alias></host></hosts>"); + } + + /** + * Build a model from an XML string of services and one of hosts. + * + * @param servicesXml The xml for services.xml + * @param hostsXml The xml for hosts.xml + * @return a producer root capable of answering getConfig requests. + */ + public TestRoot buildModel(String servicesXml, String hostsXml) { + if (!servicesXml.contains("<services")) { + servicesXml = "<services version='1.0'>" + servicesXml + "</services>"; + } + return buildModel(new MockApplicationPackage.Builder().withHosts(hostsXml).withServices(servicesXml).build()); + } + + /** + * Build a model from an application package. + * + * @param applicationPackage Any type of application package. + * @return a producer root capable of answering getConfig requests. + */ + public TestRoot buildModel(ApplicationPackage applicationPackage) { + return buildModel(new DeployState.Builder().applicationPackage(applicationPackage).build()); + } + + /** + * Build a model given a deploy state. + * + * @param deployState An instance of {@link com.yahoo.config.model.deploy.DeployState} + * @return a producer root capable of answering getConfig requests. + */ + public TestRoot buildModel(DeployState deployState) { + MapConfigModelRegistry registry = new MapConfigModelRegistry(builders); + try { + validate(deployState.getApplicationPackage()); + return new TestRoot(new VespaModel(registry, deployState)); + } catch (IOException | SAXException e) { + throw new RuntimeException(e); + } + } + + private void validate(ApplicationPackage appPkg) throws IOException { + if (!validate) { + return; + } + SchemaValidator validator = SchemaValidator.createTestValidatorHosts(); + if (appPkg.getHosts() != null) { + validator.validate(appPkg.getHosts()); + } + validator = SchemaValidator.createTestValidatorServices(); + validator.validate(appPkg.getServices()); + } +} diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java new file mode 100644 index 00000000000..44491e33479 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.test; + +import com.google.common.annotations.Beta; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.model.ConfigModel; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.VespaModel; + +import java.util.List; + +/** + * Test utility class that provides many methods for inspecting the state of a completely built model + * + * @author lulf + * @since 5.1 + */ +@Beta +public class TestRoot { + private final VespaModel model; + TestRoot(VespaModel model) { + this.model = model; + } + + /** + * Get a list of all config models of a particular type. + * + * @param clazz The class of the models to find. + * @return A list of models of given type. + */ + public <MODEL extends ConfigModel> List<MODEL> getConfigModels(Class<MODEL> clazz) { + return model.configModelRepo().getModels(clazz); + } + + /** + * Ask model to populate builder with config for a given config id. This method gives the same config as the + * configserver would return to its clients. + * @param builder The builder to populate + * @param configId The config id of the producer to ask for config. + * @return the same builder. + */ + public <BUILDER extends ConfigInstance.Builder> BUILDER getConfig(BUILDER builder, String configId) { + return (BUILDER) model.getConfig(builder, configId); + } + + /** + * Request config of a given type and id. This method gives the same config as the configserver would return to + * its clients. + * + * @param clazz Type of config to request. + * @param configId The config id of the producer to ask for config. + * @return A config object of the appropriate type with config values set. + */ + public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> clazz, String configId) { + return model.getConfig(clazz, configId); + } + + /** + * Retrieve the hosts available in this model. Useful to verify that hostnames are set correctly etc. + * + * @return A list of hosts. + */ + public List<HostResource> getHosts() { + return model.getHostSystem().getHosts(); + } +} diff --git a/config-model/src/main/java/com/yahoo/config/model/test/package-info.java b/config-model/src/main/java/com/yahoo/config/model/test/package-info.java new file mode 100644 index 00000000000..6c33505df83 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/test/package-info.java @@ -0,0 +1,6 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +//TODO: Temporary export due to standalone container package, remove later. +@ExportPackage +package com.yahoo.config.model.test; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DataTypeCollection.java b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeCollection.java new file mode 100644 index 00000000000..822ddcd0da7 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeCollection.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentmodel; + +import com.yahoo.document.DataType; + +import java.util.Collection; + +/** + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public interface DataTypeCollection { + public DataType getDataType(String name); + public DataType getDataType(int id); + public Collection<DataType> getTypes(); +} diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java new file mode 100644 index 00000000000..6d332ba16fb --- /dev/null +++ b/config-model/src/main/java/com/yahoo/documentmodel/DataTypeRepo.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentmodel; + +import com.yahoo.document.DataType; + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public class DataTypeRepo implements DataTypeCollection { + Map<Integer, DataType> typeById = new LinkedHashMap<>(); + Map<String, DataType> typeByName = new LinkedHashMap<>(); + + public DataType getDataType(String name) { + return typeByName.get(name); + } + + public DataType getDataType(int id) { + return typeById.get(id); + } + + public Collection<DataType> getTypes() { return typeById.values(); } + + public DataTypeRepo add(DataType type) { + if (typeByName.containsKey(type.getName()) || + typeById.containsKey(type.getId())) + { + throw new IllegalStateException("Data type '" + type.getName() + "', id '" + type.getId() + "' is already registered."); + } + typeByName.put(type.getName(), type); + typeById.put(type.getId(), type); + return this; + } + + public DataTypeRepo addAll(DataTypeCollection repo) { + for (DataType dataType : repo.getTypes()) { + add(dataType); + } + return this; + } + + public DataTypeRepo replace(DataType type) { + if (!typeByName.containsKey(type.getName()) || + !typeById.containsKey(type.getId())) + { + throw new IllegalStateException("Data type '" + type.getName() + "' is not registered."); + } + typeByName.remove(type.getName()); + typeByName.put(type.getName(), type); + typeById.remove(type.getId()); + typeById.put(type.getId(), type); + return this; + } + +} diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeCollection.java b/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeCollection.java new file mode 100644 index 00000000000..6c71410c048 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeCollection.java @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentmodel; + +import java.util.Collection; + +/** + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public interface DocumentTypeCollection { + public NewDocumentType getDocumentType(NewDocumentType.Name name); + public NewDocumentType getDocumentType(int id); + public Collection<NewDocumentType> getTypes(); +} diff --git a/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java b/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java new file mode 100644 index 00000000000..3585a12ac2f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/documentmodel/DocumentTypeRepo.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentmodel; + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public class DocumentTypeRepo implements DocumentTypeCollection { + final Map<Integer, NewDocumentType> typeById = new LinkedHashMap<>(); + final Map<NewDocumentType.Name, NewDocumentType> typeByName = new LinkedHashMap<>(); + + public final NewDocumentType getDocumentType(String name) { + return typeByName.get(new NewDocumentType.Name(name)); + } + public NewDocumentType getDocumentType(NewDocumentType.Name name) { + return typeByName.get(name); + } + + public NewDocumentType getDocumentType(int id) { + return typeById.get(id); + } + + public Collection<NewDocumentType> getTypes() { return typeById.values(); } + + public DocumentTypeRepo add(NewDocumentType type) { + if (typeByName.containsKey(type.getFullName())) { + throw new IllegalStateException("Document type " + type.toString() + " is already registered"); + } + if (typeById.containsKey(type.getFullName().getId())) { + throw new IllegalStateException("Document type " + type.toString() + " is already registered"); + } + typeByName.put(type.getFullName(), type); + typeById.put(type.getFullName().getId(), type); + return this; + } +} diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java new file mode 100644 index 00000000000..51171e97704 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java @@ -0,0 +1,366 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentmodel; + +import com.yahoo.document.*; +import com.yahoo.document.annotation.AnnotationType; +import com.yahoo.document.annotation.AnnotationTypeRegistry; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.searchdefinition.FieldSets; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.FieldSet; +import com.yahoo.searchdefinition.processing.BuiltInFieldSets; + +import java.util.*; + +/** + * TODO: What is this and why? + * + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public final class NewDocumentType extends StructuredDataType implements DataTypeCollection { + + /** + * TODO: What is this and why? + */ + public static final class Name { + + // TODO: privatize + final String name; + final int id; + + public Name(String name) { + this(name.hashCode(),name); + } + + public Name(int id,String name) { + this.id = id; + this.name = name; + } + + public String toString() { return name; } + + public final String getName() { return name; } + + public final int getId() { return id; } + + public int hashCode() { return name.hashCode(); } + + public boolean equals(Object other) { + if ( ! (other instanceof Name)) return false; + return name.equals(((Name)other).getName()); + } + } + + private final Name name; + private final DataTypeRepo dataTypes = new DataTypeRepo(); + private final Map<Integer, NewDocumentType> inherits = new LinkedHashMap<>(); + private final AnnotationTypeRegistry annotations = new AnnotationTypeRegistry(); + private final StructDataType header; + private final StructDataType body; + private final Set<FieldSet> fieldSets = new LinkedHashSet<>(); + + public NewDocumentType(Name name) { + this(name, + new StructDataType(name.getName() + ".header"), + new StructDataType(name.getName() + ".body"), new FieldSets()); + } + public NewDocumentType(Name name, StructDataType header, StructDataType body, FieldSets fs) { + super(name.getName()); + this.name = name; + this.header = header; + this.body = body; + if (fs != null) { + this.fieldSets.addAll(fs.userFieldSets().values()); + for (FieldSet f : fs.builtInFieldSets().values()) { + if ((f.getName() != BuiltInFieldSets.INTERNAL_FIELDSET_NAME) && + (f.getName() != BuiltInFieldSets.SEARCH_FIELDSET_NAME)) { + fieldSets.add(f); + } + } + } + } + + public Name getFullName() { + return name; + } + + public DataType getHeader() { return header; } + public DataType getBody() { return body; } + public Collection<NewDocumentType> getInherited() { return inherits.values(); } + public NewDocumentType getInherited(Name inherited) { return inherits.get(inherited.getId()); } + public NewDocumentType removeInherited(Name inherited) { return inherits.remove(inherited.getId()); } + + /** + * Data type of the header fields of this and all inherited document types + * @return merged {@link StructDataType} + */ + public StructDataType allHeader() { + StructDataType ret = new StructDataType(header.getName()); + for (Field f : header.getFields()) { + ret.addField(f); + } + for (NewDocumentType inherited : getInherited()) { + for (Field f : ((StructDataType) inherited.getHeader()).getFields()) { + ret.addField(f); + } + } + return ret; + } + + /** + * Data type of the body fields of this and all inherited document types + * @return merged {@link StructDataType} + */ + public StructDataType allBody() { + StructDataType ret = new StructDataType(body.getName()); + for (Field f : body.getFields()) { + ret.addField(f); + } + for (NewDocumentType inherited : getInherited()) { + for (Field f : ((StructDataType) inherited.getBody()).getFields()) { + ret.addField(f); + } + } + return ret; + } + + @Override + public Class getValueClass() { + return Document.class; + } + + @Override + public boolean isValueCompatible(FieldValue value) { + if (!(value instanceof Document)) { + return false; + } + /** Temporary disabled due to clash with document and covariant return type + Document doc = (Document) value; + if (((NewDocumentType) doc.getDataType()).inherits(this)) { + //the value is of this type; or the supertype of the value is of this type, etc.... + return true; + } + */ + return false; + } + + private boolean verifyInheritance(NewDocumentType inherited) { + for (Field f : getFields()) { + Field inhF = inherited.getField(f.getName()); + if (inhF != null && !inhF.equals(f)) { + throw new IllegalArgumentException("Inherited document '" + inherited.toString() + "' already contains field '" + + inhF.getName() + "'. Can not override with '" + f.getName() + "'."); + } + } + for (Field f : inherited.getAllFields()) { + for (NewDocumentType side : inherits.values()) { + Field sideF = side.getField(f.getName()); + if (sideF != null && !sideF.equals(f)) { + throw new IllegalArgumentException("Inherited document '" + side.toString() + "' already contains field '" + + sideF.getName() + "'. Document '" + inherited.toString() + "' also defines field '" + f.getName() + + "'.Multiple inheritance must be disjunctive."); + } + } + } + return true; + } + public void inherit(NewDocumentType inherited) { + if ( ! inherits.containsKey(inherited.getId())) { + verifyInheritance(inherited); + inherits.put(inherited.getId(), inherited); + } + } + public boolean inherits(NewDocumentType superType) { + if (getId() == superType.getId()) return true; + for (NewDocumentType type : inherits.values()) { + if (type.inherits(superType)) return true; + } + return false; + } + + @Override + public Field getField(String name) { + Field field = header.getField(name); + if (field == null) { + field = body.getField(name); + } + if (field == null) { + for (NewDocumentType inheritedType : inherits.values()) { + field = inheritedType.getField(name); + if (field != null) { + return field; + } + } + } + return field; + } + + public boolean containsField(String fieldName) { + return getField(fieldName) != null; + } + + @Override + public Field getField(int id) { + Field field = header.getField(id); + if (field == null) { + field = body.getField(id); + } + if (field == null) { + for (NewDocumentType inheritedType : inherits.values()) { + field = inheritedType.getField(id); + if (field != null) { + return field; + } + } + } + return field; + } + + public Collection<Field> getAllFields() { + Collection<Field> collection = new LinkedList<>(); + + for (NewDocumentType type : inherits.values()) { + collection.addAll(type.getAllFields()); + } + + collection.addAll(header.getFields()); + collection.addAll(body.getFields()); + return Collections.unmodifiableCollection(collection); + } + + public Collection<Field> getFields() { + Collection<Field> collection = new LinkedList<>(); + collection.addAll(header.getFields()); + collection.addAll(body.getFields()); + return Collections.unmodifiableCollection(collection); + } + + @Override + public Document createFieldValue() { + return new Document(null, (DocumentId)null); + } + + @Override + public Collection<DataType> getTypes() { + return dataTypes.getTypes(); + } + + public DataTypeCollection getAllTypes() { + DataTypeRepo repo = new DataTypeRepo(); + Set<Name> seen = new HashSet<>(); + Deque<NewDocumentType> stack = new LinkedList<>(); + stack.push(this); + while (!stack.isEmpty()) { + NewDocumentType docType = stack.pop(); + if (seen.contains(docType.name)) { + continue; // base type + } + seen.add(docType.name); + for (DataType dataType : docType.getTypes()) { + if (repo.getDataType(dataType.getId()) == null) { + repo.add(dataType); + } + } + stack.addAll(docType.inherits.values()); + } + return repo; + } + + public Collection<AnnotationType> getAnnotations() { return annotations.getTypes().values(); } + public Collection<AnnotationType> getAllAnnotations() { + Collection<AnnotationType> collection = new LinkedList<>(); + + for (NewDocumentType type : inherits.values()) { + collection.addAll(type.getAllAnnotations()); + } + collection.addAll(getAnnotations()); + + return Collections.unmodifiableCollection(collection); + } + + public DataType getDataType(String name) { + return dataTypes.getDataType(name); + } + public DataType getDataType(int id) { + return dataTypes.getDataType(id); + } + public DataType getDataTypeRecursive(String name) { + DataType a = dataTypes.getDataType(name); + if (a != null) { + return a; + } else { + for (NewDocumentType dt : getInherited()) { + a = dt.getDataTypeRecursive(name); + if (a != null) { + return a; + } + } + } + return null; + } + + public DataType getDataTypeRecursive(int id) { + DataType a = dataTypes.getDataType(id); + if (a != null) { + return a; + } else { + for (NewDocumentType dt : getInherited()) { + a = dt.getDataTypeRecursive(id); + if (a != null) { + return a; + } + } + } + return null; + } + + public AnnotationType getAnnotationType(String name) { + AnnotationType a = annotations.getType(name); + if (a != null) { + return a; + } else { + for (NewDocumentType dt : getInherited()) { + a = dt.getAnnotationType(name); + if (a != null) { + return a; + } + } + } + return null; + } + public AnnotationType getAnnotationType(int id) { + AnnotationType a = annotations.getType(id); + if (a != null) { + return a; + } else { + for (NewDocumentType dt : getInherited()) { + a = dt.getAnnotationType(id); + if (a != null) { + return a; + } + } + } + return null; + } + + public NewDocumentType add(AnnotationType type) { + annotations.register(type); + return this; + } + public NewDocumentType add(DataType type) { + dataTypes.add(type); + return this; + } + public NewDocumentType replace(DataType type) { + dataTypes.replace(type); + return this; + } + + /** + * The field sets defined for this type and its {@link Search} + * @return fieldsets + */ + public Set<FieldSet> getFieldSets() { + return Collections.unmodifiableSet(fieldSets); + } +} diff --git a/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java new file mode 100644 index 00000000000..bf8ec8d3da7 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/documentmodel/VespaDocumentType.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.documentmodel; + +import com.yahoo.document.DataType; +import com.yahoo.document.DataTypeName; +import com.yahoo.document.PositionDataType; + +/** + * This class represents the builtin 'doument' document type that all other documenttypes inherits. + * Remember that changes here must be compatible. Changes to types of fields can not be done here. + * This must also match the mirroring class in c++. + * + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public class VespaDocumentType { + + public static NewDocumentType INSTANCE = newInstance(); + + public static DataTypeName NAME = new DataTypeName("document"); + + private static NewDocumentType newInstance() { + NewDocumentType vespa = new NewDocumentType(new NewDocumentType.Name(8, "document")); + vespa.add(DataType.BYTE); + vespa.add(DataType.INT); + vespa.add(DataType.LONG); + vespa.add(DataType.STRING); + vespa.add(DataType.RAW); + vespa.add(DataType.TAG); + vespa.add(DataType.FLOAT); + vespa.add(DataType.DOUBLE); + vespa.add(DataType.DOCUMENT); + vespa.add(PositionDataType.INSTANCE); + vespa.add(DataType.URI); + vespa.add(DataType.PREDICATE); + vespa.add(DataType.TENSOR); + return vespa; + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java b/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java new file mode 100644 index 00000000000..98eb0a4b77c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/ConstantTensorTransformer.java @@ -0,0 +1,74 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.searchlib.rankingexpression.evaluation.Value; +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.NameNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * Transforms named references to constant tensors with the rank feature 'constant'. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +class ConstantTensorTransformer extends ExpressionTransformer { + + private final Map<String, Value> constants; + private final Map<String, String> rankPropertiesOutput; + + public ConstantTensorTransformer(Map<String, Value> constants, + Map<String, String> rankPropertiesOutput) { + this.constants = constants; + this.rankPropertiesOutput = rankPropertiesOutput; + } + + @Override + public ExpressionNode transform(ExpressionNode node) { + if (node instanceof ReferenceNode) { + return transformFeature((ReferenceNode) node); + } else if (node instanceof CompositeNode) { + return transformChildren((CompositeNode) node); + } else { + return node; + } + } + + private ExpressionNode transformFeature(ReferenceNode node) { + if (!node.getArguments().isEmpty()) { + return transformArguments(node); + } else { + return transformConstantReference(node); + } + } + + private ExpressionNode transformArguments(ReferenceNode node) { + List<ExpressionNode> arguments = node.getArguments().expressions(); + List<ExpressionNode> transformedArguments = new ArrayList<>(arguments.size()); + for (ExpressionNode argument : arguments) { + transformedArguments.add(transform(argument)); + } + return node.setArguments(transformedArguments); + } + + private ExpressionNode transformConstantReference(ReferenceNode node) { + Value value = constants.get(node.getName()); + if (value == null || !(value instanceof TensorValue)) { + return node; + } + TensorValue tensorValue = (TensorValue)value; + String featureName = "constant(" + node.getName() + ")"; + String tensorType = (tensorValue.getType().isPresent() ? tensorValue.getType().get().toString() : "tensor"); + rankPropertiesOutput.put(featureName + ".value", tensorValue.toString()); + rankPropertiesOutput.put(featureName + ".type", tensorType); + return new ReferenceNode("constant", Arrays.asList(new NameNode(node.getName())), null); + } + +}
\ No newline at end of file diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java new file mode 100644 index 00000000000..c0dcf27c88d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java @@ -0,0 +1,140 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.document.SDField; + +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * The rank profile containing default settings. This is derived from the fields + * whenever this is accessed. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class DefaultRankProfile extends RankProfile { + + /** + * Creates a new rank profile + * + * @param rankProfileRegistry The {@link com.yahoo.searchdefinition.RankProfileRegistry} to use for storing and looking up rank profiles. + */ + public DefaultRankProfile(Search search, RankProfileRegistry rankProfileRegistry) { + super("default", search, rankProfileRegistry); + } + + /** + * Does nothing, the default rank profile can not inherit anything + */ + // TODO: Why not? If that's the case, then fail attempts at it + public void setInherited(String inheritedName) { + } + + /** + * Returns null, the default rank profile can not inherit anything + */ + public String getInheritedName() { + return null; + } + + /** + * Returns the rank boost value of the given field + */ + public RankSetting getRankSetting(String fieldOrIndex,RankSetting.Type type) { + RankSetting setting=super.getRankSetting(fieldOrIndex,type); + if (setting!=null) return setting; + + SDField field=getSearch().getField(fieldOrIndex); + if (field!=null) { + setting=toRankSetting(field,type); + if (setting!=null) + return setting; + } + + Index index=getSearch().getIndex(fieldOrIndex); + if (index!=null) { + setting=toRankSetting(index,type); + if (setting!=null) + return setting; + } + + return null; + } + + private RankSetting toRankSetting(SDField field,RankSetting.Type type) { + if (type.equals(RankSetting.Type.WEIGHT) && field.getWeight()>0 && field.getWeight()!=100) + return new RankSetting(field.getName(),type,field.getWeight()); + if (type.equals(RankSetting.Type.RANKTYPE)) + return new RankSetting(field.getName(),type,field.getRankType()); + if (type.equals(RankSetting.Type.LITERALBOOST) && field.getLiteralBoost()>0) + return new RankSetting(field.getName(),type,field.getLiteralBoost()); + + // Index level setting really + if (type.equals(RankSetting.Type.PREFERBITVECTOR) && field.getRanking().isFilter()) { + return new RankSetting(field.getName(), type, true); + } + + return null; + } + + private RankSetting toRankSetting(Index index, RankSetting.Type type) { + /* TODO: Add support for indexes by adding a ranking object to the index + if (type.equals(RankSetting.Type.PREFERBITVECTOR) && index.isPreferBitVector()) { + return new RankSetting(index.getName(), type, new Boolean(true)); + } + */ + return null; + } + + /** + * Returns the names of the fields which have a rank boost setting + * explicitly in this profile or in fields + */ + public Set<RankSetting> rankSettings() { + Set<RankSetting> settings=new LinkedHashSet<>(20); + settings.addAll(this.rankSettings); + for (SDField field : getSearch().allFieldsList() ) { + addSetting(field,RankSetting.Type.WEIGHT,settings); + addSetting(field,RankSetting.Type.RANKTYPE,settings); + addSetting(field,RankSetting.Type.LITERALBOOST,settings); + addSetting(field,RankSetting.Type.PREFERBITVECTOR,settings); + } + + // Foer settings that really pertains to indexes do the explicit indexes too + for (Index index : getSearch().getExplicitIndices()) { + addSetting(index,RankSetting.Type.PREFERBITVECTOR,settings); + } + return settings; + } + + private void addSetting(SDField field,RankSetting.Type type,Set<RankSetting> settings) { + if (type.isIndexLevel()) { + addIndexSettings(field,type,settings); + } + else { + RankSetting setting=toRankSetting(field,type); + if (setting==null) return; + settings.add(setting); + } + } + + private void addIndexSettings(SDField field,RankSetting.Type type,Set<RankSetting> settings) { + for (Iterator i = field.getFieldNameAsIterator(); i.hasNext(); ) { + String indexName=(String)i.next(); + Index explicitIndex=field.getIndex(indexName); + + // TODO: Make a ranking object in the index override the field level ranking object + if (type.equals(RankSetting.Type.PREFERBITVECTOR) && field.getRanking().isFilter()) { + settings.add(new RankSetting(indexName, type, true)); + } + } + } + + private void addSetting(Index index,RankSetting.Type type,Set<RankSetting> settings) { + RankSetting setting=toRankSetting(index,type); + if (setting==null) return; + settings.add(setting); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java new file mode 100644 index 00000000000..4fd7048159e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java @@ -0,0 +1,415 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.document.*; +import com.yahoo.document.annotation.AnnotationReferenceDataType; +import com.yahoo.document.annotation.AnnotationType; +import com.yahoo.documentmodel.DataTypeCollection; +import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.documentmodel.VespaDocumentType; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferenceDataType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.documentmodel.DocumentModel; +import com.yahoo.vespa.documentmodel.FieldView; +import com.yahoo.vespa.documentmodel.SearchDef; +import com.yahoo.vespa.documentmodel.SearchField; + +import java.util.*; + +/** + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public class DocumentModelBuilder { + + public static class RetryLaterException extends IllegalArgumentException { + public RetryLaterException(String message) { + super(message); + } + } + private DocumentModel model; + private final Map<NewDocumentType, List<SDDocumentType>> scratchInheritsMap = new HashMap<>(); + public DocumentModelBuilder(DocumentModel model) { + this.model = model; + model.getDocumentManager().add(VespaDocumentType.INSTANCE); + } + public boolean valid() { + return scratchInheritsMap.isEmpty(); + } + public void addToModel(Collection<Search> searchList) { + List<SDDocumentType> docList = new LinkedList<>(); + for (Search search : searchList) { + docList.add(search.getDocument()); + } + docList = sortDocumentTypes(docList); + addDocumentTypes(docList); + for (Collection<Search> toAdd = tryAdd(searchList); + !toAdd.isEmpty() && (toAdd.size() < searchList.size()); toAdd = tryAdd(searchList)) { + searchList = toAdd; + } + } + + private List<SDDocumentType> sortDocumentTypes(List<SDDocumentType> docList) { + Set<String> doneNames = new HashSet<>(); + doneNames.add(SDDocumentType.VESPA_DOCUMENT.getName()); + List<SDDocumentType> doneList = new LinkedList<>(); + List<SDDocumentType> prevList = null; + List<SDDocumentType> nextList = docList; + while (prevList == null || nextList.size() < prevList.size()) { + prevList = nextList; + nextList = new LinkedList<>(); + for (SDDocumentType doc : prevList) { + boolean isDone = true; + for (SDDocumentType inherited : doc.getInheritedTypes()) { + if (!doneNames.contains(inherited.getName())) { + isDone = false; + break; + } + } + if (isDone) { + doneNames.add(doc.getName()); + doneList.add(doc); + } else { + nextList.add(doc); + } + } + } + if (!nextList.isEmpty()) { + throw new IllegalArgumentException("Could not resolve inheritance of document types " + + toString(prevList) + "."); + } + return doneList; + } + + private static String toString(List<SDDocumentType> lst) { + StringBuilder out = new StringBuilder(); + for (int i = 0, len = lst.size(); i < len; ++i) { + out.append("'").append(lst.get(i).getName()).append("'"); + if (i < len - 2) { + out.append(", "); + } else if (i < len - 1) { + out.append(" and "); + } + } + return out.toString(); + } + + private Collection<Search> tryAdd(Collection<Search> searchList) { + Collection<Search> left = new ArrayList<>(); + for (Search search : searchList) { + try { + addToModel(search); + } catch (RetryLaterException e) { + left.add(search); + } + } + return left; + } + public void addToModel(Search search) { + // Then we add the search specific stuff + SearchDef searchDef = new SearchDef(search.getName()); + addSearchFields(search.extraFieldList(), searchDef); + for (Field f : search.getDocument().fieldSet()) { + addSearchField((SDField) f, searchDef); + } + for(SDField field : search.allFieldsList()) { + for(Attribute attribute : field.getAttributes().values()) { + if (!searchDef.getFields().containsKey(attribute.getName())) { + searchDef.add(new SearchField(new Field(attribute.getName(), field), !field.getIndices().isEmpty(), true)); + } + } + } + + for (Field f : search.getDocument().fieldSet()) { + addAlias((SDField) f, searchDef); + } + model.getSearchManager().add(searchDef); + } + private static void addSearchFields(Collection<SDField> fields, SearchDef searchDef) { + for (SDField field : fields) { + addSearchField(field, searchDef); + } + } + private static void addSearchField(SDField field, SearchDef searchDef) { + + SearchField searchField = + new SearchField(field, + field.getIndices().containsKey(field.getName()) && field.getIndices().get(field.getName()).getType().equals(Index.Type.VESPA), + field.getAttributes().containsKey(field.getName())); + searchDef.add(searchField); + + // Add field to views + addToView(field.getIndices().keySet(), searchField, searchDef); + } + + private static void addAlias(SDField field, SearchDef searchDef) { + for (Map.Entry<String, String> entry : field.getAliasToName().entrySet()) { + searchDef.addAlias(entry.getKey(), entry.getValue()); + } + } + + private static void addToView(Collection<String> views, Field field, SearchDef searchDef) { + for (String viewName : views) { + addToView(viewName, field, searchDef); + } + } + + private static void addToView(String viewName, Field field, SearchDef searchDef) { + if (searchDef.getViews().containsKey(viewName)) { + searchDef.getViews().get(viewName).add(field); + } else { + if (!searchDef.getFields().containsKey(viewName)) { + FieldView view = new FieldView(viewName); + view.add(field); + searchDef.add(view); + } + } + } + private void addDocumentTypes(List<SDDocumentType> docList) { + LinkedList<NewDocumentType> lst = new LinkedList<>(); + for (SDDocumentType doc : docList) { + lst.add(convert(doc)); + model.getDocumentManager().add(lst.getLast()); + } + for(NewDocumentType doc : lst) { + resolveTemporaries(doc.getAllTypes(), lst); + } + } + private static void resolveTemporaries(DataTypeCollection dtc, Collection<NewDocumentType> docs) { + for (DataType type : dtc.getTypes()) { + resolveTemporariesRecurse(type, dtc, docs); + } + } + private static DataType resolveTemporariesRecurse(DataType type, DataTypeCollection repo, + Collection<NewDocumentType> docs) { + if (type instanceof TemporaryStructuredDataType) { + NewDocumentType docType = getDocumentType(docs, type.getId()); + if (docType != null) { + type = docType; + return type; + } + DataType real = repo.getDataType(type.getId()); + if (real == null) { + throw new NullPointerException("Can not find type '" + type.toString() + "', impossible."); + } + type = real; + } else if (type instanceof StructDataType) { + StructDataType dt = (StructDataType) type; + for (com.yahoo.document.Field field : dt.getFields()) { + if (field.getDataType() != type) { + field.setDataType(resolveTemporariesRecurse(field.getDataType(), repo, docs)); + } + } + } else if (type instanceof MapDataType) { + MapDataType t = (MapDataType) type; + t.setKeyType(resolveTemporariesRecurse(t.getKeyType(), repo, docs)); + t.setValueType(resolveTemporariesRecurse(t.getValueType(), repo, docs)); + } else if (type instanceof CollectionDataType) { + CollectionDataType t = (CollectionDataType) type; + t.setNestedType(resolveTemporariesRecurse(t.getNestedType(), repo, docs)); + } + return type; + } + + private static NewDocumentType getDocumentType(Collection<NewDocumentType> docs, int id) { + for (NewDocumentType doc : docs) { + if (doc.getId() == id) { + return doc; + } + } + return null; + } + + private static void specialHandleAnnotationReference(NewDocumentType docType, Field field) { + DataType fieldType = specialHandleAnnotationReferenceRecurse(docType, field.getName(), field.getDataType()); + if (fieldType == null) { + return; + } + field.setDataType(fieldType); + } + + private static DataType specialHandleAnnotationReferenceRecurse(NewDocumentType docType, String fieldName, + DataType dataType) + { + if (dataType instanceof TemporaryAnnotationReferenceDataType) { + TemporaryAnnotationReferenceDataType refType = (TemporaryAnnotationReferenceDataType)dataType; + if (refType.getId() != 0) { + return null; + } + AnnotationType target = docType.getAnnotationType(refType.getTarget()); + if (target == null) { + throw new RetryLaterException("Annotation '" + refType.getTarget() + "' in reference '" + fieldName + + "' does not exist."); + } + dataType = new AnnotationReferenceDataType(target); + addType(docType, dataType); + return dataType; + } else if (dataType instanceof MapDataType) { + MapDataType mapType = (MapDataType)dataType; + DataType valueType = specialHandleAnnotationReferenceRecurse(docType, fieldName, mapType.getValueType()); + if (valueType == null) { + return null; + } + mapType = mapType.clone(); + mapType.setValueType(valueType); + addType(docType, mapType); + return mapType; + } else if (dataType instanceof CollectionDataType) { + CollectionDataType lstType = (CollectionDataType)dataType; + DataType nestedType = specialHandleAnnotationReferenceRecurse(docType, fieldName, lstType.getNestedType()); + if (nestedType == null) { + return null; + } + lstType = lstType.clone(); + lstType.setNestedType(nestedType); + addType(docType, lstType); + return lstType; + } + return null; + } + + private static StructDataType handleStruct(NewDocumentType dt, SDDocumentType type) { + StructDataType s = new StructDataType(type.getName()); + for (Field f : type.getDocumentType().getHeaderType().getFieldsThisTypeOnly()) { + specialHandleAnnotationReference(dt, f); + s.addField(f); + } + for (StructDataType inherited : type.getDocumentType().getHeaderType().getInheritedTypes()) { + s.inherit(inherited); + } + extractNestedTypes(dt, s); + addType(dt, s); + return s; + } + + private static StructDataType handleStruct(NewDocumentType dt, StructDataType s) { + for (Field f : s.getFieldsThisTypeOnly()) { + specialHandleAnnotationReference(dt, f); + } + extractNestedTypes(dt, s); + addType(dt, s); + return s; + } + private static boolean anyParentsHavePayLoad(SDAnnotationType sa, SDDocumentType sdoc) { + if (sa.getInherits() != null) { + AnnotationType tmp = sdoc.findAnnotation(sa.getInherits()); + SDAnnotationType inherited = (SDAnnotationType) tmp; + return ((inherited.getSdDocType() != null) || anyParentsHavePayLoad(inherited, sdoc)); + } + return false; + } + private NewDocumentType convert(SDDocumentType sdoc) { + Map<AnnotationType, String> annotationInheritance = new HashMap<>(); + Map<StructDataType, String> structInheritance = new HashMap<>(); + NewDocumentType dt = new NewDocumentType(new NewDocumentType.Name(sdoc.getName()), + sdoc.getDocumentType().getHeaderType(), + sdoc.getDocumentType().getBodyType(), + sdoc.getFieldSets()); + for (SDDocumentType n : sdoc.getInheritedTypes()) { + NewDocumentType.Name name = new NewDocumentType.Name(n.getName()); + NewDocumentType inherited = model.getDocumentManager().getDocumentType(name); + if (inherited != null) { + dt.inherit(inherited); + } + } + for (SDDocumentType type : sdoc.getTypes()) { + if (type.isStruct()) { + handleStruct(dt, type); + } else { + throw new IllegalArgumentException("Data type '" + sdoc.getName() + "' is not a struct => tostring='" + sdoc.toString() + "'."); + } + } + for (AnnotationType annotation : sdoc.getAnnotations()) { + dt.add(annotation); + } + for (AnnotationType annotation : sdoc.getAnnotations()) { + SDAnnotationType sa = (SDAnnotationType) annotation; + if (annotation.getInheritedTypes().isEmpty() && (sa.getInherits() != null) ) { + annotationInheritance.put(annotation, sa.getInherits()); + } + if (annotation.getDataType() == null) { + if (sa.getSdDocType() != null) { + StructDataType s = handleStruct(dt, sa.getSdDocType()); + annotation.setDataType(s); + if ((sa.getInherits() != null)) { + structInheritance.put(s, "annotation."+sa.getInherits()); + } + } else if (sa.getInherits() != null) { + StructDataType s = new StructDataType("annotation."+annotation.getName()); + if (anyParentsHavePayLoad(sa, sdoc)) { + annotation.setDataType(s); + addType(dt, s); + } + structInheritance.put(s, "annotation."+sa.getInherits()); + } + } + } + for (Map.Entry<AnnotationType, String> e : annotationInheritance.entrySet()) { + e.getKey().inherit(dt.getAnnotationType(e.getValue())); + } + for (Map.Entry<StructDataType, String> e : structInheritance.entrySet()) { + StructDataType s = (StructDataType)dt.getDataType(e.getValue()); + if (s != null) { + e.getKey().inherit(s); + } + } + handleStruct(dt, sdoc.getDocumentType().getHeaderType()); + handleStruct(dt, sdoc.getDocumentType().getBodyType()); + + extractDataTypesFromFields(dt, sdoc.fieldSet()); + return dt; + } + private static void extractDataTypesFromFields(NewDocumentType dt, Collection<Field> fields) { + for (Field f : fields) { + DataType type = f.getDataType(); + if (testAddType(dt, type)) { + extractNestedTypes(dt, type); + addType(dt, type); + } + } + } + private static void extractNestedTypes(NewDocumentType dt, DataType type) { + if (type instanceof StructDataType) { + StructDataType tmp = (StructDataType) type; + extractDataTypesFromFields(dt, tmp.getFieldsThisTypeOnly()); + } else if (type instanceof DocumentType) { + throw new IllegalArgumentException("Can not handle nested document definitions. In document type '" + dt.getName().toString() + + "', we can not define document type '" + type.toString()); + } else if (type instanceof CollectionDataType) { + CollectionDataType tmp = (CollectionDataType) type; + extractNestedTypes(dt, tmp.getNestedType()); + addType(dt, tmp.getNestedType()); + } else if (type instanceof MapDataType) { + MapDataType tmp = (MapDataType) type; + extractNestedTypes(dt, tmp.getKeyType()); + extractNestedTypes(dt, tmp.getValueType()); + addType(dt, tmp.getKeyType()); + addType(dt, tmp.getValueType()); + } else if (type instanceof TemporaryAnnotationReferenceDataType) { + throw new IllegalArgumentException(type.toString()); + } + } + private static boolean testAddType(NewDocumentType dt, DataType type) { return internalAddType(dt, type, true); } + private static boolean addType(NewDocumentType dt, DataType type) { return internalAddType(dt, type, false); } + private static boolean internalAddType(NewDocumentType dt, DataType type, boolean dryRun) { + DataType oldType = dt.getDataTypeRecursive(type.getId()); + if (oldType == null) { + if ( ! dryRun) { + dt.add(type); + } + return true; + } else if ((type instanceof StructDataType) && (oldType instanceof StructDataType)) { + StructDataType s = (StructDataType) type; + StructDataType os = (StructDataType) oldType; + if ((os.getFieldCount() == 0) && (s.getFieldCount() > os.getFieldCount())) { + if ( ! dryRun) { + dt.replace(type); + } + return true; + } + } + return false; + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplier.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplier.java new file mode 100644 index 00000000000..9cb3dea36a8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplier.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.document.Field; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class FieldOperationApplier { + public void process(SDDocumentType sdoc) { + if (!sdoc.isStruct()) { + apply(sdoc); + } + } + + protected void apply(SDDocumentType type) { + for (Field field : type.fieldSet()) { + apply(field); + } + } + + protected void apply(Field field) { + if (field instanceof SDField) { + SDField sdField = (SDField) field; + sdField.applyOperations(); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForSearch.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForSearch.java new file mode 100644 index 00000000000..301d78bbaec --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForSearch.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.document.Field; +import com.yahoo.searchdefinition.document.SDDocumentType; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class FieldOperationApplierForSearch extends FieldOperationApplier { + @Override + public void process(SDDocumentType sdoc) { + //Do nothing + } + + public void process(Search search) { + for (Field field : search.extraFieldList()) { + apply(field); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java new file mode 100644 index 00000000000..5bef71ac920 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class FieldOperationApplierForStructs extends FieldOperationApplier { + @Override + public void process(SDDocumentType sdoc) { + for (SDDocumentType type : sdoc.getAllTypes()) { + if (type.isStruct()) { + apply(type); + copyFields(type, sdoc); + } + } + } + + private void copyFields(SDDocumentType structType, SDDocumentType sdoc) { + //find all fields in OTHER types that have this type: + List<SDDocumentType> list = new ArrayList<>(); + list.add(sdoc); + list.addAll(sdoc.getTypes()); + for (SDDocumentType anyType : list) { + Iterator<Field> fields = anyType.fieldIterator(); + while (fields.hasNext()) { + SDField field = (SDField) fields.next(); + DataType structUsedByField = field.getFirstStructRecursive(); + if (structUsedByField == null) { + continue; + } + if (structUsedByField.getName().equals(structType.getName())) { + //this field is using this type!! + field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), field.isHeader(), 0); + field.populateWithStructMatching(sdoc, field.getName(), field.getDataType(), field.getMatching()); + } + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldSets.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldSets.java new file mode 100644 index 00000000000..503977a8f49 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldSets.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +import com.yahoo.searchdefinition.document.FieldSet; + +/** + * The field sets owned by a {@link Search} + * Both built in and user defined. + * @author vegardh + * + */ +public class FieldSets { + + private final Map<String, FieldSet> userFieldSets = new LinkedHashMap<>(); + private final Map<String, FieldSet> builtInFieldSets = new LinkedHashMap<>(); + + /** + * Adds an entry to user field sets, creating entries as needed + * + * @param setName name of a field set + * @param field field to add to field set + */ + public void addUserFieldSetItem(String setName, String field) { + if (userFieldSets.get(setName) == null) { + // First entry in this set + userFieldSets.put(setName, new FieldSet(setName)); + } + userFieldSets.get(setName).addFieldName(field); + } + + /** + * Adds an entry to built in field sets, creating entries as needed + * + * @param setName name of a field set + * @param field field to add to field set + */ + public void addBuiltInFieldSetItem(String setName, String field) { + if (builtInFieldSets.get(setName) == null) { + // First entry in this set + builtInFieldSets.put(setName, new FieldSet(setName)); + } + builtInFieldSets.get(setName).addFieldName(field); + } + + /** + * The built in field sets, unmodifiable + * @return built in field sets + */ + public Map<String, FieldSet> builtInFieldSets() { + return Collections.unmodifiableMap(builtInFieldSets); + } + + /** + * The user defined field sets, unmodifiable + * @return user field sets + */ + public Map<String, FieldSet> userFieldSets() { + return Collections.unmodifiableMap(userFieldSets); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java new file mode 100644 index 00000000000..9ed7d8677b9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Index.java @@ -0,0 +1,203 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.document.BooleanIndexDefinition; +import com.yahoo.searchdefinition.document.RankType; +import com.yahoo.searchdefinition.document.Stemming; + +import java.io.Serializable; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * An index definition in a search definition. + * Two indices are equal if they have the same name and the same settings, except + * alias settings (which are excluded). + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class Index implements Cloneable, Serializable { + + public static enum Type { + VESPA("vespa"); + private String name; + private Type(String name) { this.name = name; } + public String getName() { return name; } + + } + + // Please see hashCode, equals and copy when adding attributes to this + + /** The search definition-unique name of this index */ + private String name; + + /** The rank type of this index */ + private RankType rankType=null; + + /** Whether this index supports prefix search */ + private boolean prefix; + + /** The list of aliases (Strings) to this index name */ + private Set<String> aliases=new java.util.LinkedHashSet<>(1); + + /** + * The stemming setting of this field, or null to use the default. + * Default is determined by the owning search definition. + */ + private Stemming stemming=null; + + /** Whether the content of this index is normalized */ + private boolean normalized=true; + + /** The set of all searchable fields which should be searched with this index. May not null */ + public Set<String> matchGroup=new LinkedHashSet<>(); + + private Type type = Type.VESPA; + + /** The boolean index definition, if set */ + private BooleanIndexDefinition boolIndex; + + public Index(String name) { + this(name, false); + } + + public Index(String name, boolean prefix) { + this.name=name; + this.prefix=prefix; + } + + public void setName(String name) { this.name=name; } + + public String getName() { return name; } + + /** Sets the rank type of this field */ + public void setRankType(RankType rankType) { this.rankType=rankType; } + + /** Returns the rank type of this field, or null if nothing is set */ + public RankType getRankType() { return rankType; } + + /** Return the stemming setting of this index, may be null */ + public Stemming getStemming() { return stemming; } + + /** + * Returns the (unmodifiable) set of searchable fields which should be searched + * when this index is searched. This is useful to specify that some attributes should be + * searched as well when an index is searched. + * This set is either empty, or if set contains both the name of this index, and some other + * indexes. + */ + public Set<String> getMatchGroup() { return Collections.unmodifiableSet(matchGroup); } + + /** Adds a searchable field name to be searched when this index is searched */ + public void addToMatchGroup(String name) { + if (name.equals(this.name)) return; + if (matchGroup.size()==0) matchGroup.add(this.name); + matchGroup.add(name); + } + + /** + * Whether this field should be stemmed in this search definition, + * this is never null + */ + public Stemming getStemming(Search search) { + if (stemming!=null) + return stemming; + else + return search.getStemming(); + } + + /** + * Sets how this field should be stemmed, or set to null to use the default. + */ + public void setStemming(Stemming stemming) { this.stemming=stemming; } + + /** Returns whether this index supports prefix search, default is false */ + public boolean isPrefix() { return prefix; } + + /** Sets whether this index supports prefix search */ + public void setPrefix(boolean prefix) { this.prefix=prefix; } + + /** Adds an alias to this index name */ + public void addAlias(String alias) { + aliases.add(alias); + } + + /** Returns a read-only iterator of the aliases (Strings) to this index name */ + public Iterator<String> aliasIterator() { + return Collections.unmodifiableSet(aliases).iterator(); + } + + public int hashCode() { + return name.hashCode() + ( prefix ? 17 : 0 ); + } + + public boolean equals(Object object) { + if ( ! (object instanceof Index)) return false; + + Index other=(Index)object; + return + this.name.equals(other.name) && + this.prefix==other.prefix && + this.stemming==other.stemming && + this.normalized==other.normalized; + } + + public String toString() { + String rankTypeName=rankType==null ? "(none)" : rankType.name(); + return + "index '" + name + + "' [ranktype: " + rankTypeName + + ", prefix: " + prefix + "]"; + } + + /** Makes a deep copy of this index */ + public Object clone() { + try { + Index copy=(Index)super.clone(); + copy.aliases=new LinkedHashSet<>(this.aliases); + return copy; + } + catch (CloneNotSupportedException e) { + throw new RuntimeException("Programming error",e); + } + } + + public Index copy() { + return (Index)clone(); + } + + /** + * The index engine type + * @return the type + */ + public Type getType() { + return type; + } + + /** + * Sets the index engine type + * @param type a index engine type + */ + public void setType(Type type) { + this.type = type; + } + + /** + * The boolean index definition + * @return the boolean index definition + */ + public BooleanIndexDefinition getBooleanIndexDefiniton() { + return boolIndex; + } + + /** + * Sets the boolean index definition + * @param def boolean index definition + */ + public void setBooleanIndexDefiniton(BooleanIndexDefinition def) { + boolIndex = def; + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MacroInliner.java b/config-model/src/main/java/com/yahoo/searchdefinition/MacroInliner.java new file mode 100644 index 00000000000..a248ca30a8e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/MacroInliner.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; +import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; + +import java.util.Map; + +/** + * Inlines macros in ranking expressions + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +class MacroInliner extends ExpressionTransformer { + + private final Map<String, RankProfile.Macro> macros; + + public MacroInliner(Map<String, RankProfile.Macro> macros) { + this.macros = macros; + } + + @Override + public ExpressionNode transform(ExpressionNode node) { + if (node instanceof ReferenceNode) + return transformFeatureNode((ReferenceNode)node); + if (node instanceof CompositeNode) + return transformChildren((CompositeNode)node); + return node; + } + + private ExpressionNode transformFeatureNode(ReferenceNode feature) { + RankProfile.Macro macro = macros.get(feature.getName()); + if (macro == null) return feature; + return transform(macro.getRankingExpression().getRoot()); // inline recursively and return + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java new file mode 100644 index 00000000000..3e943377611 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -0,0 +1,968 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.search.query.ranking.Diversity; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; +import com.yahoo.searchlib.rankingexpression.FeatureList; +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.searchlib.rankingexpression.evaluation.Value; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.transform.ConstantDereferencer; +import com.yahoo.searchlib.rankingexpression.transform.Simplifier; +import com.yahoo.config.application.api.ApplicationPackage; + +import java.io.*; +import java.util.*; + +/** + * Represents a rank profile - a named set of ranking settings + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class RankProfile implements Serializable, Cloneable { + + /** The search definition-unique name of this rank profile */ + private final String name; + + /** The search definition owning this profile, or null if none */ + private Search search=null; + + /** The name of the rank profile inherited by this */ + private String inheritedName = null; + + /** The match settings of this profile */ + protected MatchPhaseSettings matchPhaseSettings = null; + + /** The rank settings of this profile */ + protected Set<RankSetting> rankSettings = new java.util.LinkedHashSet<>(); + + /** The ranking expression to be used for first phase */ + private RankingExpression firstPhaseRanking= null; + + /** The ranking expression to be used for second phase */ + private RankingExpression secondPhaseRanking = null; + + /** Number of hits to be reranked in second phase, -1 means use default */ + private int rerankCount = -1; + + /** Mysterious attribute */ + private int keepRankCount = -1; + + private int numThreadsPerSearch = -1; + private int numSearchPartitions = -1; + + private double termwiseLimit = 1.0; + + /** The drop limit used to drop hits with rank score less than or equal to this value */ + private double rankScoreDropLimit = -Double.MAX_VALUE; + + private Set<ReferenceNode> summaryFeatures; + + private Set<ReferenceNode> rankFeatures; + + /** The properties of this - a multimap */ + private Map<String, List<RankProperty>> rankProperties = new LinkedHashMap<>(); + + private Boolean ignoreDefaultRankFeatures=null; + + private String secondPhaseRankingString=null; + + private String firstPhaseRankingString=null; + + private Map<String, Macro> macros= new LinkedHashMap<>(); + + private Set<String> filterFields = new HashSet<>(); + + private final RankProfileRegistry rankProfileRegistry; + + /** Constants in ranking expressions */ + private Map<String, Value> constants = new HashMap<>(); + + private final TypeSettings attributeTypes = new TypeSettings(); + + private final TypeSettings queryFeatureTypes = new TypeSettings(); + + /** + * Creates a new rank profile + * + * @param name the name of the new profile + * @param search the search definition owning this profile + * @param rankProfileRegistry The {@link com.yahoo.searchdefinition.RankProfileRegistry} to use for storing and looking up rank profiles. + */ + public RankProfile(String name, Search search, RankProfileRegistry rankProfileRegistry) { + this.name = name; + this.search = search; + this.rankProfileRegistry = rankProfileRegistry; + } + + public String getName() { return name; } + + /** + * Returns the search definition owning this, or null if none + * + * @return The search definition. + */ + public Search getSearch() { + return search; + } + + /** + * Sets the name of the rank profile this inherits. Both rank profiles must be present in the same search + * definition + * + * @param inheritedName The name of the profile that this inherits from. + */ + public void setInherited(String inheritedName) { + this.inheritedName = inheritedName; + } + + /** + * Returns the name of the profile this one inherits, or null if none is inherited + * + * @return The inherited name. + */ + public String getInheritedName() { + return inheritedName; + } + + /** + * Returns the inherited rank profile, or null if there is none + * + * @return The inherited profile. + */ + public RankProfile getInherited() { + if (getSearch()==null) return getInheritedFromRegistry(inheritedName); + RankProfile inheritedInThisSearch = rankProfileRegistry.getRankProfile(search, inheritedName); + if (inheritedInThisSearch!=null) return inheritedInThisSearch; + return getInheritedFromRegistry(inheritedName); + } + + private RankProfile getInheritedFromRegistry(String inheritedName) { + for (RankProfile r : rankProfileRegistry.allRankProfiles()) { + if (r.getName().equals(inheritedName)) { + return r; + } + } + return null; + } + + /** + * Returns whether this profile inherits (directly or indirectly) the given profile + * + * @param name The profile name to compare this to. + * @return Whether or not this inherits from the named profile. + */ + public boolean inherits(String name) { + RankProfile parent = getInherited(); + while (parent != null) { + if (parent.getName().equals(name)) + return true; + parent = parent.getInherited(); + } + return false; + } + + /** + * change match settings + * @param settings The new match settings + **/ + public void setMatchPhaseSettings(MatchPhaseSettings settings) { + settings.checkValid(); + this.matchPhaseSettings = settings; + } + + public MatchPhaseSettings getMatchPhaseSettings() { + MatchPhaseSettings settings = this.matchPhaseSettings; + if (settings != null ) return settings; + if (getInherited() != null) return getInherited().getMatchPhaseSettings(); + return null; + } + + public void addRankSetting(RankSetting rankSetting) { + rankSettings.add(rankSetting); + } + + public void addRankSetting(String fieldName, RankSetting.Type type, Object value) { + addRankSetting(new RankSetting(fieldName, type, value)); + } + + /** + * Returns the a rank setting of a field, or null if there is no such rank setting in this profile + * + * @param field The field whose settings to return. + * @param type The type that the field is required to be. + * @return The rank setting found, or null. + */ + public RankSetting getDeclaredRankSetting(String field, RankSetting.Type type) { + for (Iterator<RankSetting> i = declaredRankSettingIterator(); i.hasNext();) { + RankSetting setting = i.next(); + if (setting.getFieldName().equals(field) && + setting.getType().equals(type)) { + return setting; + } + } + return null; + } + + /** + * Returns a rank setting of field or index, or null if there is no such rank setting in this profile or one it + * inherits + * + * @param field The field whose settings to return. + * @param type The type that the field is required to be. + * @return The rank setting found, or null. + */ + public RankSetting getRankSetting(String field, RankSetting.Type type) { + RankSetting rankSetting = getDeclaredRankSetting(field, type); + if (rankSetting != null) return rankSetting; + + if (getInherited() != null) return getInherited().getRankSetting(field, type); + + return null; + } + + /** + * Returns the rank settings in this rank profile + * + * @return An iterator for the declared rank setting. + */ + public Iterator<RankSetting> declaredRankSettingIterator() { + return Collections.unmodifiableSet(rankSettings).iterator(); + } + + /** + * Returns all settings in this profile or any profile it inherits + * + * @return An iterator for all rank settings of this. + */ + public Iterator<RankSetting> rankSettingIterator() { + return rankSettings().iterator(); + } + + /** + * Returns a snapshot of the rank settings of this and everything it inherits + * Changes to the returned set will not be reflected in this rank profile. + */ + public Set<RankSetting> rankSettings() { + Set<RankSetting> allSettings = new LinkedHashSet<>(rankSettings); + RankProfile parent = getInherited(); + if (parent != null) + allSettings.addAll(parent.rankSettings()); + + return allSettings; + } + + public void addConstant(String name, Value value) { + constants.put(name, value.freeze()); + } + + public void addConstantTensor(String name, TensorValue value) { + addConstant(name, value); + } + + /** + * Returns an unmodifiable view of the constants to use in this. + */ + public Map<String, Value> getConstants() { + if (constants.isEmpty()) + return getInherited() != null ? getInherited().getConstants() : Collections.<String,Value>emptyMap(); + if (getInherited() == null || getInherited().getConstants().isEmpty()) + return Collections.unmodifiableMap(constants); + + Map<String, Value> combinedConstants = new HashMap<>(getInherited().getConstants()); + combinedConstants.putAll(constants); + return combinedConstants; + } + + public void addAttributeType(String attributeName, String attributeType) { + attributeTypes.addType(attributeName, attributeType); + } + + public Map<String, String> getAttributeTypes() { + return attributeTypes.getTypes(); + } + + public void addQueryFeatureType(String queryFeature, String queryFeatureType) { + queryFeatureTypes.addType(queryFeature, queryFeatureType); + } + + public Map<String, String> getQueryFeatureTypes() { + return queryFeatureTypes.getTypes(); + } + + /** + * Returns the ranking expression to use by this. This expression must not be edited. + * Returns null if no expression is set. + */ + public RankingExpression getFirstPhaseRanking() { + if (firstPhaseRanking!=null) return firstPhaseRanking; + if (getInherited()!=null) return getInherited().getFirstPhaseRanking(); + return null; + } + + public void setFirstPhaseRanking(RankingExpression rankingExpression) { + this.firstPhaseRanking=rankingExpression; + } + + /** + * Returns the ranking expression to use by this. This expression must not be edited. + * Returns null if no expression is set. + */ + public RankingExpression getSecondPhaseRanking() { + if (secondPhaseRanking!=null) return secondPhaseRanking; + if (getInherited()!=null) return getInherited().getSecondPhaseRanking(); + return null; + } + + public void setSecondPhaseRanking(RankingExpression rankingExpression) { + this.secondPhaseRanking=rankingExpression; + } + + /** + * Called by parser to store the expression string, for delayed evaluation + * @param exp ranking expression for second phase + */ + public void setSecondPhaseRankingString(String exp) { + this.secondPhaseRankingString = exp; + } + + /** + * Called by parser to store the expression string, for delayed evaluation + * @param exp ranking expression for first phase + */ + public void setFirstPhaseRankingString(String exp) { + this.firstPhaseRankingString = exp; + } + + /** Returns a read-only view of the summary features to use in this profile. This is never null */ + public Set<ReferenceNode> getSummaryFeatures() { + if (summaryFeatures!=null) return Collections.unmodifiableSet(summaryFeatures); + if (getInherited()!=null) return getInherited().getSummaryFeatures(); + return Collections.emptySet(); + } + + public void addSummaryFeature(ReferenceNode feature) { + if (summaryFeatures==null) + summaryFeatures=new LinkedHashSet<>(); + summaryFeatures.add(feature); + } + + /** + * Adds the content of the given feature list to the internal list of summary features. + * + * @param features The features to add. + */ + public void addSummaryFeatures(FeatureList features) { + for (ReferenceNode feature : features) { + addSummaryFeature(feature); + } + } + + /** Returns a read-only view of the rank features to use in this profile. This is never null */ + public Set<ReferenceNode> getRankFeatures() { + if (rankFeatures != null) return Collections.unmodifiableSet(rankFeatures); + if (getInherited() != null) return getInherited().getRankFeatures(); + return Collections.emptySet(); + } + + public void addRankFeature(ReferenceNode feature) { + if (rankFeatures==null) + rankFeatures=new LinkedHashSet<>(); + rankFeatures.add(feature); + } + + /** + * Adds the content of the given feature list to the internal list of rank features. + * + * @param features The features to add. + */ + public void addRankFeatures(FeatureList features) { + for (ReferenceNode feature : features) { + addRankFeature(feature); + } + } + + /** Returns a read only flattened list view of the rank properties to use in this profile. This is never null. */ + public List<RankProperty> getRankProperties() { + List<RankProperty> properties = new ArrayList<>(); + for (List<RankProperty> propertyList : getRankPropertyMap().values()) { + properties.addAll(propertyList); + } + return Collections.unmodifiableList(properties); + } + + /** Returns a read only map view of the rank properties to use in this profile. This is never null. */ + public Map<String, List<RankProperty>> getRankPropertyMap() { + if (rankProperties.size() == 0 && getInherited() == null) return Collections.emptyMap(); + if (rankProperties.size() == 0) return getInherited().getRankPropertyMap(); + if (getInherited() == null) return Collections.unmodifiableMap(rankProperties); + + // Neither is null + Map<String, List<RankProperty>> combined = new LinkedHashMap<>(getInherited().getRankPropertyMap()); + combined.putAll(rankProperties); // Don't combine values across inherited properties + return Collections.unmodifiableMap(combined); + } + + public void addRankProperty(String name, String parameter) { + addRankProperty(new RankProperty(name, parameter)); + } + + public void addRankProperty(RankProperty rankProperty) { + // Just the usual multimap semantics here + List<RankProperty> properties = rankProperties.get(rankProperty.getName()); + if (properties == null) { + properties = new ArrayList<>(1); + rankProperties.put(rankProperty.getName(), properties); + } + properties.add(rankProperty); + } + + public String toString() { + return "rank profile " + getName(); + } + + public int getRerankCount() { + if (rerankCount>=0) return rerankCount; + if (getInherited()!=null) return getInherited().getRerankCount(); + return -1; + } + + public int getNumThreadsPerSearch() { + return numThreadsPerSearch; + } + + public void setNumThreadsPerSearch(int numThreads) { + this.numThreadsPerSearch = numThreads; + } + + public void setNumSearchPartitions(int numSearchPartitions) { + this.numSearchPartitions = numSearchPartitions; + } + + public int getNumSearchPartitions() { return numSearchPartitions; } + + public double getTermwiseLimit() { return termwiseLimit; } + public void setTermwiseLimit(double termwiseLimit) { this.termwiseLimit = termwiseLimit; } + + /** Sets the rerank count. Set to -1 to use inherited */ + public void setRerankCount(int rerankCount) { + this.rerankCount = rerankCount; + } + + /** Whether we should ignore the default rank features. Set to null to use inherited */ + public void setIgnoreDefaultRankFeatures(Boolean ignoreDefaultRankFeatures) { + this.ignoreDefaultRankFeatures = ignoreDefaultRankFeatures; + } + + public boolean getIgnoreDefaultRankFeatures() { + if (ignoreDefaultRankFeatures!=null) return ignoreDefaultRankFeatures; + return (getInherited()!=null) && getInherited().getIgnoreDefaultRankFeatures(); + } + + /** + * Returns the string form of the second phase ranking expression. + * @return string form of second phase ranking expression + */ + public String getSecondPhaseRankingString() { + if (secondPhaseRankingString != null) return secondPhaseRankingString; + if (getInherited() != null) return getInherited().getSecondPhaseRankingString(); + return null; + } + + /** + * Returns the string form of the first phase ranking expression. + * @return string form of first phase ranking expression + */ + public String getFirstPhaseRankingString() { + if (firstPhaseRankingString != null) return firstPhaseRankingString; + if (getInherited() != null) return getInherited().getFirstPhaseRankingString(); + return null; + } + + public void addMacro(String name, boolean inline) { + macros.put(name, new Macro(name, inline)); + } + + /** Returns an unmodifiable view of the macros in this */ + public Map<String, Macro> getMacros() { + if (macros.size() == 0 && getInherited()==null) return Collections.emptyMap(); + if (macros.size() == 0) return getInherited().getMacros(); + if (getInherited() == null) return Collections.unmodifiableMap(macros); + + // Neither is null + Map<String, Macro> allMacros = new LinkedHashMap<>(getInherited().getMacros()); + allMacros.putAll(macros); + return Collections.unmodifiableMap(allMacros); + + } + + public int getKeepRankCount() { + if (keepRankCount>=0) return keepRankCount; + if (getInherited()!=null) return getInherited().getKeepRankCount(); + return -1; + } + + public void setKeepRankCount(int rerankArraySize) { + this.keepRankCount = rerankArraySize; + } + + public double getRankScoreDropLimit() { + if (rankScoreDropLimit>-Double.MAX_VALUE) return rankScoreDropLimit; + if (getInherited()!=null) return getInherited().getRankScoreDropLimit(); + return rankScoreDropLimit; + } + + public void setRankScoreDropLimit(double rankScoreDropLimit) { + this.rankScoreDropLimit = rankScoreDropLimit; + } + + public Set<String> filterFields() { + return filterFields; + } + + /** + * Returns all filter fields in this profile and any profile it inherits. + * @return the set of all filter fields + */ + public Set<String> allFilterFields() { + RankProfile parent = getInherited(); + Set<String> retval = new LinkedHashSet<>(); + if (parent != null) { + retval.addAll(parent.allFilterFields()); + } + retval.addAll(filterFields()); + return retval; + } + + /** + * Will take the parser-set textual ranking expressions and turn into objects + */ + public void parseExpressions() { + try { + parseRankingExpressions(); + parseMacros(); + } catch (ParseException e) { + throw new IllegalArgumentException(e); + } + } + + /** + * Passes the contents of macros on to parser. Then put all the implied rank properties + * from those macros into the profile's props map. + */ + private void parseMacros() throws ParseException { + for (Map.Entry<String, Macro> e : getMacros().entrySet()) { + String macroName = e.getKey(); + Macro macro = e.getValue(); + RankingExpression expr = parseRankingExpression(macroName, macro.getTextualExpression()); + macro.setRankingExpression(expr); + macro.setTextualExpression(expr.getRoot().toString()); + } + } + + /** + * Passes ranking expressions on to parser + * @throws ParseException if either of the ranking expressions could not be parsed + */ + private void parseRankingExpressions() throws ParseException { + if (getFirstPhaseRankingString() != null) + setFirstPhaseRanking(parseRankingExpression("firstphase", getFirstPhaseRankingString())); + if (getSecondPhaseRankingString() != null) + setSecondPhaseRanking(parseRankingExpression("secondphase", getSecondPhaseRankingString())); + } + + private RankingExpression parseRankingExpression(String expName, String exp) throws ParseException { + if (exp.trim().length() == 0) + throw new ParseException("Encountered an empty ranking expression in " + getName()+ ", " + expName + "."); + + try { + RankingExpression expression = new RankingExpression(openRankingExpressionReader(expName, exp.trim())); + expression.setName(expName); + return expression; + } + catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) { + ParseException exception = new ParseException("Could not parse ranking expression '" + exp.trim() + + "' in " + getName()+ ", " + expName + "."); + throw (ParseException)exception.initCause(e); + } + } + + private Reader openRankingExpressionReader(String expName, String expression) { + if ( ! expression.startsWith("file:")) return new StringReader(expression); + + String fileName = expression.substring("file:".length()).trim(); + if ( ! fileName.endsWith(ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX)) + fileName = fileName + ApplicationPackage.RANKEXPRESSION_NAME_SUFFIX; + + final File file = new File(fileName); + if ( ! (file.isAbsolute()) && file.getPath().contains("/")) // See ticket 4102122 + throw new IllegalArgumentException("In " + getName() +", " + expName + ", ranking references file '" + file + + "' in subdirectory, which is not supported."); + + return search.getRankingExpression(fileName); + } + + /** Shallow clones this */ + @Override + public RankProfile clone() { + try { + // Note: This treats RankingExpression in Macros as immutables even though they are not + RankProfile clone = (RankProfile)super.clone(); + clone.rankSettings = new LinkedHashSet<>(this.rankSettings); + clone.matchPhaseSettings = this.matchPhaseSettings; // hmm? + clone.summaryFeatures = summaryFeatures != null ? new LinkedHashSet<>(this.summaryFeatures) : null; + clone.rankFeatures = rankFeatures != null ? new LinkedHashSet<>(this.rankFeatures) : null; + clone.rankProperties = new LinkedHashMap<>(this.rankProperties); + clone.macros = new LinkedHashMap<>(this.macros); + clone.filterFields = new HashSet<>(this.filterFields); + clone.constants = new HashMap<>(this.constants); + return clone; + } + catch (CloneNotSupportedException e) { + throw new RuntimeException("Won't happen", e); + } + } + + /** + * Returns a copy of this where the content is optimized for execution. + * Compiled profiles should never be modified. + */ + public RankProfile compile() { + try { + RankProfile compiled = this.clone(); + compiled.compileThis(); + return compiled; + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Rank profile '" + getName() + "' is invalid", e); + } + } + + private void compileThis() { + parseExpressions(); + + checkNameCollisions(getMacros(), getConstants()); + + Map<String, Macro> compiledMacros = new LinkedHashMap<>(); + for (Map.Entry<String, Macro> macroEntry : getMacros().entrySet()) { + Macro compiledMacro = macroEntry.getValue().clone(); + compiledMacro.setRankingExpression(compile(macroEntry.getValue().getRankingExpression(), + getConstants(), Collections.<String, Macro>emptyMap())); + compiledMacros.put(macroEntry.getKey(), compiledMacro); + } + macros = compiledMacros; + Map<String, Macro> inlineMacros = keepInline(compiledMacros); + firstPhaseRanking = compile(this.getFirstPhaseRanking(), getConstants(), inlineMacros); + secondPhaseRanking = compile(this.getSecondPhaseRanking(), getConstants(), inlineMacros); + } + + private void checkNameCollisions(Map<String, Macro> macros, Map<String, Value> constants) { + for (Map.Entry<String, Macro> macroEntry : macros.entrySet()) { + if (constants.get(macroEntry.getKey()) != null) + throw new IllegalArgumentException("Cannot have both a constant and macro named '" + macroEntry.getKey() + "'"); + } + } + + private Map<String, Macro> keepInline(Map<String, Macro> macros) { + Map<String, Macro> inlineMacros = new HashMap<>(); + for (Map.Entry<String, Macro> entry : macros.entrySet()) + if (entry.getValue().getInline()) + inlineMacros.put(entry.getKey(), entry.getValue()); + return inlineMacros; + } + + private RankingExpression compile(RankingExpression expression, + Map<String, Value> constants, + Map<String, Macro> inlineMacros) { + if (expression == null) return null; + Map<String, String> rankPropertiesOutput = new HashMap<>(); + expression = new ConstantDereferencer(constants).transform(expression); + expression = new ConstantTensorTransformer(constants, rankPropertiesOutput).transform(expression); + expression = new MacroInliner(inlineMacros).transform(expression); + expression = new Simplifier().transform(expression); + for (Map.Entry<String, String> rankProperty : rankPropertiesOutput.entrySet()) { + addRankProperty(rankProperty.getKey(), rankProperty.getValue()); + } + return expression; + } + + /** + * A rank setting. The identity of a rank setting is its field name and type (not value). + * A rank setting is immutable. + */ + public static class RankSetting implements Serializable { + + private String fieldName; + + private Type type; + + /** The rank value */ + private Object value; + + public enum Type { + + RANKTYPE("rank-type"), + LITERALBOOST("literal-boost"), + WEIGHT("weight"), + PREFERBITVECTOR("preferbitvector",true); + + private String name; + + /** True if this setting really pertains to an index, not a field within an index */ + private boolean isIndexLevel; + + private Type(String name) { + this(name,false); + } + + private Type(String name,boolean isIndexLevel) { + this.name = name; + this.isIndexLevel=isIndexLevel; + } + + /** True if this setting really pertains to an index, not a field within an index */ + public boolean isIndexLevel() { return isIndexLevel; } + + /** @return The name of this type */ + public String getName() { + return name; + } + + public String toString() { + return "type: " + name; + } + + } + + public RankSetting(String fieldName, RankSetting.Type type, Object value) { + this.fieldName = fieldName; + this.type = type; + this.value = value; + } + + public String getFieldName() { return fieldName; } + + public Type getType() { return type; } + + public Object getValue() { return value; } + + /** @return The value as an int, or a negative value if it is not an integer */ + public int getIntValue() { + if (value instanceof Integer) { + return ((Integer)value); + } + else { + return -1; + } + } + + public int hashCode() { + return fieldName.hashCode() + 17 * type.hashCode(); + } + + public boolean equals(Object object) { + if (!(object instanceof RankSetting)) { + return false; + } + RankSetting other = (RankSetting)object; + return + fieldName.equals(other.fieldName) && + type.equals(other.type); + } + + public String toString() { + return type + " setting " + fieldName + ": " + value; + } + + } + + /** A rank property. Rank properties are Value Objects */ + public static class RankProperty implements Serializable { + + private String name; + private String value; + + public RankProperty(String name, String value) { + this.name = name; + this.value = value; + } + + public String getName() { return name; } + + public String getValue() { return value; } + + @Override + public int hashCode() { + return name.hashCode() + 17*value.hashCode(); + } + + @Override + public boolean equals(Object object) { + if (! (object instanceof RankProperty)) return false; + RankProperty other=(RankProperty)object; + return (other.name.equals(this.name) && other.value.equals(this.value)); + } + + @Override + public String toString() { + return name + " = " + value; + } + + } + + /** + * Represents a declared macro in the profile. It is, after parsing, transformed into ExpressionMacro + * + * @author vegardh + */ + public static class Macro implements Serializable, Cloneable { + + private String name=null; + private String textualExpression=null; + private RankingExpression expression=null; + private List<String> formalParams = new ArrayList<>(); + + /** True if this should be inlined into calling expressions. Useful for very cheap macros. */ + private final boolean inline; + + public Macro(String name, boolean inline) { + this.name = name; + this.inline = inline; + } + + public void addParam(String name) { + formalParams.add(name); + } + + public List<String> getFormalParams() { + return formalParams; + } + + public String getTextualExpression() { + return textualExpression; + } + + public void setTextualExpression(String textualExpression) { + this.textualExpression = textualExpression; + } + + public void setRankingExpression(RankingExpression expr) { + this.expression=expr; + } + + public RankingExpression getRankingExpression() { + return expression; + } + + public String getName() { + return name; + } + + public boolean getInline() { + return inline && formalParams.size() == 0; // only inline no-arg macros; + } + + public ExpressionFunction toExpressionMacro() { + return new ExpressionFunction(getName(), getFormalParams(), getRankingExpression()); + } + + @Override + public Macro clone() { + try { + return (Macro)super.clone(); + } + catch (CloneNotSupportedException e) { + throw new RuntimeException("Won't happen", e); + } + } + + @Override + public String toString() { + return "macro " + getName() + ": " + expression; + } + + } + + public static final class DiversitySettings { + private String attribute = null; + private int minGroups = 0; + private double cutoffFactor = 10; + private Diversity.CutoffStrategy cutoffStrategy = Diversity.CutoffStrategy.loose; + + public void setAttribute(String value) { attribute = value; } + public void setMinGroups(int value) { minGroups = value; } + public void setCutoffFactor(double value) { cutoffFactor = value; } + public void setCutoffStrategy(Diversity.CutoffStrategy strategy) { cutoffStrategy = strategy; } + public void setCutoffStrategy(String strategy) { cutoffStrategy = Diversity.CutoffStrategy.valueOf(strategy); } + public String getAttribute() { return attribute; } + public int getMinGroups() { return minGroups; } + public double getCutoffFactor() { return cutoffFactor; } + public Diversity.CutoffStrategy getCutoffStrategy() { return cutoffStrategy; } + public void checkValid() { + if (attribute == null || attribute.isEmpty()) { + throw new IllegalArgumentException("'diversity' did not set non-empty diversity attribute name."); + } + if (minGroups <= 0) { + throw new IllegalArgumentException("'diversity' did not set min-groups > 0"); + } + if (cutoffFactor < 1.0) { + throw new IllegalArgumentException("diversity.cutoff.factor must be larger or equal to 1.0."); + } + } + } + + public static class MatchPhaseSettings { + private String attribute = null; + private boolean ascending = false; + private int maxHits = 0; // try to get this many hits before degrading the match phase + private double maxFilterCoverage = 1.0; // Max coverage of original corpus that will trigger the filter. + private DiversitySettings diversity = null; + private double evaluationPoint = 0.20; + private double prePostFilterTippingPoint = 1.0; + + public void setDiversity(DiversitySettings value) { + value.checkValid(); + diversity = value; + } + public void setAscending(boolean value) { ascending = value; } + public void setAttribute(String value) { attribute = value; } + public void setMaxHits(int value) { maxHits = value; } + public void setMaxFilterCoverage(double value) { maxFilterCoverage = value; } + public void setEvaluationPoint(double evaluationPoint) { this.evaluationPoint = evaluationPoint; } + public void setPrePostFilterTippingPoint(double prePostFilterTippingPoint) { this.prePostFilterTippingPoint = prePostFilterTippingPoint; } + + public boolean getAscending() { return ascending; } + public String getAttribute() { return attribute; } + public int getMaxHits() { return maxHits; } + public double getMaxFilterCoverage() { return maxFilterCoverage; } + public DiversitySettings getDiversity() { return diversity; } + public double getEvaluationPoint() { return evaluationPoint; } + public double getPrePostFilterTippingPoint() { return prePostFilterTippingPoint; } + + public void checkValid() { + if (attribute == null) { + throw new IllegalArgumentException("match-phase did not set any attribute"); + } + if (! (maxHits > 0)) { + throw new IllegalArgumentException("match-phase did not set max-hits > 0"); + } + } + + } + + public static class TypeSettings { + + private final Map<String, String> types = new HashMap<>(); + + public void addType(String name, String type) { + types.put(name, type); + } + + public Map<String, String> getTypes() { + return Collections.unmodifiableMap(types); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java new file mode 100644 index 00000000000..a9ee3c3cc5c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java @@ -0,0 +1,90 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import java.util.*; + +/** + * Mapping from name to {@link RankProfile} as well as a reverse mapping of {@link RankProfile} to {@link Search}. + * Having both of these mappings consolidated here will make it easier to remove dependencies on these mappings at + * run time, since it is essentially only used when building rank profile config at deployment time. + * + * TODO: Reconsider the difference between local and global maps. Right now, the local maps might better be + * served from a different class owned by SearchBuilder. + * + * @author lulf + * @since 5.20 + */ +public class RankProfileRegistry { + + private final Map<RankProfile, Search> rankProfileToSearch = new LinkedHashMap<>(); + private final Map<Search, Map<String, RankProfile>> rankProfiles = new LinkedHashMap<>(); + /* These rank profiles can be overridden: 'default' rank profile, as that is documented to work. And 'unranked'. */ + static final Set<String> overridableRankProfileNames = new HashSet<>(Arrays.asList("default", "unranked")); + + public RankProfileRegistry() { + } + + public static RankProfileRegistry createRankProfileRegistryWithBuiltinRankProfiles(Search search) { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + rankProfileRegistry.addRankProfile(new DefaultRankProfile(search, rankProfileRegistry)); + rankProfileRegistry.addRankProfile(new UnrankedRankProfile(search, rankProfileRegistry)); + return rankProfileRegistry; + } + + /** + * Adds a rank profile to this registry + * + * @param rankProfile the rank profile to add + */ + public void addRankProfile(RankProfile rankProfile) { + if (!rankProfiles.containsKey(rankProfile.getSearch())) { + rankProfiles.put(rankProfile.getSearch(), new LinkedHashMap<>()); + } + checkForDuplicateRankProfile(rankProfile); + rankProfiles.get(rankProfile.getSearch()).put(rankProfile.getName(), rankProfile); + rankProfileToSearch.put(rankProfile, rankProfile.getSearch()); + } + + private void checkForDuplicateRankProfile(RankProfile rankProfile) { + final String rankProfileName = rankProfile.getName(); + RankProfile existingRangProfileWithSameName = rankProfiles.get(rankProfile.getSearch()).get(rankProfileName); + if (existingRangProfileWithSameName == null) return; + + if (!overridableRankProfileNames.contains(rankProfileName)) { + throw new IllegalArgumentException("Cannot add rank profile '" + rankProfileName + "' in search definition '" + + rankProfile.getSearch().getName() + "', since it already exists"); + } + } + + /** + * Returns a named rank profile, null if the search definition doesn't have one with the given name + * + * @param search The {@link Search} that owns the rank profile. + * @param name The name of the rank profile + * @return The RankProfile to return. + */ + public RankProfile getRankProfile(Search search, String name) { + return rankProfiles.get(search).get(name); + } + + /** + * Rank profiles that are collected across clusters. + * @return A set of global {@link RankProfile} instances. + */ + public Set<RankProfile> allRankProfiles() { + return rankProfileToSearch.keySet(); + } + + /** + * Rank profiles that are collected for a given search definition + * @param search {@link Search} to get rank profiles for. + * @return A collection of local {@link RankProfile} instances. + */ + public Collection<RankProfile> localRankProfiles(Search search) { + Map<String, RankProfile> mapping = rankProfiles.get(search); + if (mapping == null) { + return Collections.emptyList(); + } + return mapping.values(); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java new file mode 100644 index 00000000000..7d8a87cee8c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java @@ -0,0 +1,126 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.*; +import com.yahoo.document.annotation.AnnotationReferenceDataType; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.TemporarySDDocumentType; + +import java.util.*; +import java.util.logging.Level; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class SDDocumentTypeOrderer { + private Map<DataTypeName, SDDocumentType> createdSDTypes = new LinkedHashMap<>(); + private Set<Integer> seenTypes = new LinkedHashSet<>(); + List<SDDocumentType> processingOrder = new LinkedList<>(); + private DeployLogger deployLogger; + + public SDDocumentTypeOrderer(List<SDDocumentType> sdTypes, DeployLogger deployLogger) { + this.deployLogger = deployLogger; + for (SDDocumentType type : sdTypes) { + createdSDTypes.put(type.getDocumentName(), type); + } + DocumentTypeManager dtm = new DocumentTypeManager(); + for (DataType type : dtm.getDataTypes()) { + seenTypes.add(type.getId()); + } + + } + + List<SDDocumentType> getOrdered() { return processingOrder; } + + public void process() { + for (SDDocumentType type : createdSDTypes.values()) { + process(type); + } + } + private void process(SDDocumentType type) { + List<DataTypeName> toReplace = new ArrayList<>(); + for (SDDocumentType sdoc : type.getInheritedTypes()) { + if (sdoc instanceof TemporarySDDocumentType) { + toReplace.add(sdoc.getDocumentName()); + } + } + for (DataTypeName name : toReplace) { + SDDocumentType inherited = createdSDTypes.get(name); + if (inherited == null) { + throw new IllegalStateException("Document type '" + name + "' not found."); + } + process(inherited); + type.inherit(inherited); + } + visit(type); + } + + private void visit(SDDocumentType docOrStruct) { + int id; + if (docOrStruct.isStruct()) { + id = new StructDataType(docOrStruct.getName()).getId(); + } else { + id = new DocumentType(docOrStruct.getName()).getId(); + } + + if (seenTypes.contains(id)) { + return; + } else { + seenTypes.add((new StructDataType(docOrStruct.getName()).getId())); + } + + + for (Field field : docOrStruct.fieldSet()) { + if (!seenTypes.contains(field.getDataType().getId())) { + //we haven't seen this before, do it + visit(field.getDataType()); + } + } + processingOrder.add(docOrStruct); + } + + private SDDocumentType find(String name) { + SDDocumentType sdDocType = createdSDTypes.get(new DataTypeName(name)); + if (sdDocType != null) { + return sdDocType; + } + for(SDDocumentType sdoc : createdSDTypes.values()) { + for (SDDocumentType stype : sdoc.getTypes()) { + if (stype.getName().equals(name)) { + return stype; + } + } + } + return null; + } + private void visit(DataType type) { + if (type instanceof StructuredDataType) { + StructuredDataType structType = (StructuredDataType) type; + SDDocumentType sdDocType = find(structType.getName()); + if (sdDocType == null) { + throw new IllegalArgumentException("Could not find struct '" + type.getName() + "'."); + } + visit(sdDocType); + return; + } + + if (type instanceof MapDataType) { + MapDataType mType = (MapDataType) type; + visit(mType.getValueType()); + visit(mType.getKeyType()); + } else if (type instanceof WeightedSetDataType) { + WeightedSetDataType wType = (WeightedSetDataType) type; + visit(wType.getNestedType()); + } else if (type instanceof CollectionDataType) { + CollectionDataType cType = (CollectionDataType) type; + visit(cType.getNestedType()); + } else if (type instanceof AnnotationReferenceDataType) { + //do nothing + } else if (type instanceof PrimitiveDataType) { + //do nothing + } else { + deployLogger.log(Level.WARNING, "Unknown type : " + type); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java new file mode 100644 index 00000000000..43c6bb4b441 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java @@ -0,0 +1,555 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.document.*; +import com.yahoo.searchdefinition.document.*; +import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; + +import java.io.Reader; +import java.io.Serializable; +import java.util.*; +import java.util.logging.Logger; + +/** + * <p>A search definition describes (or uses) some document types, defines how these are turned into a relevancy tuned + * index through indexing and how data from documents should be served at search time.</p> <p>The identity of this + * class is its name.</p> + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +// TODO: Make a class owned by this, for each of these responsibilities: +// Managing indexes, managing attributes, managing summary classes. +// Ensure that after the processing step, all implicit instances of the above types are explicitly represented +public class Search implements Serializable { + + private static final Logger log = Logger.getLogger(Search.class.getName()); + private static final String SD_DOC_FIELD_NAME = "sddocname"; + private static final List<String> RESERVED_NAMES = Arrays.asList( + "index", "index_url", "summary", "attribute", "select_input", "host", "documentid", + "position", "split_foreach", "tokenize", "if", "else", "switch", "case", SD_DOC_FIELD_NAME, "relevancy"); + + /** + * @return True if the given field name is a reserved name. + */ + public static boolean isReservedName(String name) { + return RESERVED_NAMES.contains(name); + } + + // Field sets + private FieldSets fieldSets = new FieldSets(); + + // Whether or not this object has been processed. + private boolean processed; + + // The unique name of this search definition. + private String name; + + // True if this doesn't define a search, just some documents. + private boolean documentsOnly = false; + + // The stemming setting of this search definition. Default is SHORTEST. + private Stemming stemming = Stemming.SHORTEST; + + // Documents contained in this definition. + private SDDocumentType docType; + + // The extra fields of this search definition. + private Map<String, SDField> fields = new LinkedHashMap<>(); + + // The explicitly defined indices of this search definition. + private Map<String, Index> indices = new LinkedHashMap<>(); + + // The explicitly defined summaries of this search definition. + private Map<String, DocumentSummary> summaries = new LinkedHashMap<>(); + // _Must_ preserve order + + private ApplicationPackage sourceApplication; + + /** + * Creates a search definition which just holds a set of documents which should not (here, directly) be searchable + */ + protected Search() { + documentsOnly = true; + } + + /** + * Creates a proper search definition + * @param name of the the searchdefinition + * @param sourceApplication the application containing this + */ + public Search(String name, ApplicationPackage sourceApplication) { + this.sourceApplication = sourceApplication; + this.name = name; + } + + protected void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + /** + * Returns true if this doesn't define a search, just some documents + * + * @return if the searchdefinition only has documents + */ + public boolean isDocumentsOnly() { + return documentsOnly; + } + + /** + * Sets the stemming default of fields. Default is ALL + * + * @param stemming set default stemming for this searchdefinition + * @throws NullPointerException if this is attempted set to null + */ + public void setStemming(Stemming stemming) { + if (stemming == null) { + throw new NullPointerException("The stemming setting of a search definition " + + "can not be null"); + } + this.stemming = stemming; + } + + /** + * Returns whether fields should be stemmed by default or not. Default is ALL. This is never null. + * + * @return the default stemming for this searchdefinition + */ + public Stemming getStemming() { + return stemming; + } + + /** + * Adds a document type which is defined in this search definition + * + * @param document the document type to add + */ + public void addDocument(SDDocumentType document) { + if (docType != null) { + throw new IllegalArgumentException("Searchdefinition cannot have more than one document"); + } + docType = document; + } + + /** + * Gets a document from this search definition + * + * @param name the name of the document to return + * @return the contained or used document type, or null if there is no such document + */ + public SDDocumentType getDocument(String name) { + if (docType != null && name.equals(docType.getName())) { + return docType; + } + return null; + } + + /** + * @return true if the document has been added. + */ + public boolean hasDocument() { + return docType != null; + } + + /** + * @return The document in this search. + */ + public SDDocumentType getDocument() { + return docType; + } + + /** + * Returns a list of all the fields of this search definition, that is all fields in all documents, in the documents + * they inherit, and all extra fields. The caller receives ownership to the list - subsequent changes to it will not + * impact this Search + * + * @return the list of fields in this searchdefinition + */ + public List<SDField> allFieldsList() { + List<SDField> allFields = new ArrayList<>(); + allFields.addAll(extraFieldList()); + for (Field field : docType.fieldSet()) { + allFields.add((SDField)field); + } + return allFields; + } + + /** + * Returns the content of a ranking expression file + */ + public Reader getRankingExpression(String fileName) { + return sourceApplication.getRankingExpression(fileName); + } + + /** + * Returns a field defined in this search definition or one if its documents. Fields in this search definition takes + * precedence over document fields having the same name + * + * @param name of the field + * @return the SDField representing the field + */ + public SDField getField(String name) { + SDField field = getExtraField(name); + if (field != null) { + return field; + } + return (SDField)docType.getField(name); + } + + /** + * Returns a field defined in one of the documents of this search definition. This does <b>not</b> include the extra + * fields defined outside of a document (those accessible through the getExtraField() method). + * + * @param name The name of the field to return. + * @return The named field, or null if not found. + */ + public SDField getDocumentField(String name) { + return (SDField)docType.getField(name); + } + + /** + * Adds an extra field of this search definition not contained in a document + * + * @param field to add to the searchdefinitions list of external fields. + */ + public void addExtraField(SDField field) { + if (fields.containsKey(field.getName())) { + log.warning("Duplicate field " + field.getName() + " in search definition " + getName()); + } else { + field.setIsExtraField(true); + fields.put(field.getName(), field); + } + } + + public Collection<SDField> extraFieldList() { + return fields.values(); + } + public Collection<SDField> allExtraFields() { + Map<String, SDField> extraFields = new TreeMap<>(); + for (Field field : docType.fieldSet()) { + SDField sdField = (SDField) field; + if (sdField.isExtraField()) { + extraFields.put(sdField.getName(), sdField); + } + } + for (SDField field : extraFieldList()) { + extraFields.put(field.getName(), field); + } + return extraFields.values(); + } + + /** + * Returns a field by name, or null if it is not present + * + * @param fieldName the name of the external field to get + * @return the SDField of this name + */ + public SDField getExtraField(String fieldName) { + return fields.get(fieldName); + } + + /** + * Adds an explicitly defined index to this search definition + * + * @param index the index to add + */ + public void addIndex(Index index) { + indices.put(index.getName(), index); + } + + /** + * <p>Returns an index, or null if no index with this name has had some <b>explicit settings</b> applied. Even if + * this returns null, the index may be implicitly defined by an indexing statement.</p> + * <p>This will return the + * index whether it is defined on this search or on one of its fields</p> + * + * @param name the name of the index to get + * @return the index requested + */ + public Index getIndex(String name) { + List<Index> sameIndices = new ArrayList<>(1); + Index searchIndex = indices.get(name); + if (searchIndex != null) { + sameIndices.add(searchIndex); + } + + for (SDField field : allFieldsList()) { + Index index = field.getIndex(name); + if (index != null) { + sameIndices.add(index); + } + } + if (sameIndices.size() == 0) { + return null; + } + if (sameIndices.size() == 1) { + return sameIndices.get(0); + } + return consolidateIndices(sameIndices); + } + + public boolean existsIndex(String name) { + if (indices.get(name) != null) { + return true; + } + for (SDField field : allFieldsList()) { + if (field.existsIndex(name)) { + return true; + } + } + return false; + } + + /** + * Consolidates a set of index settings for the same index into one + * + * @param indices The list of indexes to consolidate. + * @return The consolidated index + */ + private Index consolidateIndices(List<Index> indices) { + Index first = indices.get(0); + Index consolidated = new Index(first.getName()); + consolidated.setRankType(first.getRankType()); + consolidated.setType(first.getType()); + for (Index current : indices) { + if (current.isPrefix()) { + consolidated.setPrefix(true); + } + + if (consolidated.getRankType() == null) { + consolidated.setRankType(current.getRankType()); + } else { + if (current.getRankType() != null && + !consolidated.getRankType().equals(current.getRankType())) + { + log.warning("Conflicting rank type settings for " + + first.getName() + " in " + this + ", using " + + consolidated.getRankType()); + } + } + + for (Iterator<String> j = current.aliasIterator(); j.hasNext();) { + consolidated.addAlias(j.next()); + } + } + return consolidated; + } + + /** + * All explicitly defined indices, both on this search definition itself (returned first) and all its fields + * + * @return The list of explicit defined indexes. + */ + public List<Index> getExplicitIndices() { + List<Index> allIndices = new ArrayList<>(indices.values()); + for (SDField field : allFieldsList()) { + for (Index index : field.getIndices().values()) { + allIndices.add(index); + } + } + return Collections.unmodifiableList(allIndices); + } + + /** + * Adds an explicitly defined summary to this search definition + * + * @param summary The summary to add. + */ + public void addSummary(DocumentSummary summary) { + summaries.put(summary.getName(), summary); + } + + /** + * <p>Returns a summary class defined by this search definition, or null if no summary with this name is defined. + * The default summary, named "default" is always present.</p> + * + * @param name the name of the summary to get. + * @return Summary found. + */ + public DocumentSummary getSummary(String name) { + return summaries.get(name); + } + + /** + * Returns the first explicit instance found of a summary field with this name, or null if not present (implicitly + * or explicitly) in any summary class. + * + * @param name The name of the summaryfield to get. + * @return SummaryField to return. + */ + public SummaryField getSummaryField(String name) { + for (DocumentSummary summary : summaries.values()) { + SummaryField summaryField = summary.getSummaryField(name); + if (summaryField != null) { + return summaryField; + } + } + return null; + } + + /** + * Returns the first explicit instance found of a summary field with this name, or null if not present explicitly in + * any summary class + * + * @param name Thge name of the explicit summary field to get. + * @return The SummaryField found. + */ + public SummaryField getExplicitSummaryField(String name) { + for (DocumentSummary summary : summaries.values()) { + SummaryField summaryField = summary.getSummaryField(name); + if (summaryField != null && !summaryField.isImplicit()) { + return summaryField; + } + } + return null; + } + + /** + * Summaries defined by fields of this search definition. The default summary, named "default", is always the first + * one in the returned iterator. + * + * @return The map of document summaries. + */ + public Map<String, DocumentSummary> getSummaries() { + return summaries; + } + + /** + * <p>Returns all summary fields, of all document summaries, which has the given field as source. If there are + * multiple summary fields with the same name, the last one will be used (they should all have the same content, if + * this is a valid search definition).</p> <p>The map gets owned by the receiver.</p> + * + * @param field The source field. + * @return The map of summary fields found. + */ + public Map<String, SummaryField> getSummaryFields(SDField field) { + Map<String, SummaryField> summaryFields = new java.util.LinkedHashMap<>(); + for (DocumentSummary documentSummary : summaries.values()) { + for (SummaryField summaryField : documentSummary.getSummaryFields()) { + if (summaryField.hasSource(field.getName())) { + summaryFields.put(summaryField.getName(), summaryField); + } + } + } + return summaryFields; + } + + /** + * <p>Returns one summary field for each summary field name. If there are multiple summary fields with the same + * name, the last one will be used. Multiple fields of the same name should all have the same content in a valid + * search definition, except from the destination set. So this method can be used for all summary handling except + * processing the destination set.</p> <p>The map gets owned by the receiver.</p> + * + * @return Map of unique summary fields + */ + public Map<String, SummaryField> getUniqueNamedSummaryFields() { + Map<String, SummaryField> summaryFields = new java.util.LinkedHashMap<>(); + for (DocumentSummary documentSummary : summaries.values()) { + for (SummaryField summaryField : documentSummary.getSummaryFields()) { + summaryFields.put(summaryField.getName(), summaryField); + } + } + return summaryFields; + } + + public int hashCode() { + return name.hashCode(); + } + + /** + * Returns the first occurence of an attribute having this name, or null if none + * + * @param name Name of attribute + * @return The Attribute with given name. + */ + public Attribute getAttribute(String name) { + for (SDField field : allFieldsList()) { + Attribute attribute = field.getAttributes().get(name); + if (attribute != null) { + return attribute; + } + } + return null; + } + + public boolean equals(Object o) { + if (!(o instanceof Search)) { + return false; + } + + Search other = (Search)o; + return getName().equals(other.getName()); + } + + public String toString() { + return "search definition '" + getName() + "'"; + } + + public boolean isAccessingDiskSummary(SummaryField field) { + if (!field.getTransform().isInMemory()) { + return true; + } + if (field.getSources().size() == 0) { + return isAccessingDiskSummary(getName()); + } + for (SummaryField.Source source : field.getSources()) { + if (isAccessingDiskSummary(source.getName())) { + return true; + } + } + return false; + } + + private boolean isAccessingDiskSummary(String source) { + SDField field = getField(source); + if (field == null) { + return false; + } + if (field.doesSummarying() && !field.doesAttributing()) { + return true; + } + return false; + } + + public void process() { + if (processed) { + throw new IllegalStateException("Search '" + getName() + "' already processed."); + } + processed = true; + } + + public boolean isProcessed() { + return processed; + } + + /** + * The field set settings for this search + * + * @return field set settings for this + */ + public FieldSets fieldSets() { + return fieldSets; + } + + /** + * For adding structs defined in document scope + * + * @param dt The struct to add. + * @return self, for chaining + */ + public Search addType(SDDocumentType dt) { + docType.addType(dt); // TODO This is a very very dirty thing. It must go + return this; + } + + public Search addAnnotation(SDAnnotationType dt) { + docType.addAnnotation(dt); + return this; + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java new file mode 100644 index 00000000000..9c9bc559dc3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/SearchBuilder.java @@ -0,0 +1,429 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.io.IOUtils; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.searchdefinition.derived.SearchOrderer; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.searchdefinition.parser.SDParser; +import com.yahoo.searchdefinition.parser.SimpleCharStream; +import com.yahoo.searchdefinition.parser.TokenMgrError; +import com.yahoo.searchdefinition.processing.Processing; +import com.yahoo.vespa.documentmodel.DocumentModel; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; + +/** + * Helper class for importing {@link Search} objects in an unambiguous way. The pattern for using this is to 1) Import + * all available search definitions, using the importXXX() methods, 2) provide the available rank types and rank + * expressions, using the setRankXXX() methods, 3) invoke the {@link #build()} method, and 4) retrieve the built + * search objects using the {@link #getSearch(String)} method. + * + * @author TODO: Who created this? + */ +// TODO: This should be cleaned up and more or maybe completely taken over by MockApplicationPackage +public class SearchBuilder { + + private final DocumentTypeManager docTypeMgr = new DocumentTypeManager(); + private List<Search> searchList = new LinkedList<>(); + private ApplicationPackage app = null; + private boolean isBuilt = false; + private DocumentModel model = new DocumentModel(); + private RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + + public SearchBuilder() { + this.app = MockApplicationPackage.createEmpty(); + } + + public SearchBuilder(ApplicationPackage app) { + this.app = app; + } + + public SearchBuilder(ApplicationPackage app, RankProfileRegistry rankProfileRegistry) { + this.app = app; + this.rankProfileRegistry = rankProfileRegistry; + } + + public SearchBuilder(RankProfileRegistry rankProfileRegistry) { + this(MockApplicationPackage.createEmpty(), rankProfileRegistry); + } + + /** + * Import search definition. + * + * @param fileName The name of the file to import. + * @param deployLogger Logger for deploy messages. + * @return The name of the imported object. + * @throws IOException Thrown if the file can not be read for some reason. + * @throws ParseException Thrown if the file does not contain a valid search definition. ``` + */ + public String importFile(String fileName, DeployLogger deployLogger) throws IOException, ParseException { + File file = new File(fileName); + return importString(IOUtils.readFile(file), file.getAbsoluteFile().getParent(), deployLogger); + } + + /** + * Import search definition. + * + * @param fileName The name of the file to import. + * @return The name of the imported object. + * @throws IOException Thrown if the file can not be read for some reason. + * @throws ParseException Thrown if the file does not contain a valid search definition. + */ + public String importFile(String fileName) throws IOException, ParseException { + return importFile(fileName, new BaseDeployLogger()); + } + public String importFile(Path file) throws IOException, ParseException { + return importFile(file.toString(), new BaseDeployLogger()); + } + + /** + * Reads and parses the search definition string provided by the given reader. Once all search definitions have been + * imported, call {@link #build()}. + * + * @param reader The reader whose content to import. + * @param searchDefDir The path to use when resolving file references. + * @return The name of the imported object. + * @throws ParseException Thrown if the file does not contain a valid search definition. + */ + public String importReader(NamedReader reader, String searchDefDir, DeployLogger deployLogger) throws IOException, ParseException { + return importString(IOUtils.readAll(reader), searchDefDir, deployLogger); + } + + /** + * See #{@link #importReader} + * + * Convenience, should only be used for testing as logs will be swallowed. + */ + public String importReader(NamedReader reader, String searchDefDir) throws IOException, ParseException { + return importString(IOUtils.readAll(reader), searchDefDir, new BaseDeployLogger()); + } + + /** + * Import search definition. + * + * @param str the string to parse. + * @return the name of the imported object. + * @throws ParseException thrown if the file does not contain a valid search definition. + */ + public String importString(String str) throws ParseException { + return importString(str, null, new BaseDeployLogger()); + } + + private String importString(String str, String searchDefDir, DeployLogger deployLogger) throws ParseException { + Search search; + SimpleCharStream stream = new SimpleCharStream(str); + try { + search = new SDParser(stream, deployLogger, app, rankProfileRegistry).search(docTypeMgr, searchDefDir); + } catch (TokenMgrError e) { + throw new ParseException("Unknown symbol: " + e.getMessage()); + } catch (ParseException pe) { + throw new ParseException(stream.formatException(pe.getMessage())); + } + return importRawSearch(search); + } + + /** + * Registers the given search object to the internal list of objects to be processed during {@link #build()}. A + * {@link Search} object is considered to be "raw" if it has not already been processed. This is the case for most + * programmatically constructed search objects used in unit tests. + * + * @param rawSearch The object to import. + * @return The name of the imported object. + * @throws IllegalArgumentException Thrown if the given search object has already been processed. + */ + public String importRawSearch(Search rawSearch) { + if (rawSearch.getName() == null) { + throw new IllegalArgumentException("Search has no name."); + } + String rawName = rawSearch.getName(); + if (rawSearch.isProcessed()) { + throw new IllegalArgumentException("A search definition with a search section called '" + rawName + + "' has already been processed."); + } + for (Search search : searchList) { + if (rawName.equals(search.getName())) { + throw new IllegalArgumentException("A search definition with a search section called '" + rawName + + "' has already been added."); + } + } + searchList.add(rawSearch); + return rawName; + } + + /** + * Registers the given search object to the internal list of objects to be processed during {@link #build()}. A + * {@link Search} object is considered to be "processed" if it has not already been processed. This is the case for most + * programmatically constructed search objects used in unit tests. + * + * @param processed The object to import. + * @return The name of the imported object. + * @throws IllegalArgumentException Thrown if the given search object has already been processed. + */ + public String importProcessedSearch(Search processed) { + if (processed.getName() == null) { + throw new IllegalArgumentException("Search has no name."); + } + String rawName = processed.getName(); + if (!processed.isProcessed()) { + throw new IllegalArgumentException("A search definition with a search section called '" + rawName + + "' has not been processed."); + } + for (Search search : searchList) { + if (rawName.equals(search.getName())) { + throw new IllegalArgumentException("A search definition with a search section called '" + rawName + + "' has already been added."); + } + } + searchList.add(processed); + return rawName; + } + + /** + * Only for testing. + * + * Processes and finalizes the imported search definitions so that they become available through the {@link + * #getSearch(String)} method. + * + * @throws IllegalStateException Thrown if this method has already been called. + */ + public void build() { + build(new BaseDeployLogger(), new QueryProfiles()); + } + + /** + * Processes and finalizes the imported search definitions so that they become available through the {@link + * #getSearch(String)} method. + * + * @throws IllegalStateException Thrown if this method has already been called. + * @param deployLogger The logger to use during build + * @param queryProfiles The query profiles contained in the application this search is part of. + */ + public void build(DeployLogger deployLogger, QueryProfiles queryProfiles) { + if (isBuilt) { + throw new IllegalStateException("Searches already built."); + } + List<Search> built = new ArrayList<>(); + List<SDDocumentType> sdocs = new ArrayList<>(); + sdocs.add(SDDocumentType.VESPA_DOCUMENT); + for (Search search : searchList) { + if (search.hasDocument()) { + sdocs.add(search.getDocument()); + } + } + SDDocumentTypeOrderer orderer = new SDDocumentTypeOrderer(sdocs, deployLogger); + orderer.process(); + for (SDDocumentType sdoc : orderer.getOrdered()) { + new FieldOperationApplierForStructs().process(sdoc); + new FieldOperationApplier().process(sdoc); + } + + DocumentModelBuilder builder = new DocumentModelBuilder(model); + for (Search search : new SearchOrderer().order(searchList)) { + new FieldOperationApplierForSearch().process(search); + // These two needed for a couple of old unit tests, ideally these are just read from app + process(search, deployLogger, queryProfiles); + built.add(search); + } + builder.addToModel(searchList); + if ( ! builder.valid() ) { + throw new IllegalArgumentException("Impossible to build a correct model."); + } + searchList = built; + isBuilt = true; + } + + /** + * Processes and returns the given {@link Search} object. This method has been factored out of the {@link + * #build()} method so that subclasses can choose not to build anything. + * + * @param search The object to build. + */ + protected void process(Search search, DeployLogger deployLogger, QueryProfiles queryProfiles) { + Processing.process(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + /** + * Convenience method to call {@link #getSearch(String)} when there is only a single {@link Search} object + * built. This method will never return null. + * + * @return The build object. + * @throws IllegalStateException Thrown if there is not exactly one search. + */ + public Search getSearch() { + if ( ! isBuilt) throw new IllegalStateException("Searches not built."); + if (searchList.size() != 1) + throw new IllegalStateException("This call only works if we have 1 search definition. Search definitions: " + searchList); + + return searchList.get(0); + } + + public DocumentModel getModel() { + return model; + } + + /** + * Returns the built {@link Search} object that has the given name. If the name is unknown, this method will simply + * return null. + * + * @param name the name of the search definition to return, + * or null to return the only one or throw an exception if there are multiple to choose from + * @return the built object, or null if none with this name + * @throws IllegalStateException if {@link #build()} has not been called. + */ + public Search getSearch(String name) { + if ( ! isBuilt) throw new IllegalStateException("Searches not built."); + if (name == null) return getSearch(); + + for (Search search : searchList) + if (search.getName().equals(name)) return search; + return null; + } + + /** + * Convenience method to return a list of all built {@link Search} objects. + * + * @return The list of built searches. + */ + public List<Search> getSearchList() { + return new ArrayList<>(searchList); + } + + /** + * Convenience factory method to import and build a {@link Search} object from a string. + * + * @param sd The string to build from. + * @return The built {@link SearchBuilder} object. + * @throws ParseException Thrown if there was a problem parsing the string. + */ + public static SearchBuilder createFromString(String sd) throws ParseException { + SearchBuilder builder = new SearchBuilder(MockApplicationPackage.createEmpty()); + builder.importString(sd); + builder.build(); + return builder; + } + + /** + * Convenience factory method to import and build a {@link Search} object from a file. Only for testing. + * + * @param fileName the file to build from + * @return the built {@link SearchBuilder} object + * @throws IOException if there was a problem reading the file. + * @throws ParseException if there was a problem parsing the file content. + */ + public static SearchBuilder createFromFile(String fileName) throws IOException, ParseException { + return createFromFile(fileName, new BaseDeployLogger(), new RankProfileRegistry()); + } + + /** + * Convenience factory method to import and build a {@link Search} object from a file. + * + * @param fileName the file to build from. + * @param deployLogger logger for deploy messages. + * @param rankProfileRegistry registry for rank profiles. + * @return the built {@link SearchBuilder} object. + * @throws IOException if there was a problem reading the file. + * @throws ParseException if there was a problem parsing the file content. + */ + public static SearchBuilder createFromFile(String fileName, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) + throws IOException, ParseException { + SearchBuilder builder = new SearchBuilder(MockApplicationPackage.createEmpty(), rankProfileRegistry); + builder.importFile(fileName); + builder.build(deployLogger, new QueryProfiles()); + return builder; + } + + public static SearchBuilder createFromDirectory(String dir) throws IOException, ParseException { + return createFromDirectory(dir, new RankProfileRegistry()); + } + public static SearchBuilder createFromDirectory(String dir, RankProfileRegistry rankProfileRegistry) throws IOException, ParseException { + SearchBuilder builder = new SearchBuilder(MockApplicationPackage.fromSearchDefinitionDirectory(dir), rankProfileRegistry); + for (Iterator<Path> i = Files.list(new File(dir).toPath()).filter(p -> p.getFileName().toString().endsWith(".sd")).iterator(); i.hasNext(); ) { + builder.importFile(i.next()); + } + builder.build(new BaseDeployLogger(), new QueryProfiles()); + return builder; + } + + /** + * Convenience factory method to import and build a {@link Search} object from a file. Only for testing. + * + * @param fileName The file to build from. + * @return The built {@link Search} object. + * @throws IOException Thrown if there was a problem reading the file. + * @throws ParseException Thrown if there was a problem parsing the file content. + */ + public static Search buildFromFile(String fileName) throws IOException, ParseException { + return buildFromFile(fileName, new BaseDeployLogger(), new RankProfileRegistry()); + } + + /** + * Convenience factory method to import and build a {@link Search} object from a file. + * + * @param fileName The file to build from. + * @param rankProfileRegistry Registry for rank profiles. + * @return The built {@link Search} object. + * @throws IOException Thrown if there was a problem reading the file. + * @throws ParseException Thrown if there was a problem parsing the file content. + */ + public static Search buildFromFile(String fileName, RankProfileRegistry rankProfileRegistry) + throws IOException, ParseException { + return buildFromFile(fileName, new BaseDeployLogger(), rankProfileRegistry); + } + + /** + * Convenience factory method to import and build a {@link Search} object from a file. + * + * @param fileName The file to build from. + * @param deployLogger Logger for deploy messages. + * @param rankProfileRegistry Registry for rank profiles. + * @return The built {@link Search} object. + * @throws IOException Thrown if there was a problem reading the file. + * @throws ParseException Thrown if there was a problem parsing the file content. + */ + public static Search buildFromFile(String fileName, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) + throws IOException, ParseException { + return createFromFile(fileName, deployLogger, rankProfileRegistry).getSearch(); + } + + /** + * Convenience factory method to import and build a {@link Search} object from a raw object. + * + * @param rawSearch The raw object to build from. + * @return The built {@link SearchBuilder} object. + * @see #importRawSearch(Search) + */ + public static SearchBuilder createFromRawSearch(Search rawSearch, RankProfileRegistry rankProfileRegistry) { + SearchBuilder builder = new SearchBuilder(rankProfileRegistry); + builder.importRawSearch(rawSearch); + builder.build(); + return builder; + } + + /** + * Convenience factory method to import and build a {@link Search} object from a raw object. + * + * @param rawSearch The raw object to build from. + * @return The built {@link Search} object. + * @see #importRawSearch(Search) + */ + public static Search buildFromRawSearch(Search rawSearch, RankProfileRegistry rankProfileRegistry) { + return createFromRawSearch(rawSearch, rankProfileRegistry).getSearch(); + } + + public RankProfileRegistry getRankProfileRegistry() { + return rankProfileRegistry; + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java b/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java new file mode 100644 index 00000000000..3df0c1f9953 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/UnprocessingSearchBuilder.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.io.IOException; + +/** + * A SearchBuilder that does not run the processing chain for searches + * + */ +public class UnprocessingSearchBuilder extends SearchBuilder { + + public UnprocessingSearchBuilder(ApplicationPackage app, RankProfileRegistry rankProfileRegistry) { + super(app, rankProfileRegistry); + } + + public UnprocessingSearchBuilder() { + super(); + } + + public UnprocessingSearchBuilder(RankProfileRegistry rankProfileRegistry) { + super(rankProfileRegistry); + } + + @Override + public void process(Search search, DeployLogger deployLogger, QueryProfiles queryProfiles) { + // empty + } + + public static Search buildUnprocessedFromFile(String fileName) throws IOException, ParseException { + SearchBuilder builder = new UnprocessingSearchBuilder(); + builder.importFile(fileName); + builder.build(); + return builder.getSearch(); + } +}
\ No newline at end of file diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/UnproperSearch.java b/config-model/src/main/java/com/yahoo/searchdefinition/UnproperSearch.java new file mode 100644 index 00000000000..beab254bd89 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/UnproperSearch.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + +import com.yahoo.searchdefinition.document.SDDocumentType; + +/** + * A search that was derived from an sd file containing no search element(s), only + * document specifications. + * + * @author vegardh + * + */ + // Award for best class name goes to ... +public class UnproperSearch extends Search { + // This class exists because the parser accepts SD files without search { ... , and + // there are unit tests using it too, BUT there are many nullpointer bugs if you try to + // deploy such a file. Using this class to try to catch those. + // TODO: Throw away this when we properly support doc-only SD files. + + public UnproperSearch() { + // empty + } + + @Override + public void addDocument(SDDocumentType docType) { + if (getName() == null) { + setName(docType.getName()); + } + super.addDocument(docType); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/UnrankedRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/UnrankedRankProfile.java new file mode 100644 index 00000000000..f465112fbaf --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/UnrankedRankProfile.java @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition; + + +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.parser.ParseException; + +/** + * A low-cost ranking profile to use for watcher queries etc. + * + * @author <a href="mailto:vegardh@yahoo-inc.com">Vegard Havdal</a> + */ +public class UnrankedRankProfile extends RankProfile { + + public UnrankedRankProfile(Search search, RankProfileRegistry rankProfileRegistry) { + super("unranked", search, rankProfileRegistry); + try { + RankingExpression exp = new RankingExpression("value(0)"); + this.setFirstPhaseRanking(exp); + } catch (ParseException e) { + throw new IllegalArgumentException("Could not parse the ranking expression 'value(0)' when setting up " + + "the 'unranked' rank profile"); + } + this.setIgnoreDefaultRankFeatures(true); + this.setKeepRankCount(0); + this.setRerankCount(0); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java new file mode 100644 index 00000000000..1130e5630a3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -0,0 +1,157 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.document.*; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.Ranking; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.Sorting; +import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.config.subscription.ConfigInstanceUtil; +import com.yahoo.vespa.indexinglanguage.expressions.ToPositionExpression; + +import java.util.*; + +/** + * The set of all attribute fields defined by a search definition + * + * @author <a href="mailto:bratseth@overture.com">Jon S Bratseth</a> + */ +public class AttributeFields extends Derived implements AttributesConfig.Producer { + private Map<String, Attribute> attributes = new java.util.LinkedHashMap<>(); + + /** + * Flag indicating if a position-attribute has been found + */ + private boolean hasPosition = false; + + public AttributeFields(Search search) { + derive(search); + } + + /** + * Derives everything from a field + */ + protected void derive(SDField field, Search search) { + if (field.usesStructOrMap() && + !field.getDataType().equals(PositionDataType.INSTANCE) && + !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))) + { + return; // Ignore struct fields for indexed search (only implemented for streaming search) + } + deriveAttributes(field); + } + + /** + * Return an attribute by name, or null if it doesn't exist + */ + public Attribute getAttribute(String attributeName) { + return attributes.get(attributeName); + } + + public boolean containsAttribute(String attributeName) { + return getAttribute(attributeName) != null; + } + + /** + * Derives one attribute. TODO: Support non-default named attributes + */ + private void deriveAttributes(SDField field) { + for (Attribute fieldAttribute : field.getAttributes().values()) { + Attribute attribute = getAttribute(fieldAttribute.getName()); + if (attribute == null) { + attributes.put(fieldAttribute.getName(), fieldAttribute); + attribute = getAttribute(fieldAttribute.getName()); + } + Ranking ranking = field.getRanking(); + if (ranking != null && ranking.isFilter()) { + attribute.setEnableBitVectors(true); + attribute.setEnableOnlyBitVector(true); + } + } + + if (field.containsExpression(ToPositionExpression.class)) { + // TODO: Move this check to processing and remove this + if (hasPosition) { + throw new IllegalArgumentException("Can not specify more than one " + + "set of position attributes per " + "field: " + field.getName()); + } + hasPosition = true; + } + } + + /** + * Returns a read only attribute iterator + */ + public Iterator attributeIterator() { + return attributes().iterator(); + } + + public Collection<Attribute> attributes() { + return Collections.unmodifiableCollection(attributes.values()); + } + + public String toString() { + return "attributes " + getName(); + } + + protected String getDerivedName() { + return "attributes"; + } + + private Map<String, AttributesConfig.Attribute.Builder> toMap(List<AttributesConfig.Attribute.Builder> ls) { + Map<String, AttributesConfig.Attribute.Builder> ret = new LinkedHashMap<>(); + for (AttributesConfig.Attribute.Builder builder : ls) { + ret.put((String) ConfigInstanceUtil.getField(builder, "name"), builder); + } + return ret; + } + + @Override + public void getConfig(AttributesConfig.Builder builder) { + for (Attribute attribute : attributes.values()) { + AttributesConfig.Attribute.Builder aaB = new AttributesConfig.Attribute.Builder() + .name(attribute.getName()) + .datatype(AttributesConfig.Attribute.Datatype.Enum.valueOf(attribute.getType().getExportAttributeTypeName())) + .collectiontype(AttributesConfig.Attribute.Collectiontype.Enum.valueOf(attribute.getCollectionType().getName())); + if (attribute.isRemoveIfZero()) { + aaB.removeifzero(true); + } + if (attribute.isCreateIfNonExistent()) { + aaB.createifnonexistent(true); + } + aaB.enablebitvectors(attribute.isEnabledBitVectors()); + aaB.enableonlybitvector(attribute.isEnabledOnlyBitVector()); + if (attribute.isFastSearch()) { + aaB.fastsearch(true); + } + if (attribute.isFastAccess()) { + aaB.fastaccess(true); + } + if (attribute.isHuge()) { + aaB.huge(true); + } + if (attribute.getSorting().isDescending()) { + aaB.sortascending(false); + } + if (attribute.getSorting().getFunction() != Sorting.Function.UCA) { + aaB.sortfunction(AttributesConfig.Attribute.Sortfunction.Enum.valueOf(attribute.getSorting().getFunction().toString())); + } + if (attribute.getSorting().getStrength() != Sorting.Strength.PRIMARY) { + aaB.sortstrength(AttributesConfig.Attribute.Sortstrength.Enum.valueOf(attribute.getSorting().getStrength().toString())); + } + if (!attribute.getSorting().getLocale().isEmpty()) { + aaB.sortlocale(attribute.getSorting().getLocale()); + } + aaB.arity(attribute.arity()); + aaB.lowerbound(attribute.lowerBound()); + aaB.upperbound(attribute.upperBound()); + aaB.densepostinglistthreshold(attribute.densePostingListThreshold()); + if (attribute.tensorType().isPresent()) { + aaB.tensortype(attribute.tensorType().get().toString()); + } + builder.attribute(aaB); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java new file mode 100644 index 00000000000..643eeb23c87 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java @@ -0,0 +1,136 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.document.Field; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.ConfigInstance.Builder; +import com.yahoo.io.IOUtils; +import com.yahoo.searchdefinition.Index; +import com.yahoo.searchdefinition.Search; +import com.yahoo.text.StringUtilities; +import java.io.IOException; +import java.io.Writer; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.List; + +/** + * Superclass of all derived configurations + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public abstract class Derived implements Exportable { + + private String name; + + public String getName() { return name; } + + protected final void setName(String name) { this.name=name; } + + /** + * Derives the content of this configuration. This + * default calls derive(Document) for each document + * and derive(SDField) for each search definition level field + * AND sets the name of this to the name of the input search definition + */ + protected void derive(Search search) { + setName(search.getName()); + derive(search.getDocument(), search); + for (Index index : search.getExplicitIndices()) { + derive(index, search); + } + for (SDField field : search.allExtraFields() ) { + derive(field,search); + } + } + + + /** + * Derives the content of this configuration. This + * default calls derive(SDField) for each document field + */ + protected void derive(SDDocumentType document,Search search) { + for (Field field : document.fieldSet()) { + SDField sdField = (SDField) field; + if (!sdField.isExtraField()) { + derive(sdField, search); + } + } + } + + /** + * Derives the content of this configuration. This + * default does nothing. + */ + protected void derive(SDField field,Search search) {} + + /** + * Derives the content of this configuration. This + * default does nothing. + */ + protected void derive(Index index, Search search) { + } + + protected abstract String getDerivedName(); + + /** Returns the value of getName if true, the given number as a string otherwise */ + protected String getIndex(int number,boolean labels) { + return labels ? getName() : String.valueOf(number); + } + + /** + * Exports this derived configuration to its .cfg file + * in toDirectory + * + * @param toDirectory the directory to export to, or null + * + */ + public final void export(String toDirectory) + throws IOException { + Writer writer=null; + try { + String fileName=getDerivedName() + ".cfg"; + if (toDirectory!=null) writer=IOUtils.createWriter(toDirectory + "/" + fileName,false); + try { + exportBuilderConfig(writer); + } catch (ClassNotFoundException | InstantiationException + | IllegalAccessException | NoSuchMethodException + | SecurityException | IllegalArgumentException + | InvocationTargetException e) { + throw new RuntimeException(e); + } + } + finally { + if (writer!=null) IOUtils.closeWriter(writer); + } + } + + /** + * Checks what this is a producer of, instantiate that and export to writer + */ + // TODO move to ReflectionUtil, and move that to unexported pkg + private void exportBuilderConfig(Writer writer) throws ClassNotFoundException, InstantiationException, IllegalAccessException, NoSuchMethodException, SecurityException, IllegalArgumentException, InvocationTargetException, IOException { + for (Class<?> intf : getClass().getInterfaces()) { + if (ConfigInstance.Producer.class.isAssignableFrom(intf)) { + Class<?> configClass = intf.getEnclosingClass(); + String builderClassName = configClass.getCanonicalName()+"$Builder"; + Class<?> builderClass = Class.forName(builderClassName); + ConfigInstance.Builder builder = (Builder) builderClass.newInstance(); + Method getConfig = getClass().getMethod("getConfig", builderClass); + getConfig.invoke(this, builder); + ConfigInstance inst = (ConfigInstance) configClass.getConstructor(builderClass).newInstance(builder); + List<String> payloadL = ConfigInstance.serialize(inst); + String payload = StringUtilities.implodeMultiline(payloadL); + writer.write(payload); + } + } + } + + @Override + public String getFileName() { + return getDerivedName() + ".cfg"; + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java new file mode 100644 index 00000000000..973e4e6100c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java @@ -0,0 +1,184 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.DocumenttypesConfig; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.io.IOUtils; +import com.yahoo.protect.Validator; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.derived.validation.Validation; + +import java.io.IOException; +import java.io.Writer; +import java.util.List; + +/** + * A set of all derived configuration of a search definition. Use this as a facade to individual configurations when + * necessary. + * + * @author bratseth + */ +public class DerivedConfiguration { + + private Search search; + private Summaries summaries; + private SummaryMap summaryMap; + private Juniperrc juniperrc; + private AttributeFields attributeFields; + private RankProfileList rankProfileList; + private IndexingScript indexingScript; + private IndexInfo indexInfo; + private VsmFields streamingFields; + private VsmSummary streamingSummary; + private IndexSchema indexSchema; + + /** + * Creates a complete derived configuration from a search definition. + * + * @param search The search to derive a configuration from. Derived objects will be snapshots, but this argument is + * live. Which means that this object will be inconsistent when the given search definition is later + * modified. + * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} + */ + public DerivedConfiguration(Search search, RankProfileRegistry rankProfileRegistry) { + this(search, null, new BaseDeployLogger(), rankProfileRegistry); + } + + /** + * Creates a complete derived configuration snapshot from a search definition. + * + * @param search The search to derive a configuration from. Derived objects will be snapshots, but this + * argument is live. Which means that this object will be inconsistent when the given + * search definition is later modified. + * @param abstractSearchList Search definition this one inherits from, only superclass configuration should be + * generated. Null or empty list if there is none. + * @param deployLogger a {@link DeployLogger} for logging when + * doing operations on this + * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} + */ + public DerivedConfiguration(Search search, List<Search> abstractSearchList, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry) { + Validator.ensureNotNull("Search definition", search); + if (!search.isProcessed()) { + throw new IllegalArgumentException("Search '" + search.getName() + "' not processed."); + } + this.search = search; + if (!search.isDocumentsOnly()) { + streamingFields = new VsmFields(search); + streamingSummary = new VsmSummary(search); + } + if (abstractSearchList != null) { + for (Search abstractSearch : abstractSearchList) { + if (!abstractSearch.isProcessed()) { + throw new IllegalArgumentException("Search '" + search.getName() + "' not processed."); + } + } + } + if (!search.isDocumentsOnly()) { + summaries = new Summaries(search, deployLogger); + summaryMap = new SummaryMap(search, summaries); + juniperrc = new Juniperrc(search); + attributeFields = new AttributeFields(search); + rankProfileList = new RankProfileList(search, attributeFields, rankProfileRegistry); + indexingScript = new IndexingScript(search); + indexInfo = new IndexInfo(search); + indexSchema = new IndexSchema(search); + } + Validation.validate(this, search); + } + + /** + * Exports a complete set of configuration-server format config files. + * + * @param toDirectory the directory to export to, current dir if null + * @throws IOException if exporting fails, some files may still be created + */ + public void export(String toDirectory) throws IOException { + if (!search.isDocumentsOnly()) { + summaries.export(toDirectory); + summaryMap.export(toDirectory); + juniperrc.export(toDirectory); + attributeFields.export(toDirectory); + streamingFields.export(toDirectory); + streamingSummary.export(toDirectory); + indexSchema.export(toDirectory); + rankProfileList.export(toDirectory); + indexingScript.export(toDirectory); + indexInfo.export(toDirectory); + } + } + + public static void exportDocuments(DocumentmanagerConfig.Builder documentManagerCfg, String toDirectory) throws IOException { + exportCfg(new DocumentmanagerConfig(documentManagerCfg), toDirectory + "/" + "documentmanager.cfg"); + } + + public static void exportDocuments(DocumenttypesConfig.Builder documentTypesCfg, String toDirectory) throws IOException { + exportCfg(new DocumenttypesConfig(documentTypesCfg), toDirectory + "/" + "documenttypes.cfg"); + } + + private static void exportCfg(ConfigInstance instance, String fileName) throws IOException { + Writer writer = null; + try { + writer = IOUtils.createWriter(fileName, false); + if (writer != null) { + writer.write(instance.toString()); + writer.write("\n"); + } + } finally { + if (writer != null) { + IOUtils.closeWriter(writer); + } + } + } + + public Summaries getSummaries() { + return summaries; + } + + public AttributeFields getAttributeFields() { + return attributeFields; + } + + public IndexingScript getIndexingScript() { + return indexingScript; + } + + public IndexInfo getIndexInfo() { + return indexInfo; + } + + public void setIndexingScript(IndexingScript script) { + this.indexingScript = script; + } + + public Search getSearch() { + return search; + } + + public RankProfileList getRankProfileList() { + return rankProfileList; + } + + public VsmSummary getVsmSummary() { + return streamingSummary; + } + + public VsmFields getVsmFields() { + return streamingFields; + } + + public IndexSchema getIndexSchema() { + return indexSchema; + } + + public Juniperrc getJuniperrc() { + return juniperrc; + } + + public SummaryMap getSummaryMap() { + return summaryMap; + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java new file mode 100644 index 00000000000..76bf4492788 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java @@ -0,0 +1,82 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; +import com.yahoo.document.DocumenttypesConfig; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.UnprocessingSearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.configmodel.producers.DocumentManager; +import com.yahoo.vespa.configmodel.producers.DocumentTypes; +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +/** + * Auxiliary facade for deriving configs from search definitions + * + * @author bratseth + */ +public class Deriver { + + /** + * Derives only document manager. + * + * + * @param sdFileNames The name of the search definition files to derive from. + * @param toDir The directory to write configuration to. + * @return The list of Search objects, possibly "unproper ones", from sd files containing only document + */ + public static SearchBuilder deriveDocuments(List<String> sdFileNames, String toDir) { + SearchBuilder builder = getUnprocessingSearchBuilder(sdFileNames); + DocumentmanagerConfig.Builder documentManagerCfg = new DocumentManager().produce(builder.getModel(), new DocumentmanagerConfig.Builder()); + try { + DerivedConfiguration.exportDocuments(documentManagerCfg, toDir); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + return builder; + } + + public static SearchBuilder getSearchBuilder(List<String> sds) { + SearchBuilder builder = new SearchBuilder(); + try { + for (String s : sds) { + builder.importFile(s); + } + } catch (ParseException | IOException e) { + throw new IllegalArgumentException(e); + } + builder.build(); + return builder; + } + + public static SearchBuilder getUnprocessingSearchBuilder(List<String> sds) { + SearchBuilder builder = new UnprocessingSearchBuilder(); + try { + for (String s : sds) { + builder.importFile(s); + } + } catch (ParseException | IOException e) { + throw new IllegalArgumentException(e); + } + builder.build(); + return builder; + } + + public static DocumentmanagerConfig.Builder getDocumentManagerConfig(String sd) { + return getDocumentManagerConfig(Collections.singletonList(sd)); + } + + public static DocumentmanagerConfig.Builder getDocumentManagerConfig(List<String> sds) { + return new DocumentManager().produce(getSearchBuilder(sds).getModel(), new DocumentmanagerConfig.Builder()); + } + + public static DocumenttypesConfig.Builder getDocumentTypesConfig(String sd) { + return getDocumentTypesConfig(Collections.singletonList(sd)); + } + + public static DocumenttypesConfig.Builder getDocumentTypesConfig(List<String> sds) { + return new DocumentTypes().produce(getSearchBuilder(sds).getModel(), new DocumenttypesConfig.Builder()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Exportable.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Exportable.java new file mode 100644 index 00000000000..bcc295b91e0 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Exportable.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +/** + * Classes exportable to configurations + * + * @author <a href="mailto:bratseth@overture.com">bratseth</a> + */ +public interface Exportable { + + /** + * Exports the configuration of this object + * + * + * @param toDirectory the directory to export to, does not write to disk if null + * @throws java.io.IOException if exporting fails, some files may still be created + */ + public void export(String toDirectory) throws java.io.IOException; + + /** + * The (short) name of the exported file + * @return a String with the (short) name of the exported file + */ + public String getFileName(); + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java new file mode 100644 index 00000000000..9dfc1bc2af7 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java @@ -0,0 +1,79 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.logging.Logger; + +/** + * The rank settings of a field used for native rank features. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class FieldRankSettings { + + private static final Logger logger = Logger.getLogger(FieldRankSettings.class.getName()); + + private String fieldName; + + private final Map<String, NativeTable> tables = new LinkedHashMap<>(); + + public FieldRankSettings(String fieldName) { + this.fieldName = fieldName; + } + + public void addTable(NativeTable table) { + NativeTable existing = tables.get(table.getType().getName()); + if (existing != null) { + logger.info("Using already specified rank table " + existing + " for field " + fieldName + ", not " + table); + return; + } + tables.put(table.getType().getName(), table); + } + + public static boolean isIndexFieldTable(NativeTable table) { + return isFieldMatchTable(table) || isProximityTable(table); + } + + public static boolean isAttributeFieldTable(NativeTable table) { + return isAttributeMatchTable(table); + } + + private static boolean isFieldMatchTable(NativeTable table) { + return (table.getType().equals(NativeTable.Type.FIRST_OCCURRENCE) || + table.getType().equals(NativeTable.Type.OCCURRENCE_COUNT)); + } + + private static boolean isAttributeMatchTable(NativeTable table) { + return (table.getType().equals(NativeTable.Type.WEIGHT)); + } + + private static boolean isProximityTable(NativeTable table) { + return (table.getType().equals(NativeTable.Type.PROXIMITY) || + table.getType().equals(NativeTable.Type.REVERSE_PROXIMITY)); + } + + public Map<String,String> deriveRankProperties(int part) { + Map<String,String> ret = new LinkedHashMap<>(); + int i = part; + for (Iterator<NativeTable> itr = tables.values().iterator(); itr.hasNext(); ++i) { + NativeTable table = itr.next(); + if (isFieldMatchTable(table)) { + ret.put("nativeFieldMatch." + table.getType().getName() + "." + fieldName + ".part" + i, table.getName()); + } + if (isAttributeMatchTable(table)) { + ret.put("nativeAttributeMatch." + table.getType().getName() + "." + fieldName + ".part" + i, table.getName()); + } + if (isProximityTable(table)) { + ret.put("nativeProximity." + table.getType().getName() + "." + fieldName + ".part" + i, table.getName()); + } + } + return ret; + } + + public String toString() { + return "rank settings of field " + fieldName; + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java new file mode 100644 index 00000000000..d61e57a621e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.vespa.documentmodel.SummaryTransform; + +/** + * The result transformation of a named field + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class FieldResultTransform { + + private String fieldName; + + private SummaryTransform transform; + + private String argument; + + public FieldResultTransform(String fieldName,SummaryTransform transform,String argument) { + this.fieldName=fieldName; + this.transform=transform; + this.argument = argument; + } + + public String getFieldName() { return fieldName; } + + public SummaryTransform getTransform() { return transform; } + + public void setTransform(SummaryTransform transform) { this.transform=transform; } + + /** Returns the argument of this (used as input to the backend docsum rewriter) */ + public String getArgument() { return argument; } + + public int hashCode() { + return fieldName.hashCode() + 11*transform.hashCode() + 17* argument.hashCode(); + } + + public boolean equals(Object o) { + if (! (o instanceof FieldResultTransform)) return false; + FieldResultTransform other=(FieldResultTransform)o; + + return + this.fieldName.equals(other.fieldName) && + this.transform.equals(other.transform) && + this.argument.equals(other.argument); + } + + public String toString() { + String sourceString=""; + if ( ! argument.equals(fieldName)) + sourceString=" (argument: " + argument + ")"; + return "field " + fieldName + ": " + transform + sourceString; + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Index.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Index.java new file mode 100644 index 00000000000..cab4176d696 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Index.java @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.NumericDataType; +import com.yahoo.document.datatypes.*; + +/** + * A type of an index structure + * + * @author bratseth + */ +public class Index { + + /** The index type enumeration */ + public static class Type { + + public static final Type TEXT=new Type("text"); + public static final Type INT64=new Type("long"); + public static final Type BOOLEANTREE=new Type("booleantree"); + + private String name; + + private Type(String name) { + this.name=name; + } + + public int hashCode() { + return name.hashCode(); + } + + public String getName() { return name; } + + public boolean equals(Object other) { + if ( ! (other instanceof Type)) return false; + return this.name.equals(((Type)other).name); + } + + public String toString() { + return "type: " + name; + } + + } + + /** Sets the right index type from a field type */ + public static Type convertType(DataType fieldType) { + FieldValue fval = fieldType.createFieldValue(); + if (fieldType instanceof NumericDataType) { + return Type.INT64; + } else if (fval instanceof StringFieldValue) { + return Type.TEXT; + } else if (fval instanceof Raw) { + return Type.BOOLEANTREE; + } else if (fval instanceof PredicateFieldValue) { + return Type.BOOLEANTREE; + } else if (fieldType instanceof CollectionDataType) { + return convertType(((CollectionDataType) fieldType).getNestedType()); + } else { + throw new IllegalArgumentException("Don't know which index type to " + + "convert " + fieldType + " to"); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java new file mode 100644 index 00000000000..e98ee662b3a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java @@ -0,0 +1,559 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.document.*; +import com.yahoo.searchdefinition.Index; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.*; +import com.yahoo.searchdefinition.processing.ExactMatch; +import com.yahoo.searchdefinition.processing.NGramMatch; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.search.config.IndexInfoConfig; + +import java.util.*; + +/** + * Per-index commands which should be applied to queries prior to searching + * + * @author bratseth + */ +public class IndexInfo extends Derived implements IndexInfoConfig.Producer { + + private static final String CMD_ATTRIBUTE = "attribute"; + private static final String CMD_DEFAULT_POSITION = "default-position"; + private static final String CMD_DYNTEASER = "dynteaser"; + private static final String CMD_FULLURL = "fullurl"; + private static final String CMD_HIGHLIGHT = "highlight"; + private static final String CMD_INDEX = "index"; + private static final String CMD_LOWERCASE = "lowercase"; + private static final String CMD_MATCH_GROUP = "match-group "; + private static final String CMD_NORMALIZE = "normalize"; + private static final String CMD_STEM = "stem"; + private static final String CMD_URLHOST = "urlhost"; + private static final String CMD_WORD = "word"; + private static final String CMD_PLAIN_TOKENS = "plain-tokens"; + private static final String CMD_MULTIVALUE = "multivalue"; + private static final String CMD_FAST_SEARCH = "fast-search"; + private static final String CMD_PREDICATE_BOUNDS = "predicate-bounds"; + private static final String CMD_NUMERICAL = "numerical"; + private Set<IndexCommand> commands = new java.util.LinkedHashSet<>(); + private Map<String, String> aliases = new java.util.LinkedHashMap<>(); + private Map<String, FieldSet> fieldSets; + private Search search; + + public IndexInfo(Search search) { + this.fieldSets = search.fieldSets().userFieldSets(); + addIndexCommand("sddocname", CMD_INDEX); + addIndexCommand("sddocname", CMD_WORD); + derive(search); + } + + protected void derive(Search search) { + super.derive(search); // Derive per field + this.search = search; + // Populate fieldsets with actual field objects, bit late to do that here but + for (FieldSet fs : fieldSets.values()) { + for (String fieldName : fs.getFieldNames()) { + fs.fields().add(search.getField(fieldName)); + } + } + // Must follow, because index settings overrides field settings + for (Index index : search.getExplicitIndices()) { + derive(index, search); + } + + // Commands for summary fields + // TODO: Move to fieldinfo and implement differently. This is not right + for (SummaryField summaryField : search.getUniqueNamedSummaryFields().values()) { + if (summaryField.getTransform().isTeaser()) { + addIndexCommand(summaryField.getName(), CMD_DYNTEASER); + } + if (summaryField.getTransform().isBolded()) { + addIndexCommand(summaryField.getName(), CMD_HIGHLIGHT); + } + } + } + + protected void derive(Index index, Search search) { + if (index.getMatchGroup().size() > 0) { + addIndexCommand(index.getName(), CMD_MATCH_GROUP + toSpaceSeparated(index.getMatchGroup())); + } + } + + private String toSpaceSeparated(Collection c) { + StringBuffer b = new StringBuffer(); + for (Iterator i = c.iterator(); i.hasNext();) { + b.append(i.next()); + if (i.hasNext()) { + b.append(" "); + } + } + return b.toString(); + } + + protected void derive(SDField field, Search search) { + if (field.getDataType().equals(DataType.PREDICATE)) { + Index index = field.getIndex(field.getName()); + if (index != null) { + BooleanIndexDefinition options = index.getBooleanIndexDefiniton(); + if (options.hasLowerBound() || options.hasUpperBound()) { + addIndexCommand(field.getName(), CMD_PREDICATE_BOUNDS + " [" + + (options.hasLowerBound() ? Long.toString(options.getLowerBound()) : "") + ".." + + (options.hasUpperBound() ? Long.toString(options.getUpperBound()) : "") + "]"); + } + } + } + + // Field level aliases + for (Map.Entry<String, String> e : field.getAliasToName().entrySet()) { + String alias = e.getKey(); + String name = e.getValue(); + addIndexAlias(alias, name); + } + if (field.usesStructOrMap()) { + for (SDField structField : field.getStructFields()) { + derive(structField, search); // Recursion + } + } + + if (field.getDataType().equals(PositionDataType.INSTANCE) || + field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))) + { + addIndexCommand(field.getName(), CMD_DEFAULT_POSITION); + } + + addIndexCommand(field, CMD_INDEX); // List the indices + + if (field.doesIndexing() || field.doesLowerCasing()) { + addIndexCommand(field, CMD_LOWERCASE); + } + + if (field.getDataType().isMultivalue()) { + addIndexCommand(field, CMD_MULTIVALUE); + } + + if (field.doesAttributing() && !field.doesIndexing()) { + addIndexCommand(field.getName(), CMD_ATTRIBUTE); + Attribute attribute = field.getAttributes().get(field.getName()); + if (attribute != null && attribute.isFastSearch()) + addIndexCommand(field.getName(), CMD_FAST_SEARCH); + } else if (field.doesIndexing()) { + if (stemSomehow(field, search)) { + addIndexCommand(field, stemCmd(field, search), new StemmingOverrider(this, search)); + } + if (normalizeAccents(field)) { + addIndexCommand(field, CMD_NORMALIZE); + } + } + + if (isUriField(field)) { + addUriIndexCommands(field); + } + + if (field.getDataType() instanceof NumericDataType) { + addIndexCommand(field, CMD_NUMERICAL); + } + + // Explicit commands + for (String command : field.getQueryCommands()) { + addIndexCommand(field, command); + } + + } + + static String stemCmd(SDField field, Search search) { + return CMD_STEM + ":" + field.getStemming(search).toStemMode(); + } + + private boolean stemSomehow(SDField field, Search search) { + if (field.getStemming(search).equals(Stemming.NONE)) return false; + return isTypeOrNested(field, DataType.STRING); + } + + private boolean normalizeAccents(SDField field) { + return field.getNormalizing().doRemoveAccents() && isTypeOrNested(field, DataType.STRING); + } + + private boolean isTypeOrNested(SDField field, DataType type) { + return field.getDataType().equals(type) || field.getDataType().equals(DataType.getArray(type)) || + field.getDataType().equals(DataType.getWeightedSet(type)); + } + + private boolean isUriField(Field field) { + DataType fieldType = field.getDataType(); + if (DataType.URI.equals(fieldType)) { + return true; + } + if (fieldType instanceof CollectionDataType && + DataType.URI.equals(((CollectionDataType)fieldType).getNestedType())) + { + return true; + } + return false; + } + + private void addUriIndexCommands(SDField field) { + String fieldName = field.getName(); + addIndexCommand(fieldName, CMD_FULLURL); + addIndexCommand(fieldName, CMD_LOWERCASE); + addIndexCommand(fieldName + "." + fieldName, CMD_FULLURL); + addIndexCommand(fieldName + "." + fieldName, CMD_LOWERCASE); + addIndexCommand(fieldName + ".path", CMD_FULLURL); + addIndexCommand(fieldName + ".path", CMD_LOWERCASE); + addIndexCommand(fieldName + ".query", CMD_FULLURL); + addIndexCommand(fieldName + ".query", CMD_LOWERCASE); + addIndexCommand(fieldName + ".hostname", CMD_URLHOST); + addIndexCommand(fieldName + ".hostname", CMD_LOWERCASE); + + // XXX hack + Index index = field.getIndex("hostname"); + if (index != null) { + addIndexCommand(index, CMD_URLHOST); + } + } + + /** + * Sets a command for all indices of a field + */ + private void addIndexCommand(Index index, String command) { + addIndexCommand(index.getName(), command); + } + + /** + * Sets a command for all indices of a field + */ + private void addIndexCommand(SDField field, String command) { + addIndexCommand(field, command, null); + } + + /** + * Sets a command for all indices of a field + */ + private void addIndexCommand(SDField field, String command, IndexOverrider overrider) { + if (overrider == null || !overrider.override(field.getName(), command, field)) { + addIndexCommand(field.getName(), command); + } + } + + private void addIndexCommand(String indexName, String command) { + commands.add(new IndexCommand(indexName, command)); + } + + private void addIndexAlias(String alias, String indexName) { + aliases.put(alias, indexName); + } + + /** + * Returns whether a particular command is prsent in this index info + */ + public boolean hasCommand(String indexName, String command) { + return commands.contains(new IndexCommand(indexName, command)); + } + + private boolean notInCommands(String index) { + for (IndexCommand command : commands) { + if (command.getIndex().equals(index)) { + return false; + } + } + return true; + } + + @Override + public void getConfig(IndexInfoConfig.Builder builder) { + IndexInfoConfig.Indexinfo.Builder iiB = new IndexInfoConfig.Indexinfo.Builder(); + iiB.name(getName()); + for (IndexCommand command : commands) { + iiB.command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(command.getIndex()) + .command(command.getCommand())); + } + // Make user defined field sets searchable + for (FieldSet fieldSet : fieldSets.values()) { + if (notInCommands(fieldSet.getName())) { + addFieldSetCommands(iiB, fieldSet); + } + } + + for (Map.Entry<String, String> e : aliases.entrySet()) { + iiB.alias( + new IndexInfoConfig.Indexinfo.Alias.Builder() + .alias(e.getKey()) + .indexname(e.getValue())); + } + builder.indexinfo(iiB); + } + + // TODO: This implementation is completely brain dead. + // Move it to the FieldSetValidity processor (and rename it) as that already has to look at this. + // Also add more explicit testing of this, e.g to indexinfo_fieldsets in ExportingTestCase. - Jon + private void addFieldSetCommands(IndexInfoConfig.Indexinfo.Builder iiB, FieldSet fieldSet) { + // Explicit query commands on the field set, overrides everything. + if (!fieldSet.queryCommands().isEmpty()) { + for (String qc : fieldSet.queryCommands()) { + iiB.command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command(qc)); + } + return; + } + boolean anyIndexing = false; + boolean anyAttributing = false; + boolean anyLowerCasing = false; + boolean anyStemming = false; + boolean anyNormalizing = false; + String stemmingCommand = null; + Matching fieldSetMatching = fieldSet.getMatching(); // null if no explicit matching + // First a pass over the fields to read some params to decide field settings implicitly: + for (SDField field : fieldSet.fields()) { + if (field.doesIndexing()) { + anyIndexing = true; + } + if (field.doesAttributing()) { + anyAttributing = true; + } + if (field.doesIndexing() || field.doesLowerCasing()) { + anyLowerCasing = true; + } + if (stemming(field)) { + anyStemming = true; + stemmingCommand = CMD_STEM + ":" + getEffectiveStemming(field).toStemMode(); + } + if (field.getNormalizing().doRemoveAccents()) { + anyNormalizing = true; + } + if (fieldSetMatching == null && field.getMatching().getType() != Matching.defaultType) + fieldSetMatching = field.getMatching(); + } + if (anyIndexing && anyAttributing && fieldSet.getMatching() == null) { + // We have both attributes and indexes and no explicit match setting -> + // use default matching as that at least works if the data in the attribute consists + // of single tokens only. + fieldSetMatching = new Matching(); + } + if (anyLowerCasing) { + iiB.command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command(CMD_LOWERCASE)); + } + if (hasMultiValueField(fieldSet)) { + iiB.command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command(CMD_MULTIVALUE)); + } + if (anyIndexing) { + iiB.command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command(CMD_INDEX)); + if ( ! isExactMatch(fieldSetMatching)) { + if (anyStemming) { + iiB.command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command(stemmingCommand)); + } + if (anyNormalizing) + iiB.command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command(CMD_NORMALIZE)); + } + } else { + // Assume only attribute fields + iiB + .command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command(CMD_ATTRIBUTE)) + .command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command(CMD_INDEX)); + } + if (fieldSetMatching != null) { + // Explicit matching set on fieldset + if (fieldSetMatching.getType().equals(Matching.Type.EXACT)) { + String term = fieldSetMatching.getExactMatchTerminator(); + if (term==null) term=ExactMatch.DEFAULT_EXACT_TERMINATOR; + iiB.command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command("exact "+term)); + } else if (fieldSetMatching.getType().equals(Matching.Type.WORD)) { + iiB.command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command(CMD_WORD)); + } else if (fieldSetMatching.getType().equals(Matching.Type.GRAM)) { + iiB.command( + new IndexInfoConfig.Indexinfo.Command.Builder() + .indexname(fieldSet.getName()) + .command("ngram "+(fieldSetMatching.getGramSize()>0 ? fieldSetMatching.getGramSize() : NGramMatch.DEFAULT_GRAM_SIZE))); + } else if (fieldSetMatching.getType().equals(Matching.Type.TEXT)) { + + } + + } + + } + + private boolean hasMultiValueField(FieldSet fieldSet) { + for (SDField field : fieldSet.fields()) { + if (field.getDataType().isMultivalue()) + return true; + } + return false; + } + + private Stemming getEffectiveStemming(SDField field) { + Stemming active = field.getStemming(search); + if (field.getIndex(field.getName()) != null) { + if (field.getIndex(field.getName()).getStemming()!=null) { + active = field.getIndex(field.getName()).getStemming(); + } + } + if (active != null) { + return active; + } + // assume default + return Stemming.SHORTEST; + } + + private boolean stemming(SDField field) { + if (field.getStemming() != null) { + return !field.getStemming().equals(Stemming.NONE); + } + if (search.getStemming()==Stemming.NONE) return false; + if (field.getIndex(field.getName())==null) return true; + if (field.getIndex(field.getName()).getStemming()==null) return true; + return !(field.getIndex(field.getName()).getStemming().equals(Stemming.NONE)); + } + + private boolean isExactMatch(Matching m) { + if (m==null) return false; + if (m.getType().equals(Matching.Type.EXACT)) return true; + if (m.getType().equals(Matching.Type.WORD)) return true; + return false; + } + + /** + * Returns a read only iterator over the index commands of this + */ + public Iterator<IndexCommand> indexCommandIterator() { + return Collections.unmodifiableSet(commands).iterator(); + } + + protected String getDerivedName() { + return "index-info"; + } + + /** + * An index command. Null commands are also represented, to detect consistency issues. This is an (immutable) value + * object. + */ + public static class IndexCommand { + + private String index; + + private String command; + + public IndexCommand(String index, String command) { + this.index = index; + this.command = command; + } + + public String getIndex() { + return index; + } + + public String getCommand() { + return command; + } + + /** + * Returns true if this is the null command (do nothing) + */ + public boolean isNull() { + return command.equals(""); + } + + public int hashCode() { + return index.hashCode() + 17 * command.hashCode(); + } + + public boolean equals(Object object) { + if (!(object instanceof IndexCommand)) { + return false; + } + + IndexCommand other = (IndexCommand)object; + return + other.index.equals(this.index) && + other.command.equals(this.command); + } + + public String toString() { + return "index command " + command + " on index " + index; + } + + } + + /** + * A command which may override the command setting of a field for a particular index + */ + private static abstract class IndexOverrider { + + protected IndexInfo owner; + + public IndexOverrider(IndexInfo owner) { + this.owner = owner; + } + + /** + * Override the setting of this index for this field, returns true if overriden, false if this index should be + * set according to the field + */ + public abstract boolean override(String indexName, String command, SDField field); + + } + + private static class StemmingOverrider extends IndexOverrider { + + private Search search; + + public StemmingOverrider(IndexInfo owner, Search search) { + super(owner); + this.search = search; + } + + public boolean override(String indexName, String command, SDField field) { + if (search == null) { + return false; + } + + Index index = search.getIndex(indexName); + if (index == null) { + return false; + } + + Stemming indexStemming = index.getStemming(); + if (indexStemming == null) { + return false; + } + + if (Stemming.NONE.equals(indexStemming)) { + // Add nothing + } else { + owner.addIndexCommand(indexName, CMD_STEM + ":" + indexStemming.toStemMode()); + } + return true; + } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java new file mode 100644 index 00000000000..05b8a1bf5e7 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java @@ -0,0 +1,231 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.document.*; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.BooleanIndexDefinition; +import com.yahoo.searchdefinition.document.FieldSet; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.config.search.IndexschemaConfig; + +import java.util.*; + +/** + * Deriver of indexschema config containing information of all index fields with name and data type. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class IndexSchema extends Derived implements IndexschemaConfig.Producer { + + private final List<IndexField> fields = new ArrayList<>(); + private final Map<String, FieldCollection> collections = new LinkedHashMap<>(); + private final Map<String, FieldSet> fieldSets = new LinkedHashMap<>(); + + public IndexSchema(Search search) { + fieldSets.putAll(search.fieldSets().userFieldSets()); + derive(search); + } + + public boolean containsField(String fieldName) { + return fields.stream().anyMatch(field -> field.getName().equals(fieldName)); + } + + @Override + protected void derive(Search search) { + super.derive(search); + } + + private void deriveIndexFields(SDField field, Search search) { + if (!field.doesIndexing() && + !field.isIndexStructureField()) + { + return; + } + List<Field> lst = flattenField(field); + if (lst.isEmpty()) { + return; + } + String fieldName = field.getName(); + for (Field flatField : lst) { + deriveIndexFields(flatField, search); + } + if (lst.size() > 1) { + FieldSet fieldSet = new FieldSet(fieldName); + for (Field flatField : lst) { + fieldSet.addFieldName(flatField.getName()); + } + fieldSets.put(fieldName, fieldSet); + } + } + + private void deriveIndexFields(Field field, Search search) { + IndexField toAdd = new IndexField(field.getName(), Index.convertType(field.getDataType()), field.getDataType()); + com.yahoo.searchdefinition.Index definedIndex = search.getIndex(field.getName()); + if (definedIndex != null) { + toAdd.setIndexSettings(definedIndex); + } + fields.add(toAdd); + addFieldToCollection(field.getName(), field.getName()); // implicit + } + + private FieldCollection getCollection(String collectionName) { + FieldCollection retval = collections.get(collectionName); + if (retval == null) { + collections.put(collectionName, new FieldCollection(collectionName)); + return collections.get(collectionName); + } + return retval; + } + + private void addFieldToCollection(String fieldName, String collectionName) { + FieldCollection collection = getCollection(collectionName); + collection.fields.add(fieldName); + } + + @Override + protected void derive(SDField field, Search search) { + if (field.usesStructOrMap()) { + return; // unsupported + } + deriveIndexFields(field, search); + } + + @Override + protected String getDerivedName() { + return "indexschema"; + } + + @Override + public void getConfig(IndexschemaConfig.Builder icB) { + for (int i = 0; i < fields.size(); ++i) { + IndexField f = fields.get(i); + IndexschemaConfig.Indexfield.Builder ifB = new IndexschemaConfig.Indexfield.Builder() + .name(f.getName()) + .datatype(IndexschemaConfig.Indexfield.Datatype.Enum.valueOf(f.getType())) + .prefix(f.hasPrefix()) + .phrases(f.hasPhrases()) + .positions(f.hasPositions()); + if (f.getSdType() !=null && !f.getSdType().equals(com.yahoo.searchdefinition.Index.Type.VESPA)) { + ifB.indextype(IndexschemaConfig.Indexfield.Indextype.Enum.valueOf(f.getSdType().toString())); + } + if (!f.getCollectionType().equals("SINGLE")) { + ifB.collectiontype(IndexschemaConfig.Indexfield.Collectiontype.Enum.valueOf(f.getCollectionType())); + } + icB.indexfield(ifB); + } + for (FieldSet fieldSet : fieldSets.values()) { + IndexschemaConfig.Fieldset.Builder fsB = new IndexschemaConfig.Fieldset.Builder() + .name(fieldSet.getName()); + for (String f : fieldSet.getFieldNames()) { + fsB.field(new IndexschemaConfig.Fieldset.Field.Builder() + .name(f)); + } + icB.fieldset(fsB); + } + } + + static List<Field> flattenField(Field field) { + DataType fieldType = field.getDataType(); + if (fieldType.getPrimitiveType() != null){ + return Collections.singletonList(field); + } + if (fieldType instanceof ArrayDataType) { + boolean header = field.isHeader(); + List<Field> ret = new LinkedList<>(); + Field innerField = new Field(field.getName(), ((ArrayDataType)fieldType).getNestedType(), header); + for (Field flatField : flattenField(innerField)) { + ret.add(new Field(flatField.getName(), DataType.getArray(flatField.getDataType()), header)); + } + return ret; + } + if (fieldType instanceof StructuredDataType) { + List<Field> ret = new LinkedList<>(); + String fieldName = field.getName(); + for (Field childField : ((StructuredDataType)fieldType).getFields()) { + for (Field flatField : flattenField(childField)) { + ret.add(new Field(fieldName + "." + flatField.getName(), flatField)); + } + } + return ret; + } + throw new UnsupportedOperationException(fieldType.getName()); + } + + public List<IndexField> getFields() { + return fields; + } + + /** + * Representation of an index field with name and data type. + */ + public static class IndexField { + private String name; + private Index.Type type; + private com.yahoo.searchdefinition.Index.Type sdType; // The index type in "user intent land" + private DataType sdFieldType; + private boolean prefix = false; + private boolean phrases = false; // TODO dead, but keep a while to ensure config compatibility? + private boolean positions = true;// TODO dead, but keep a while to ensure config compatibility? + private BooleanIndexDefinition boolIndex = null; + + public IndexField(String name, Index.Type type, DataType sdFieldType) { + this.name = name; + this.type = type; + this.sdFieldType = sdFieldType; + } + public void setIndexSettings(com.yahoo.searchdefinition.Index index) { + if (type.equals(Index.Type.TEXT)) { + prefix = index.isPrefix(); + } + sdType = index.getType(); + boolIndex = index.getBooleanIndexDefiniton(); + } + public String getName() { return name; } + public Index.Type getRawType() { return type; } + public String getType() { + return type.equals(Index.Type.INT64) + ? "INT64" + : type.equals(Index.Type.BOOLEANTREE) + ? "BOOLEANTREE" + : "STRING"; + } + public String getCollectionType() { + return (sdFieldType == null) + ? "SINGLE" + : (sdFieldType instanceof WeightedSetDataType) + ? "WEIGHTEDSET" + : (sdFieldType instanceof ArrayDataType) + ? "ARRAY" + : "SINGLE"; + } + public boolean hasPrefix() { return prefix; } + public boolean hasPhrases() { return phrases; } + public boolean hasPositions() { return positions; } + + public BooleanIndexDefinition getBooleanIndexDefinition() { + return boolIndex; + } + + /** + * The user set index type + * @return the type + */ + public com.yahoo.searchdefinition.Index.Type getSdType() { + return sdType; + } + } + + /** + * Representation of a collection of fields (aka index, physical view). + */ + @SuppressWarnings({ "UnusedDeclaration" }) + private static class FieldCollection { + + private final String name; + private final List<String> fields = new ArrayList<>(); + + FieldCollection(String name) { + this.name = name; + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java new file mode 100644 index 00000000000..94e4dec567f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java @@ -0,0 +1,146 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.document.*; +import com.yahoo.searchdefinition.*; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; +import com.yahoo.vespa.indexinglanguage.expressions.*; +import com.yahoo.vespa.configdefinition.IlscriptsConfig; +import com.yahoo.vespa.configdefinition.IlscriptsConfig.Ilscript.Builder; + +import java.util.*; + +/** + * An indexing language script derived from a search definition. An indexing script contains a set of indexing + * statements, organized in a composite structure of indexing code snippets. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public final class IndexingScript extends Derived implements IlscriptsConfig.Producer { + + private final List<String> docFields = new LinkedList<>(); + private final List<Expression> expressions = new LinkedList<>(); + + public IndexingScript(Search search) { + derive(search); + } + + protected void derive(SDField field, Search search) { + if (!field.isExtraField()) { + docFields.add(field.getName()); + } + if (field.usesStructOrMap() && + !field.getDataType().equals(PositionDataType.INSTANCE) && + !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))) + { + return; // unsupported + } + ScriptExpression script = field.getIndexingScript(); + if (!script.isEmpty()) { + expressions.add(new StatementExpression(new ClearStateExpression(), + new GuardExpression(script))); + } + } + + public Iterable<Expression> expressions() { + return Collections.unmodifiableCollection(expressions); + } + + @Override + public String getDerivedName() { + return "ilscripts"; + } + + @Override + public void getConfig(IlscriptsConfig.Builder configBuilder) { + IlscriptsConfig.Ilscript.Builder ilscriptBuilder = new IlscriptsConfig.Ilscript.Builder(); + ilscriptBuilder.doctype(getName()); + for (String fieldName : docFields) { + ilscriptBuilder.docfield(fieldName); + } + addContentInOrder(ilscriptBuilder); + configBuilder.ilscript(ilscriptBuilder); + } + + private void addContentInOrder(IlscriptsConfig.Ilscript.Builder ilscriptBuilder) { + ArrayList<Expression> later = new ArrayList<>(); + Set<String> touchedFields = new HashSet<String>(); + for (Expression exp : expressions) { + FieldScanVisitor fieldFetcher = new FieldScanVisitor(); + if (modifiesSelf(exp)) { + later.add(exp); + } else { + ilscriptBuilder.content(exp.toString()); + } + fieldFetcher.visit(exp); + touchedFields.addAll(fieldFetcher.touchedFields()); + } + for (Expression exp : later) { + ilscriptBuilder.content(exp.toString()); + } + generateSyntheticStatementsForUntouchedFields(ilscriptBuilder, touchedFields); + } + + private void generateSyntheticStatementsForUntouchedFields(Builder ilscriptBuilder, Set<String> touchedFields) { + Set<String> fieldsWithSyntheticStatements = new HashSet<String>(docFields); + fieldsWithSyntheticStatements.removeAll(touchedFields); + List<String> orderedFields = new ArrayList<String>(fieldsWithSyntheticStatements); + Collections.sort(orderedFields); + for (String fieldName : orderedFields) { + StatementExpression copyField = new StatementExpression(new InputExpression(fieldName), + new PassthroughExpression(fieldName)); + ilscriptBuilder.content(copyField.toString()); + } + } + + private boolean modifiesSelf(Expression exp) { + MyExpVisitor visitor = new MyExpVisitor(); + visitor.visit(exp); + return visitor.modifiesSelf(); + } + + private class MyExpVisitor extends ExpressionVisitor { + private String inputField = null; + private String outputField = null; + + public boolean modifiesSelf() { return outputField != null && outputField.equals(inputField); } + + @Override + protected void doVisit(Expression expression) { + if (modifiesSelf()) { + return; + } + if (expression instanceof InputExpression) { + inputField = ((InputExpression) expression).getFieldName(); + } + if (expression instanceof OutputExpression) { + outputField = ((OutputExpression) expression).getFieldName(); + } + } + } + + private static class FieldScanVisitor extends ExpressionVisitor { + List<String> touchedFields = new ArrayList<String>(); + List<String> candidates = new ArrayList<String>(); + + @Override + protected void doVisit(Expression exp) { + if (exp instanceof OutputExpression) { + touchedFields.add(((OutputExpression) exp).getFieldName()); + } + if (exp instanceof InputExpression) { + candidates.add(((InputExpression) exp).getFieldName()); + } + if (exp instanceof ZCurveExpression) { + touchedFields.addAll(candidates); + } + } + + Collection<String> touchedFields() { + Collection<String> output = touchedFields; + touchedFields = null; // deny re-use to try and avoid obvious bugs + return output; + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Juniperrc.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Juniperrc.java new file mode 100644 index 00000000000..9ef0ddbc723 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Juniperrc.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.config.search.summary.JuniperrcConfig; + +import java.util.Set; + +/** + * Generated juniperrc-config for controlling juniper. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class Juniperrc extends Derived implements JuniperrcConfig.Producer { + + // List of all fields that should be bolded. + private Set<String> boldingFields = new java.util.LinkedHashSet<>(); + + /** + * Constructs a new juniper rc instance for a given search object. This will derive the configuration automatically, + * so there is no need to call {@link #derive(com.yahoo.searchdefinition.Search)}. + * + * @param search The search model to use for deriving. + */ + public Juniperrc(Search search) { + derive(search); + } + + // Inherit doc from Derived. + @Override + protected void derive(Search search) { + super.derive(search); + for (SummaryField summaryField : search.getUniqueNamedSummaryFields().values()) { + if (summaryField.getTransform() == SummaryTransform.BOLDED) { + boldingFields.add(summaryField.getName()); + } + } + } + + // Inherit doc from Derived. + protected String getDerivedName() { + return "juniperrc"; + } + + @Override + public void getConfig(JuniperrcConfig.Builder builder) { + if (boldingFields.size() != 0) { + builder.prefix(true); + for (String name : boldingFields) { + builder.override(new JuniperrcConfig.Override.Builder() + .fieldname(name) + .length(65536) + .max_matches(1) + .min_length(8192) + .surround_max(65536)); + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinition.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinition.java new file mode 100644 index 00000000000..c1176807519 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinition.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.document.RankType; + +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +/** + * The definition of a rank type used for native rank features. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class NativeRankTypeDefinition { + + /** The type this defines */ + private RankType type; + + /** The rank tables of this rank type */ + private List<NativeTable> rankTables = new java.util.ArrayList<>(); + + public NativeRankTypeDefinition(RankType type) { + this.type = type; + } + + public RankType getType() { + return type; + } + + public void addTable(NativeTable table) { + rankTables.add(table); + } + + /** Returns an unmodifiable list of the tables in this type definition */ + public Iterator<NativeTable> rankSettingIterator() { + return Collections.unmodifiableList(rankTables).iterator(); + } + + public String toString() { + return "native definition of rank type '" + type + "'"; + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionSet.java new file mode 100644 index 00000000000..18856627b70 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionSet.java @@ -0,0 +1,93 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.document.RankType; + +import java.util.Collections; +import java.util.Map; + +/** + * A set of rank type definitions used for native rank features. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class NativeRankTypeDefinitionSet { + + /** The name of this rank definition set */ + private String name; + + /** The unmodifiable rank type implementations in this set */ + private final Map<RankType, NativeRankTypeDefinition> typeDefinitions; + + /** Returns the default rank type (about) */ + public static RankType getDefaultRankType() { return RankType.ABOUT; } + + public NativeRankTypeDefinitionSet(String name) { + this.name = name; + + Map<RankType, NativeRankTypeDefinition> typeDefinitions = new java.util.LinkedHashMap<>(); + typeDefinitions.put(RankType.IDENTITY, createIdentityRankType(RankType.IDENTITY)); + typeDefinitions.put(RankType.ABOUT, createAboutRankType(RankType.ABOUT)); + typeDefinitions.put(RankType.TAGS, createTagsRankType(RankType.TAGS)); + typeDefinitions.put(RankType.EMPTY, createEmptyRankType(RankType.EMPTY)); + this.typeDefinitions = Collections.unmodifiableMap(typeDefinitions); + } + + private NativeRankTypeDefinition createEmptyRankType(RankType type) { + NativeRankTypeDefinition rank = new NativeRankTypeDefinition(type); + rank.addTable(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "linear(0,0)")); + rank.addTable(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "linear(0,0)")); + rank.addTable(new NativeTable(NativeTable.Type.PROXIMITY, "linear(0,0)")); + rank.addTable(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "linear(0,0)")); + rank.addTable(new NativeTable(NativeTable.Type.WEIGHT, "linear(0,0)")); + return rank; + } + + private NativeRankTypeDefinition createAboutRankType(RankType type) { + NativeRankTypeDefinition rank = new NativeRankTypeDefinition(type); + rank.addTable(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "expdecay(8000,12.50)")); + rank.addTable(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "loggrowth(1500,4000,19)")); + rank.addTable(new NativeTable(NativeTable.Type.PROXIMITY, "expdecay(500,3)")); + rank.addTable(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "expdecay(400,3)")); + rank.addTable(new NativeTable(NativeTable.Type.WEIGHT, "linear(1,0)")); + return rank; + } + + private NativeRankTypeDefinition createIdentityRankType(RankType type) { + NativeRankTypeDefinition rank = new NativeRankTypeDefinition(type); + rank.addTable(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "expdecay(100,12.50)")); + rank.addTable(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "loggrowth(1500,4000,19)")); + rank.addTable(new NativeTable(NativeTable.Type.PROXIMITY, "expdecay(5000,3)")); + rank.addTable(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "expdecay(3000,3)")); + rank.addTable(new NativeTable(NativeTable.Type.WEIGHT, "linear(1,0)")); + return rank; + } + + private NativeRankTypeDefinition createTagsRankType(RankType type) { + NativeRankTypeDefinition rank = new NativeRankTypeDefinition(type); + rank.addTable(new NativeTable(NativeTable.Type.FIRST_OCCURRENCE, "expdecay(8000,12.50)")); + rank.addTable(new NativeTable(NativeTable.Type.OCCURRENCE_COUNT, "loggrowth(1500,4000,19)")); + rank.addTable(new NativeTable(NativeTable.Type.PROXIMITY, "expdecay(500,3)")); + rank.addTable(new NativeTable(NativeTable.Type.REVERSE_PROXIMITY, "expdecay(400,3)")); + rank.addTable(new NativeTable(NativeTable.Type.WEIGHT, "loggrowth(38,50,1)")); + return rank; + } + + /** + * Returns a rank type definition if given an existing rank type name, + * or null if given a rank type which has no native implementation (meaning somebody forgot to add it), + */ + public NativeRankTypeDefinition getRankTypeDefinition(RankType type) { + if (type == RankType.DEFAULT) + type = getDefaultRankType(); + return typeDefinitions.get(type); + } + + /** Returns an unmodifiable map of the type definitions in this */ + public Map<RankType, NativeRankTypeDefinition> types() { return typeDefinitions; } + + public String toString() { + return "native rank type definitions " + name; + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeTable.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeTable.java new file mode 100644 index 00000000000..512d4a37647 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeTable.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +/** + * A named rank table of a certain type. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class NativeTable { + + private String name; + + private Type type; + + /** A table type enumeration */ + public static class Type { + + public static Type FIRST_OCCURRENCE = new Type("firstOccurrenceTable"); + public static Type OCCURRENCE_COUNT = new Type("occurrenceCountTable"); + public static Type WEIGHT = new Type("weightTable"); + public static Type PROXIMITY = new Type("proximityTable"); + public static Type REVERSE_PROXIMITY = new Type("reverseProximityTable"); + + private String name; + + private Type(String name) { + this.name = name; + } + + public String getName() { return name; } + + public boolean equals(Object object) { + if (!(object instanceof Type)) { + return false; + } + Type other = (Type)object; + return this.name.equals(other.name); + } + + public int hashCode() { + return name.hashCode(); + } + + public String toString() { + return getName(); + } + } + + public NativeTable(Type type, String name) { + this.type = type; + this.name = name; + } + + public Type getType() { return type; } + + public String getName() { return name; } + + public int hashCode() { + return type.hashCode() + 17*name.hashCode(); + } + + public boolean equals(Object object) { + if (! (object instanceof NativeTable)) return false; + NativeTable other = (NativeTable)object; + return other.getName().equals(this.getName()) && other.getType().equals(this.getType()); + } + + public String toString() { + return getType() + ": " + getName(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java new file mode 100644 index 00000000000..183bfd6ddd4 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchdefinition.Search; +import java.util.Map; + +/** + * The derived rank profiles of a search definition + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class RankProfileList extends Derived implements RankProfilesConfig.Producer { + + private RawRankProfile defaultProfile; + + private Map<String, RawRankProfile> rankProfiles=new java.util.LinkedHashMap<>(); + + /** + * Creates a rank profile + * + * @param search the search definition this is a rank profile from + * @param attributeFields the attribute fields to create a ranking for + */ + public RankProfileList(Search search, AttributeFields attributeFields, RankProfileRegistry rankProfileRegistry) { + setName(search.getName()); + deriveRankProfiles(rankProfileRegistry, search, attributeFields); + } + + private void deriveRankProfiles(RankProfileRegistry rankProfileRegistry, Search search, AttributeFields attributeFields) { + defaultProfile = new RawRankProfile(rankProfileRegistry.getRankProfile(search, "default"), attributeFields); + rankProfiles.put(defaultProfile.getName(), defaultProfile); + + for (RankProfile rank : rankProfileRegistry.localRankProfiles(search)) { + if ("default".equals(rank.getName())) + continue; + RawRankProfile rawRank=new RawRankProfile(rank, attributeFields); + rankProfiles.put(rawRank.getName(), rawRank); + } + } + + public Map<String, RawRankProfile> getRankProfiles() { + return rankProfiles; + } + + /** @return A named raw rank profile, or null if it is not present */ + public RawRankProfile getRankProfile(String name) { + return rankProfiles.get(name); + } + + @Override + public String getDerivedName() { return "rank-profiles"; } + + @Override + public void getConfig(RankProfilesConfig.Builder builder) { + for (RawRankProfile rank : rankProfiles.values() ) { + rank.getConfig(builder); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java new file mode 100644 index 00000000000..cf3b10ccadd --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -0,0 +1,374 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.document.RankType; +import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchlib.rankingexpression.ExpressionFunction; +import com.yahoo.searchlib.rankingexpression.RankingExpression; +import com.yahoo.searchlib.rankingexpression.parser.ParseException; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; +import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import java.util.*; + +/** + * A rank profile derived from a search definition, containing exactly the features available natively in the server + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class RawRankProfile implements RankProfilesConfig.Producer { + + private String name; + + public String getName() { + return name; + } + + public String toString() { + return " rank profile " + name; + } + + @Override + public void getConfig(RankProfilesConfig.Builder builder) { + RankProfilesConfig.Rankprofile.Builder b = new RankProfilesConfig.Rankprofile.Builder().name(getName()); + getRankProperties(b); + builder.rankprofile(b); + } + + private void getRankProperties(RankProfilesConfig.Rankprofile.Builder b) { + RankProfilesConfig.Rankprofile.Fef.Builder fefB = new RankProfilesConfig.Rankprofile.Fef.Builder(); + for (Map.Entry<String, Object> e : configProperties.entrySet()) { + String key = e.getKey().replaceFirst(".part\\d+$", ""); + String val = e.getValue().toString(); + fefB.property(new RankProfilesConfig.Rankprofile.Fef.Property.Builder().name(key).value(val)); + } + b.fef(fefB); + } + + // TODO: These are to expose coupling between the strings used here and elsewhere + public final static String summaryFeatureFefPropertyPrefix = "vespa.summary.feature"; + public final static String rankFeatureFefPropertyPrefix = "vespa.dump.feature"; + + /** + * Returns an immutable view of the config properties this returns + */ + public Map<String, Object> configProperties() { + return Collections.unmodifiableMap(configProperties); + } + + private final Map<String, Object> configProperties; + + /** + * Creates a raw rank profile from the given rank profile + */ + public RawRankProfile(RankProfile rankProfile, AttributeFields attributeFields) { + this.name = rankProfile.getName(); + configProperties = new Deriver(rankProfile, attributeFields).derive(); + } + + private static class Deriver { + + /** + * The field rank settings of this profile + */ + private Map<String, FieldRankSettings> fieldRankSettings = new java.util.LinkedHashMap<>(); + + private RankingExpression firstPhaseRanking = null; + + private RankingExpression secondPhaseRanking = null; + + private Set<ReferenceNode> summaryFeatures = new LinkedHashSet<>(); + + private Set<ReferenceNode> rankFeatures = new LinkedHashSet<>(); + + private List<RankProfile.RankProperty> rankProperties = new ArrayList<>(); + + /** + * Rank properties for weight settings to make these available to feature executors + */ + private List<RankProfile.RankProperty> boostAndWeightRankProperties = new ArrayList<>(); + + private boolean ignoreDefaultRankFeatures = false; + + private RankProfile.MatchPhaseSettings matchPhaseSettings = null; + + private int rerankCount = -1; + private int keepRankCount = -1; + private int numThreadsPerSearch = -1; + private int numSearchPartitions = -1; + private double termwiseLimit = 1.0; + private double rankScoreDropLimit = -Double.MAX_VALUE; + + /** + * The rank type definitions used to derive settings for the native rank features + */ + private NativeRankTypeDefinitionSet nativeRankTypeDefinitions = new NativeRankTypeDefinitionSet("default"); + + private RankProfile rankProfile; + + private Set<String> filterFields = new java.util.LinkedHashSet<>(); + + /** + * Creates a raw rank profile from the given rank profile + */ + public Deriver(RankProfile rankProfile, AttributeFields attributeFields) { + this.rankProfile = rankProfile.compile(); + deriveRankingFeatures(this.rankProfile); + deriveRankTypeSetting(this.rankProfile, attributeFields); + deriveFilterFields(this.rankProfile); + deriveWeightProperties(this.rankProfile); + } + + private void deriveFilterFields(RankProfile rp) { + filterFields.addAll(rp.allFilterFields()); + } + + public void deriveRankingFeatures(RankProfile rankProfile) { + firstPhaseRanking = rankProfile.getFirstPhaseRanking(); + secondPhaseRanking = rankProfile.getSecondPhaseRanking(); + summaryFeatures = new LinkedHashSet<>(rankProfile.getSummaryFeatures()); + rankFeatures = rankProfile.getRankFeatures(); + rerankCount = rankProfile.getRerankCount(); + matchPhaseSettings = rankProfile.getMatchPhaseSettings(); + numThreadsPerSearch = rankProfile.getNumThreadsPerSearch(); + numSearchPartitions = rankProfile.getNumSearchPartitions(); + termwiseLimit = rankProfile.getTermwiseLimit(); + keepRankCount = rankProfile.getKeepRankCount(); + rankScoreDropLimit = rankProfile.getRankScoreDropLimit(); + ignoreDefaultRankFeatures = rankProfile.getIgnoreDefaultRankFeatures(); + rankProperties = new ArrayList<>(rankProfile.getRankProperties()); + derivePropertiesAndSummaryFeaturesFromMacros(rankProfile.getMacros()); + } + + private void derivePropertiesAndSummaryFeaturesFromMacros(Map<String, RankProfile.Macro> macros) { + if (macros.isEmpty()) return; + Map<String, ExpressionFunction> expressionMacros = new LinkedHashMap<>(); + for (Map.Entry<String, RankProfile.Macro> macro : macros.entrySet()) { + expressionMacros.put(macro.getKey(), macro.getValue().toExpressionMacro()); + } + + Map<String, String> macroProperties = new LinkedHashMap<>(); + macroProperties.putAll(deriveMacroProperties(expressionMacros)); + if (firstPhaseRanking != null) { + macroProperties.putAll(firstPhaseRanking.getRankProperties(new ArrayList<>(expressionMacros.values()))); + } + if (secondPhaseRanking != null) { + macroProperties.putAll(secondPhaseRanking.getRankProperties(new ArrayList<>(expressionMacros.values()))); + } + for (Map.Entry<String, String> e : macroProperties.entrySet()) { + rankProperties.add(new RankProfile.RankProperty(e.getKey(), e.getValue())); + } + SerializationContext context = new SerializationContext(expressionMacros.values(), null, macroProperties); + replaceMacroSummaryFeatures(context); + } + + private Map<String, String> deriveMacroProperties(Map<String, ExpressionFunction> eMacros) { + SerializationContext context = new SerializationContext(eMacros); + for (Map.Entry<String, ExpressionFunction> e : eMacros.entrySet()) { + String script = e.getValue().getBody().getRoot().toString(context, null, null); + context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), script); + } + return context.serializedFunctions(); + } + + private void replaceMacroSummaryFeatures(SerializationContext context) { + if (summaryFeatures == null) return; + Map<String, ReferenceNode> macroSummaryFeatures = new LinkedHashMap<>(); + for (Iterator<ReferenceNode> i = summaryFeatures.iterator(); i.hasNext(); ) { + ReferenceNode referenceNode = i.next(); + // Is the feature a macro? + if (context.getFunction(referenceNode.getName()) != null) { + context.addFunctionSerialization(RankingExpression.propertyName(referenceNode.getName()), + referenceNode.toString(context, null, null)); + ReferenceNode newReferenceNode = new ReferenceNode("rankingExpression(" + referenceNode.getName() + ")", referenceNode.getArguments().expressions(), referenceNode.getOutput()); + macroSummaryFeatures.put(referenceNode.getName(), newReferenceNode); + i.remove(); // Will add the expanded one in next block + } + } + // Then, replace the summary features that were macros + for (Map.Entry<String, ReferenceNode> e : macroSummaryFeatures.entrySet()) { + summaryFeatures.add(e.getValue()); + } + } + + private void deriveWeightProperties(RankProfile rankProfile) { + + for (RankProfile.RankSetting setting : rankProfile.rankSettings()) { + if (!setting.getType().equals(RankProfile.RankSetting.Type.WEIGHT)) { + continue; + } + boostAndWeightRankProperties.add(new RankProfile.RankProperty("vespa.fieldweight." + setting.getFieldName(), + String.valueOf(setting.getIntValue()))); + } + } + + /** + * Adds the type boosts from a rank profile + */ + private void deriveRankTypeSetting(RankProfile rankProfile, AttributeFields attributeFields) { + for (Iterator<RankProfile.RankSetting> i = rankProfile.rankSettingIterator(); i.hasNext(); ) { + RankProfile.RankSetting setting = i.next(); + if (!setting.getType().equals(RankProfile.RankSetting.Type.RANKTYPE)) continue; + + deriveNativeRankTypeSetting(setting.getFieldName(), (RankType) setting.getValue(), attributeFields, + hasDefaultRankTypeSetting(rankProfile, setting.getFieldName())); + } + } + + public void deriveNativeRankTypeSetting(String fieldName, RankType rankType, AttributeFields attributeFields, boolean isDefaultSetting) { + if (isDefaultSetting) return; + + NativeRankTypeDefinition definition = nativeRankTypeDefinitions.getRankTypeDefinition(rankType); + if (definition == null) throw new IllegalArgumentException("In field '" + fieldName + "': " + + rankType + " is known but has no implementation. " + + "Supported rank types: " + + nativeRankTypeDefinitions.types().keySet()); + + FieldRankSettings settings = deriveFieldRankSettings(fieldName); + for (Iterator<NativeTable> i = definition.rankSettingIterator(); i.hasNext(); ) { + NativeTable table = i.next(); + // only add index field tables if we are processing an index field and + // only add attribute field tables if we are processing an attribute field + if ((FieldRankSettings.isIndexFieldTable(table) && attributeFields.getAttribute(fieldName) == null) || + (FieldRankSettings.isAttributeFieldTable(table) && attributeFields.getAttribute(fieldName) != null)) { + settings.addTable(table); + } + } + } + + private boolean hasDefaultRankTypeSetting(RankProfile rankProfile, String fieldName) { + RankProfile.RankSetting setting = + rankProfile.getRankSetting(fieldName, RankProfile.RankSetting.Type.RANKTYPE); + return setting != null && setting.getValue().equals(RankType.DEFAULT); + } + + public FieldRankSettings deriveFieldRankSettings(String fieldName) { + FieldRankSettings settings = fieldRankSettings.get(fieldName); + if (settings == null) { + settings = new FieldRankSettings(fieldName); + fieldRankSettings.put(fieldName, settings); + } + return settings; + } + + /** + * Derives the properties this produces. Equal keys are suffixed with .part0 etc, remove when exporting to file + * + * @return map of the derived properties + */ + public Map<String, Object> derive() { + Map<String, Object> props = new LinkedHashMap<>(); + int i = 0; + for (RankProfile.RankProperty property : rankProperties) { + if ("rankingExpression(firstphase).rankingScript".equals(property.getName())) { + // Could have been set by macro expansion. Set expressions, then skip this property. + try { + firstPhaseRanking = new RankingExpression(property.getValue()); + } catch (ParseException e) { + throw new IllegalArgumentException("Could not parse second phase expression", e); + } + continue; + } + if ("rankingExpression(secondphase).rankingScript".equals(property.getName())) { + try { + secondPhaseRanking = new RankingExpression(property.getValue()); + } catch (ParseException e) { + throw new IllegalArgumentException("Could not parse second phase expression", e); + } + continue; + } + props.put(property.getName() + ".part" + i, property.getValue()); + i++; + } + props.putAll(deriveRankingPhaseRankProperties(firstPhaseRanking, "firstphase")); + props.putAll(deriveRankingPhaseRankProperties(secondPhaseRanking, "secondphase")); + for (FieldRankSettings settings : fieldRankSettings.values()) { + props.putAll(settings.deriveRankProperties(i)); + } + i = 0; + for (RankProfile.RankProperty property : boostAndWeightRankProperties) { + props.put(property.getName() + ".part" + i, property.getValue()); + i++; + } + i = 0; + for (ReferenceNode feature : summaryFeatures) { + props.put(summaryFeatureFefPropertyPrefix + ".part" + i, feature.toString()); + i++; + } + i = 0; + for (ReferenceNode feature : rankFeatures) { + props.put(rankFeatureFefPropertyPrefix + ".part" + i, feature.toString()); + i++; + } + if (numThreadsPerSearch > 0) { + props.put("vespa.matching.numthreadspersearch", numThreadsPerSearch + ""); + } + if (numSearchPartitions >= 0) { + props.put("vespa.matching.numsearchpartitions", numSearchPartitions + ""); + } + if (termwiseLimit < 1.0) { + props.put("vespa.matching.termwise_limit", termwiseLimit + ""); + } + if (matchPhaseSettings != null) { + props.put("vespa.matchphase.degradation.attribute", matchPhaseSettings.getAttribute()); + props.put("vespa.matchphase.degradation.ascendingorder", matchPhaseSettings.getAscending() + ""); + props.put("vespa.matchphase.degradation.maxhits", matchPhaseSettings.getMaxHits() + ""); + props.put("vespa.matchphase.degradation.maxfiltercoverage", matchPhaseSettings.getMaxFilterCoverage() + ""); + props.put("vespa.matchphase.degradation.samplepercentage", matchPhaseSettings.getEvaluationPoint() + ""); + props.put("vespa.matchphase.degradation.postfiltermultiplier", matchPhaseSettings.getPrePostFilterTippingPoint() + ""); + RankProfile.DiversitySettings diversitySettings = rankProfile.getMatchPhaseSettings().getDiversity(); + if (diversitySettings != null) { + props.put("vespa.matchphase.diversity.attribute", diversitySettings.getAttribute()); + props.put("vespa.matchphase.diversity.mingroups", diversitySettings.getMinGroups()); + props.put("vespa.matchphase.diversity.cutoff.factor", diversitySettings.getCutoffFactor()); + props.put("vespa.matchphase.diversity.cutoff.strategy", diversitySettings.getCutoffStrategy()); + } + } + if (rerankCount > -1) { + props.put("vespa.hitcollector.heapsize", rerankCount + ""); + } + if (keepRankCount > -1) { + props.put("vespa.hitcollector.arraysize", keepRankCount + ""); + } + if (rankScoreDropLimit > -Double.MAX_VALUE) { + props.put("vespa.hitcollector.rankscoredroplimit", rankScoreDropLimit + ""); + } + if (ignoreDefaultRankFeatures) { + props.put("vespa.dump.ignoredefaultfeatures", true); + } + Iterator filterFieldsIterator = filterFields.iterator(); + while (filterFieldsIterator.hasNext()) { + String fieldName = (String) filterFieldsIterator.next(); + props.put("vespa.isfilterfield." + fieldName + ".part42", true); + } + for (Map.Entry<String, String> attributeType : rankProfile.getAttributeTypes().entrySet()) { + props.put("vespa.type.attribute." + attributeType.getKey(), attributeType.getValue()); + } + for (Map.Entry<String, String> queryFeatureType : rankProfile.getQueryFeatureTypes().entrySet()) { + props.put("vespa.type.query." + queryFeatureType.getKey(), queryFeatureType.getValue()); + } + if (props.size() >= 1000000) throw new RuntimeException("Too many rank properties"); + return props; + } + + private Map<String, String> deriveRankingPhaseRankProperties(RankingExpression expression, String phase) { + Map<String, String> ret = new LinkedHashMap<>(); + if (expression == null) { + return ret; + } + String name = expression.getName(); + if ("".equals(name)) { + name = phase; + } + if (expression.getRoot() instanceof ReferenceNode) { + ret.put("vespa.rank." + phase, expression.getRoot().toString()); + } else { + ret.put("vespa.rank." + phase, "rankingExpression(" + name + ")"); + ret.put("rankingExpression(" + name + ").rankingScript", expression.getRoot().toString()); + } + return ret; + } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SearchOrderer.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SearchOrderer.java new file mode 100644 index 00000000000..69133e31fb3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SearchOrderer.java @@ -0,0 +1,105 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.document.DataTypeName; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.Search; + +import java.util.*; + +/** + * <p>A class which can reorder a list of search definitions such that any supertype + * always preceed any subtype. Subject to this condition the given order + * is preserved (the minimal reordering is done).</p> + * + * <p>This class is <b>not</b> multithread safe. Only one ordering must be done + * at the time in any instance.</p> + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class SearchOrderer { + + /** A map from DataTypeName to the Search defining them */ + private Map<DataTypeName, Search> documentNameToSearch=new java.util.HashMap<>(); + + /** + * Reorders the given list of search definitions such that any supertype + * always preceed any subtype. Subject to this condition the given order + * is preserved (the minimal reordering is done). + * + * @return a new list containing the same search instances in the right order + */ + public List<Search> order(List<Search> unordered) { + Collections.sort(unordered, new Comparator<Search>() { + @Override + public int compare(Search lhs, Search rhs) { + return lhs.getName().compareTo(rhs.getName()); + } + }); + + // No, this is not a fast algorithm... + indexOnDocumentName(unordered); + List<Search> ordered=new java.util.ArrayList<>(unordered.size()); + List<Search> moveOutwards=new java.util.ArrayList<>(); + for (Search search: unordered) { + if (containsInherited(ordered,search)) { + addOrdered(ordered,search,moveOutwards); + } + else { + moveOutwards.add(search); + } + } + + // Any leftovers means we have search definitions with undefined inheritants. + // This is warned about elsewhere. + ordered.addAll(moveOutwards); + + documentNameToSearch.clear(); + return ordered; + } + + private void addOrdered(List<Search> ordered,Search search,List<Search> moveOutwards) { + ordered.add(search); + Search eligibleMove; + do { + eligibleMove=removeFirstEligibleMoveOutwards(moveOutwards,ordered); + if (eligibleMove!=null) + ordered.add(eligibleMove); + } while (eligibleMove!=null); + } + + /** Removes and returns the first search from the move list which can now be added, or null if none */ + private Search removeFirstEligibleMoveOutwards(List<Search> moveOutwards,List<Search> ordered) { + for (Search move : moveOutwards) { + if (containsInherited(ordered,move)) { + moveOutwards.remove(move); + return move; + } + } + return null; + } + + private boolean containsInherited(List<Search> list,Search search) { + if (search.getDocument() == null) { + return true; + } + for (SDDocumentType sdoc : search.getDocument().getInheritedTypes() ) { + DataTypeName inheritedName=sdoc.getDocumentName(); + if ("document".equals(inheritedName.getName())) continue; + Search inheritedSearch=documentNameToSearch.get(inheritedName); + if (!list.contains(inheritedSearch)) + return false; + } + return true; + } + + private void indexOnDocumentName(List<Search> searches) { + documentNameToSearch.clear(); + for (Search search : searches) { + if (search.getDocument() != null) { + documentNameToSearch.put(search.getDocument().getDocumentName(),search); + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java new file mode 100644 index 00000000000..357e0d40f49 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.config.search.SummaryConfig; +import java.util.List; + +/** + * A list of derived summaries + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class Summaries extends Derived implements SummaryConfig.Producer { + + private List<SummaryClass> summaries=new java.util.ArrayList<>(1); + + public Summaries(Search search, DeployLogger deployLogger) { + // Make sure the default is first + summaries.add(new SummaryClass(search,search.getSummary("default"), deployLogger)); + for (DocumentSummary summary : search.getSummaries().values()) { + if (!summary.getName().equals("default")) + summaries.add(new SummaryClass(search,summary, deployLogger)); + } + } + + protected String getDerivedName() { return "summary"; } + + @Override + public void getConfig(SummaryConfig.Builder builder) { + builder.defaultsummaryid(summaries.isEmpty() ? -1 : summaries.get(0).hashCode()); + for (SummaryClass summaryClass : summaries) { + builder.classes(summaryClass.getSummaryClassConfig()); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java new file mode 100644 index 00000000000..d21523caea2 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java @@ -0,0 +1,144 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.prelude.fastsearch.DocsumDefinitionSet; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.config.search.SummaryConfig; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import java.util.Iterator; +import java.util.Map; +import java.util.Random; +import java.util.logging.Level; + +/** + * A summary derived from a search definition. + * Each summary definition have at least one summary, the default + * which has the same name as the search definition. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class SummaryClass extends Derived { + + /** True if this summary class needs to access summary information on disk */ + private boolean accessingDiskSummary=false; + + /** The summary fields of this indexed by name */ + private Map<String,SummaryClassField> fields = new java.util.LinkedHashMap<>(); + + private DeployLogger deployLogger = new BaseDeployLogger(); + + private final Random random = new Random(7); + + /** + * Creates a summary class from a search definition summary + * + * @param deployLogger a {@link DeployLogger} + */ + public SummaryClass(Search search, DocumentSummary summary, DeployLogger deployLogger) { + this.deployLogger = deployLogger; + deriveName(summary); + deriveFields(search,summary); + deriveImplicitFields(summary); + } + + private void deriveName(DocumentSummary summary) { + setName(summary.getName()); + } + + /** MUST be called after all other fields are added */ + private void deriveImplicitFields(DocumentSummary summary) { + if (summary.getName().equals("default")) { + addField("documentid", DataType.STRING); + } + } + + private void deriveFields(Search search, DocumentSummary summary) { + for (SummaryField summaryField : summary.getSummaryFields()) { + if (!accessingDiskSummary && search.isAccessingDiskSummary(summaryField)) { + accessingDiskSummary = true; + } + addField(summaryField.getName(), summaryField.getDataType(), summaryField.getTransform()); + } + } + + private void addField(String name, DataType type) { + addField(name, type, null); + } + + private void addField(String name, DataType type, SummaryTransform transform) { + if (fields.containsKey(name)) { + SummaryClassField sf = fields.get(name); + if (!SummaryClassField.convertDataType(type, transform).equals(sf.getType())) { + deployLogger.log(Level.WARNING, "Conflicting definition of field " + name + ". " + + "Declared as type " + sf.getType() + " and " + + type); + } + } else { + fields.put(name, new SummaryClassField(name, type, transform)); + } + } + + + /** Returns an iterator of the fields of this summary. Removes on this iterator removes the field from this summary */ + public Iterator<SummaryClassField> fieldIterator() { + return fields.values().iterator(); + } + + public void addField(SummaryClassField field) { + fields.put(field.getName(),field); + } + + /** Returns the writable map of fields of this summary */ // TODO: Make read only, move writers to iterator/addField + public Map<String,SummaryClassField> getFields() { return fields; } + + public SummaryClassField getField(String name) { + return fields.get(name); + } + + public int getFieldCount() { return fields.size(); } + + public int hashCode() { + int number = 1; + int hash = getName().hashCode(); + for (Iterator i = fieldIterator(); i.hasNext(); ) { + SummaryClassField field = (SummaryClassField)i.next(); + hash += number * (field.getName().hashCode() + + 17*field.getType().getName().hashCode()); + number++; + } + if (hash < 0) + hash *= -1; + return hash; + } + + public SummaryConfig.Classes.Builder getSummaryClassConfig() { + SummaryConfig.Classes.Builder classBuilder = new SummaryConfig.Classes.Builder(); + int id = hashCode(); + if (id == DocsumDefinitionSet.SLIME_MAGIC_ID) { + deployLogger.log(Level.WARNING, "Summary class '" + getName() + "' hashes to the SLIME_MAGIC_ID '" + id + + "'. This is unlikely but I autofix it for you by adding a random number."); + id += random.nextInt(); + } + classBuilder. + id(id). + name(getName()); + for (SummaryClassField field : fields.values() ) { + classBuilder.fields(new SummaryConfig.Classes.Fields.Builder(). + name(field.getName()). + type(field.getType().getName())); + } + return classBuilder; + } + + protected String getDerivedName() { return "summary"; } + + public String toString() { + return "summary class " + getName(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java new file mode 100644 index 00000000000..bb1dd87f314 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java @@ -0,0 +1,110 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.MapDataType; +import com.yahoo.document.datatypes.*; +import com.yahoo.vespa.documentmodel.SummaryTransform; + +/** + * A summary field derived from a search definition + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class SummaryClassField { + + private final String name; + + private final Type type; + + /** The summary field type enumeration */ + public enum Type { + + BYTE("byte"), + SHORT("short"), + INTEGER("integer"), + INT64("int64"), + FLOAT("float"), + DOUBLE("double"), + STRING("string"), + DATA("data"), + LONGSTRING("longstring"), + LONGDATA("longdata"), + XMLSTRING("xmlstring"), + FEATUREDATA("featuredata"), + JSONSTRING("jsonstring"); + + private String name; + + private Type(String name) { + this.name = name; + } + + /** Returns the name of this type */ + public String getName() { + return name; + } + + public String toString() { + return "type: " + name; + } + } + + public SummaryClassField(String name, DataType type, SummaryTransform transform) { + this.name = name; + this.type = convertDataType(type, transform); + } + + public String getName() { return name; } + + public Type getType() { return type; } + + /** Converts to the right summary field type from a field datatype and a transform*/ + public static Type convertDataType(DataType fieldType, SummaryTransform transform) { + FieldValue fval = fieldType.createFieldValue(); + if (fval instanceof StringFieldValue) { + if (transform != null && transform.equals(SummaryTransform.RANKFEATURES)) { + return Type.FEATUREDATA; + } else if (transform != null && transform.equals(SummaryTransform.SUMMARYFEATURES)) { + return Type.FEATUREDATA; + } else { + return Type.LONGSTRING; + } + } else if (fval instanceof IntegerFieldValue) { + return Type.INTEGER; + } else if (fval instanceof LongFieldValue) { + return Type.INT64; + } else if (fval instanceof FloatFieldValue) { + return Type.FLOAT; + } else if (fval instanceof DoubleFieldValue) { + return Type.DOUBLE; + } else if (fval instanceof ByteFieldValue) { + return Type.BYTE; + } else if (fval instanceof Raw) { + return Type.DATA; + } else if (fval instanceof Struct) { + return Type.JSONSTRING; + } else if (fval instanceof PredicateFieldValue) { + return Type.STRING; + } else if (fval instanceof TensorFieldValue) { + return Type.JSONSTRING; + } else if (fieldType instanceof CollectionDataType) { + if (transform != null && transform.equals(SummaryTransform.POSITIONS)) { + return Type.XMLSTRING; + } else { + return Type.JSONSTRING; + } + } else if (fieldType instanceof MapDataType) { + return Type.JSONSTRING; + } else { + throw new IllegalArgumentException("Don't know which summary type to " + + "convert " + fieldType + " to"); + } + } + + public String toString() { + return "summary class field " + name; + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java new file mode 100644 index 00000000000..d59c671e6a5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java @@ -0,0 +1,111 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.config.search.SummarymapConfig; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; + +/** + * A summary map (describing search-time summary field transformations) + * derived from a search definition + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class SummaryMap extends Derived implements SummarymapConfig.Producer { + + private Map<String,FieldResultTransform> resultTransforms = new java.util.LinkedHashMap<>(); + + /** Crateate a summary map from a search definition */ + public SummaryMap(Search search, Summaries summaries) { + derive(search, summaries); + } + + protected void derive(Search search, Summaries summaries) { + // TODO: This should really derive from the 'summaries' argument. Bug? + for (DocumentSummary documentSummary : search.getSummaries().values()) { + derive(documentSummary); + } + super.derive(search); + } + + protected void derive(SDField field, Search search) { + } + + private void derive(DocumentSummary documentSummary) { + for (SummaryField summaryField : documentSummary.getSummaryFields()) { + if (summaryField.getTransform()== SummaryTransform.NONE) continue; + + if (summaryField.getTransform()==SummaryTransform.ATTRIBUTE || + summaryField.getTransform()==SummaryTransform.DISTANCE || + summaryField.getTransform()==SummaryTransform.GEOPOS || + summaryField.getTransform()==SummaryTransform.POSITIONS) { + resultTransforms.put(summaryField.getName(),new FieldResultTransform(summaryField.getName(), + summaryField.getTransform(), + summaryField.getSingleSource())); + } else { + // Note: Currently source mapping is handled in the indexing statement, + // by creating a summary field for each of the values + // This works, but is suboptimal. We could consolidate to a minimal set and + // use the right value from the minimal set as the third parameter here, + // and add "override" commands to multiple static values + resultTransforms.put(summaryField.getName(),new FieldResultTransform(summaryField.getName(), + summaryField.getTransform(), + summaryField.getName())); + } + } + } + + /** Returns a read-only iterator of the FieldResultTransforms of this summary map */ + public Iterator resultTransformIterator() { + return Collections.unmodifiableCollection(resultTransforms.values()).iterator(); + } + + protected String getDerivedName() { return "summarymap"; } + + /** Returns the command name of a transform */ + private String getCommand(SummaryTransform transform) { + if (transform.equals(SummaryTransform.DISTANCE)) + return "absdist"; + else if (transform.isDynamic()) + return "dynamicteaser"; + else + return transform.getName(); + } + + /** + * Does this summary command name stand for a dynamic transform? + * We need this because some model information is shared through configs instead of model - see usage + */ + public static boolean isDynamicCommand(String commandName) { + return (commandName.equals("dynamicteaser") || commandName.equals("smartsummary")); + } + + @Override + public void getConfig(SummarymapConfig.Builder builder) { + builder.defaultoutputclass(-1); + for (FieldResultTransform frt : resultTransforms.values()) { + SummarymapConfig.Override.Builder oB = new SummarymapConfig.Override.Builder() + .field(frt.getFieldName()) + .command(getCommand(frt.getTransform())); + if (frt.getTransform().isDynamic() || + frt.getTransform().equals(SummaryTransform.ATTRIBUTE) || + frt.getTransform().equals(SummaryTransform.DISTANCE) || + frt.getTransform().equals(SummaryTransform.GEOPOS) || + frt.getTransform().equals(SummaryTransform.POSITIONS) || + frt.getTransform().equals(SummaryTransform.TEXTEXTRACTOR)) + { + oB.arguments(frt.getArgument()); + } else { + oB.arguments(""); + } + builder.override(oB); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java new file mode 100644 index 00000000000..65698675884 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java @@ -0,0 +1,275 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.document.*; +import com.yahoo.document.datatypes.*; +import com.yahoo.searchdefinition.FieldSets; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.FieldSet; +import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Vertical streaming matcher field specification + */ +public class VsmFields extends Derived implements VsmfieldsConfig.Producer { + + private final Map<String, StreamingField> fields=new LinkedHashMap<>(); + private final Map<String, StreamingDocumentType> doctypes=new LinkedHashMap<>(); + + public VsmFields(Search search) { + addSearchdefinition(search); + } + + private void addSearchdefinition(Search search) { + derive(search); + } + + protected void derive(SDDocumentType document,Search search) { + super.derive(document, search); + StreamingDocumentType docType=getDocumentType(document.getName()); + if (docType == null) { + docType = new StreamingDocumentType(document.getName(), search.fieldSets()); + doctypes.put(document.getName(), docType); + } + for (Object o : document.fieldSet()) { + derive(docType, (SDField) o); + } + } + + protected void derive(StreamingDocumentType document, SDField field) { + if (field.usesStructOrMap()) { + for (SDField structField : field.getStructFields()) { + derive(document, structField); // Recursion + } + } else { + + if (! (field.doesIndexing() || field.doesSummarying() || field.doesAttributing()) ) + return; + + StreamingField streamingField=new StreamingField(field); + addField(streamingField.getName(),streamingField); + deriveIndices(document, field, streamingField); + } + } + + private void deriveIndices(StreamingDocumentType document, SDField field, StreamingField streamingField) { + if (field.doesIndexing()) { + addFieldToIndices(document, field.getName(), streamingField); + } else if (field.doesAttributing()) { + for (String indexName : field.getAttributes().keySet()) { + addFieldToIndices(document, indexName, streamingField); + } + } + } + + private void addFieldToIndices(StreamingDocumentType document, String indexName, StreamingField streamingField) { + if (indexName.contains(".")) { + addFieldToIndices(document, indexName.substring(0,indexName.lastIndexOf(".")), streamingField); // Recursion + } + document.addIndexField(indexName, streamingField.getName()); + } + + private void addField(String name, StreamingField field) { + fields.put(name, field); + } + + /** Returns a streaming index, or null if there is none with this name */ + public StreamingDocumentType getDocumentType(String name) { + return doctypes.get(name); + } + + public String getDerivedName() { + return "vsmfields"; + } + + @Override + public void getConfig(VsmfieldsConfig.Builder vsB) { + for (StreamingField streamingField : fields.values()) { + vsB.fieldspec(streamingField.getFieldSpecConfig()); + } + for (StreamingDocumentType streamingDocType : doctypes.values()) { + vsB.documenttype(streamingDocType.getDocTypeConfig()); + } + } + + private static class StreamingField { + + private final String name; + + /** Whether this field does prefix matching by default */ + private final Matching matching; + + /** The type of this field */ + private final Type type; + + private final boolean isAttribute; + + /** The streaming field type enumeration */ + public static class Type { + + public static Type INT8=new Type("int8","INT8"); + public static Type INT16=new Type("int16","INT16"); + public static Type INT32=new Type("int32","INT32"); + public static Type INT64=new Type("int64","INT64"); + public static Type FLOAT=new Type("float","FLOAT"); + public static Type DOUBLE=new Type("double","DOUBLE"); + public static Type STRING=new Type("string","AUTOUTF8"); + public static Type UNSEARCHABLESTRING=new Type("string","NONE"); + + private String name; + + private String searchMethod; + + private Type(String name,String searchMethod) { + this.name=name; + this.searchMethod=searchMethod; + } + + public int hashCode() { + return name.hashCode(); + } + + /** Returns the name of this type */ + public String getName() { return name; } + + public String getSearchMethod() { return searchMethod; } + + public boolean equals(Object other) { + if ( ! (other instanceof Type)) return false; + return this.name.equals(((Type)other).name); + } + + public String toString() { + return "type: " + name; + } + + } + + public StreamingField(SDField field) { + this(field.getName(),field.getDataType(),field.getMatching(), field.doesAttributing()); + } + + private StreamingField(String name,DataType sourceType, Matching matching, boolean isAttribute) { + this.name = name; + this.type = convertType(sourceType); + this.matching = matching; + this.isAttribute = isAttribute; + } + + /** Converts to the right index type from a field datatype */ + private static Type convertType(DataType fieldType) { + FieldValue fval = fieldType.createFieldValue(); + if (fieldType.equals(DataType.FLOAT)) { + return Type.FLOAT; + } else if (fieldType.equals(DataType.LONG)) { + return Type.INT64; + } else if (fieldType.equals(DataType.DOUBLE)) { + return Type.DOUBLE; + } else if (fieldType.equals(DataType.BYTE)) { + return Type.INT8; + } else if (fieldType instanceof NumericDataType) { + return Type.INT32; + } else if (fval instanceof StringFieldValue) { + return Type.STRING; + } else if (fval instanceof Raw) { + return Type.STRING; + } else if (fval instanceof PredicateFieldValue) { + return Type.UNSEARCHABLESTRING; + } else if (fval instanceof TensorFieldValue) { + return Type.UNSEARCHABLESTRING; + } else if (fieldType instanceof CollectionDataType) { + return convertType(((CollectionDataType) fieldType).getNestedType()); + } else { + throw new IllegalArgumentException("Don't know which streaming" + + " field type to " + "convert " + fieldType + " to"); + } + } + + public String getName() { return name; } + + public VsmfieldsConfig.Fieldspec.Builder getFieldSpecConfig() { + VsmfieldsConfig.Fieldspec.Builder fB = new VsmfieldsConfig.Fieldspec.Builder(); + String matchingName = matching.getType().getName(); + if (matching.getType().equals(Matching.Type.TEXT)) + matchingName = ""; + if (matching.getType() != Matching.Type.EXACT) { + if (matching.isPrefix()) { + matchingName = "prefix"; + } else if (matching.isSubstring()) { + matchingName = "substring"; + } else if (matching.isSuffix()) { + matchingName = "suffix"; + } + } + if (type != Type.STRING) { + matchingName = ""; + } + fB.name(getName()) + .searchmethod(VsmfieldsConfig.Fieldspec.Searchmethod.Enum.valueOf(type.getSearchMethod())) + .arg1(matchingName) + .fieldtype(isAttribute + ? VsmfieldsConfig.Fieldspec.Fieldtype.ATTRIBUTE + : VsmfieldsConfig.Fieldspec.Fieldtype.INDEX); + if (matching.maxLength() != null) { + fB.maxlength(matching.maxLength()); + } + return fB; + } + + public boolean equals(Object o) { + if (o.getClass().equals(getClass())) { + StreamingField sf = (StreamingField)o; + return name.equals(sf.name) && + matching.equals(sf.matching) && + type.equals(sf.type); + } + return false; + } + + } + + private static class StreamingDocumentType { + private final String name; + private final Map<String, FieldSet> fieldSets = new LinkedHashMap<>(); + private final Map<String, FieldSet> userFieldSets; + + public StreamingDocumentType(String name, FieldSets fieldSets) { + this.name=name; + userFieldSets = fieldSets.userFieldSets(); + } + + public VsmfieldsConfig.Documenttype.Builder getDocTypeConfig() { + VsmfieldsConfig.Documenttype.Builder dtB = new VsmfieldsConfig.Documenttype.Builder(); + dtB.name(name); + Map<String, FieldSet> all = new LinkedHashMap<>(); + all.putAll(fieldSets); + all.putAll(userFieldSets); + for (Map.Entry<String, FieldSet> e : all.entrySet()) { + VsmfieldsConfig.Documenttype.Index.Builder indB = new VsmfieldsConfig.Documenttype.Index.Builder(); + indB.name(e.getValue().getName()); + for (String field : e.getValue().getFieldNames()) { + indB.field(new VsmfieldsConfig.Documenttype.Index.Field.Builder().name(field)); + } + dtB.index(indB); + } + return dtB; + } + + public String getName() { return name; } + + public void addIndexField(String indexName, String fieldName) { + FieldSet fs = fieldSets.get(indexName); + if (fs == null) { + fs = new FieldSet(indexName); + fieldSets.put(indexName, fs); + } + fs.addFieldName(fieldName); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java new file mode 100644 index 00000000000..aaf376f5cd9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java @@ -0,0 +1,107 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived; + +import com.yahoo.document.PositionDataType; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig; + +import java.util.*; + +/** + * Vertical streaming matcher summary specification + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class VsmSummary extends Derived implements VsmsummaryConfig.Producer { + private Map<SummaryField, List<String>> summaryMap = new java.util.LinkedHashMap<>(1); + + public VsmSummary(Search search) { + derive(search); + } + + @Override + protected void derive(Search search) { + // Use the default class, as it is the superset + derive(search, search.getSummary("default")); + } + + private void derive(Search search, DocumentSummary documentSummary) { + if (documentSummary==null) return; + for (SummaryField summaryField : documentSummary.getSummaryFields()) { + List<String> from = toStringList(summaryField.sourceIterator()); + + if (doMapField(search, summaryField)) { + SDField sdField = search.getField(summaryField.getName()); + if (sdField != null && PositionDataType.INSTANCE.equals(sdField.getDataType())) { + summaryMap.put(summaryField, Collections.singletonList(summaryField.getName())); + } else { + summaryMap.put(summaryField, from); + } + } + } + } + + /** + * Don't include field in map if sources are the same as the struct sub fields for the SDField. + * But do map if not all do summarying. + * Don't map if not struct either. + * @param summaryField a {@link SummaryField} + */ + private boolean doMapField(Search search, SummaryField summaryField) { + SDField sdField = search.getField(summaryField.getName()); + SDDocumentType document = search.getDocument(); + if (sdField==null || ((document != null) && (document.getField(summaryField.getName()) == sdField))) { + return true; + } + if (summaryField.getVsmCommand().equals(SummaryField.VsmCommand.FLATTENJUNIPER)) { + return true; + } + if (!sdField.usesStructOrMap()) { + return !(sdField.getName().equals(summaryField.getName())); + } + if (summaryField.getSourceCount()==sdField.getStructFields().size()) { + for (SummaryField.Source source : summaryField.getSources()) { + if (!sdField.getStructFields().contains(new SDField(search.getDocument(), source.getName(), sdField.getDataType()))) { // equals() uses just name + return true; + } + if (sdField.getStructField(source.getName())!=null && !sdField.getStructField(source.getName()).doesSummarying()) { + return true; + } + } + // The sources in the summary field are the same as the sub-fields in the SD field. + // All sub fields do summarying. + // Don't map. + return false; + } + return true; + } + + private List<String> toStringList(Iterator i) { + List<String> ret = new ArrayList<>(); + while (i.hasNext()) { + ret.add(i.next().toString()); + } + return ret; + } + + public String getDerivedName() { + return "vsmsummary"; + } + + @Override + public void getConfig(VsmsummaryConfig.Builder vB) { + for (Map.Entry<SummaryField, List<String>> entry : summaryMap.entrySet()) { + VsmsummaryConfig.Fieldmap.Builder fmB = new VsmsummaryConfig.Fieldmap.Builder().summary(entry.getKey().getName()); + for (String field : entry.getValue()) { + fmB.document(new VsmsummaryConfig.Fieldmap.Document.Builder().field(field)); + } + fmB.command(VsmsummaryConfig.Fieldmap.Command.Enum.valueOf(entry.getKey().getVsmCommand().toString())); + vB.fieldmap(fmB); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/package-info.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/package-info.java new file mode 100644 index 00000000000..4e63729ea34 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.searchdefinition.derived; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/IndexStructureValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/IndexStructureValidator.java new file mode 100644 index 00000000000..360b4d0d637 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/IndexStructureValidator.java @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived.validation; + +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.searchdefinition.derived.IndexingScript; +import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; +import com.yahoo.vespa.indexinglanguage.expressions.*; + +/** + * @author <a href="mailto:mlidal@yahoo-inc.com">Mathias M Lidal</a> + */ +public class IndexStructureValidator extends Validator { + + public IndexStructureValidator(DerivedConfiguration config, Search search) { + super(config, search); + } + + public void validate() { + IndexingScript script = config.getIndexingScript(); + for (Expression exp : script.expressions()) { + new OutputVisitor(search.getDocument(), exp).visit(exp); + } + } + + private static class OutputVisitor extends ExpressionVisitor { + + final SDDocumentType docType; + final Expression exp; + + public OutputVisitor(SDDocumentType docType, Expression exp) { + this.docType = docType; + this.exp = exp; + } + + @Override + protected void doVisit(Expression exp) { + if (!(exp instanceof OutputExpression)) { + return; + } + String fieldName = ((OutputExpression)exp).getFieldName(); + if (docType.getField(fieldName) != null) { + return; + } + throw new IllegalArgumentException("Indexing expression '" + this.exp + "' refers to field '" + + fieldName + "' which does not exist in the index structure."); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validation.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validation.java new file mode 100644 index 00000000000..9b01881ddd8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validation.java @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived.validation; + +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.derived.DerivedConfiguration; + +public class Validation { + + public static void validate(DerivedConfiguration config, Search search) { + new IndexStructureValidator(config, search).validate(); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validator.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validator.java new file mode 100644 index 00000000000..8c7c5afcb15 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validator.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.derived.validation; + +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.derived.DerivedConfiguration; + +import java.util.logging.Logger; + +/** + * @author mathiasm + */ +public abstract class Validator { + protected DerivedConfiguration config; + protected Search search; + + protected Validator(DerivedConfiguration config, Search search) { + this.config = config; + this.search = search; + } + + public abstract void validate(); + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java new file mode 100644 index 00000000000..35f6a41f0d8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java @@ -0,0 +1,320 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document; + +import com.yahoo.document.*; +import com.yahoo.document.datatypes.*; +import com.yahoo.tensor.TensorType; + +import java.io.Serializable; +import java.util.LinkedHashSet; +import java.util.Optional; +import java.util.Set; + +/** + * A search-time document attribute (per-document in-memory value). + * This belongs to the field defining the attribute. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public final class Attribute implements Cloneable, Serializable { + + // Remember to change hashCode and equals when you add new fields + + private String name; + + private Type type; + private CollectionType collectionType; + + /** True if only the enum information should be read from this attribute + * (i.e. the actual values are not relevant, only which documents have the + * same values) Used for collapsing and unique. + */ + private boolean removeIfZero = false; + private boolean createIfNonExistent = false; + private boolean enableBitVectors = false; + private boolean enableOnlyBitVector = false; + + private boolean fastSearch = false; + private boolean fastAccess = false; + private boolean huge = false; + private int arity = BooleanIndexDefinition.DEFAULT_ARITY; + private long lowerBound = BooleanIndexDefinition.DEFAULT_LOWER_BOUND; + private long upperBound = BooleanIndexDefinition.DEFAULT_UPPER_BOUND; + private double densePostingListThreshold = BooleanIndexDefinition.DEFAULT_DENSE_POSTING_LIST_THRESHOLD; + private Optional<TensorType> tensorType = Optional.empty(); + + private boolean isPosition = false; + private final Sorting sorting = new Sorting(); + + /** The aliases for this attribute */ + private final Set<String> aliases = new LinkedHashSet<>(); + + /** + * True if this attribute should be returned during first pass of search. + * Null means make the default decision for this kind of attribute + */ + private Boolean prefetch = null; + + /** The attribute type enumeration */ + public enum Type { + BYTE("byte", "INT8"), + SHORT("short", "INT16"), + INTEGER("integer", "INT32"), + LONG("long", "INT64"), + FLOAT("float", "FLOAT"), + DOUBLE("double", "DOUBLE"), + STRING("string", "STRING"), + PREDICATE("predicate", "PREDICATE"), + TENSOR("tensor", "TENSOR"); + + private final String myName; // different from what name() returns. + private final String exportAttributeTypeName; + + private Type(String name, String exportAttributeTypeName) { + this.myName=name; + this.exportAttributeTypeName = exportAttributeTypeName; + } + + public String getName() { return myName; } + public String getExportAttributeTypeName() { return exportAttributeTypeName; } + + public String toString() { + return "type: " + myName; + } + } + + /** The attribute collection type enumeration */ + public enum CollectionType { + + SINGLE("SINGLE"), + ARRAY("ARRAY"), + WEIGHTEDSET ("WEIGHTEDSET"); + + private final String name; + + private CollectionType(String name) { + this.name=name; + } + + public String getName() { return name; } + + public String toString() { + return "collectiontype: " + name; + } + } + + /** Creates an attribute with default settings */ + public Attribute(String name,DataType fieldType) { + this(name,convertDataType(fieldType), convertCollectionType(fieldType)); + setRemoveIfZero(fieldType instanceof WeightedSetDataType ? ((WeightedSetDataType)fieldType).removeIfZero() : false); + setCreateIfNonExistent(fieldType instanceof WeightedSetDataType ? ((WeightedSetDataType)fieldType).createIfNonExistent() : false); + } + + public Attribute(String name,Type type, CollectionType collectionType) { + this.name=name; + setType(type); + setCollectionType(collectionType); + } + + /** + * <p>Returns whether this attribute should be included in the "attributeprefetch" summary + * which is returned to the Qrs by prefetchAttributes, used by blending, uniquing etc. + * + * <p>Single value attributes are prefetched by default if summary is true. + * Multi value attributes are not.</p> + */ + public boolean isPrefetch() { + if (prefetch!=null) return prefetch.booleanValue(); + + if (CollectionType.SINGLE.equals(collectionType)) { + return true; + } + + return false; + } + + /** Returns the prefetch value of this, null if the default is used. */ + public Boolean getPrefetchValue() { return prefetch; } + + public boolean isRemoveIfZero() { return removeIfZero; } + public boolean isCreateIfNonExistent(){ return createIfNonExistent; } + public boolean isEnabledBitVectors() { return enableBitVectors; } + public boolean isEnabledOnlyBitVector() { return enableOnlyBitVector; } + public boolean isFastSearch() { return fastSearch; } + public boolean isFastAccess() { return fastAccess; } + public boolean isHuge() { return huge; } + public boolean isPosition() { return isPosition; } + + public int arity() { return arity; } + public long lowerBound() { return lowerBound; } + public long upperBound() { return upperBound; } + public double densePostingListThreshold() { return densePostingListThreshold; } + public Optional<TensorType> tensorType() { return tensorType; } + + public Sorting getSorting() { return sorting; } + + public void setRemoveIfZero(boolean remove) { this.removeIfZero = remove; } + public void setCreateIfNonExistent(boolean create) { this.createIfNonExistent = create; } + + /** + * Sets whether this should be included in the "attributeprefetch" document summary. + * True or false to override default, null to use default + */ + public void setPrefetch(Boolean prefetch) { this.prefetch = prefetch; } + public void setEnableBitVectors(boolean enableBitVectors) { this.enableBitVectors = enableBitVectors; } + public void setEnableOnlyBitVector(boolean enableOnlyBitVector) { this.enableOnlyBitVector = enableOnlyBitVector; } + public void setFastSearch(boolean fastSearch) { this.fastSearch = fastSearch; } + public void setHuge(boolean huge) { this.huge = huge; } + public void setFastAccess(boolean fastAccess) { this.fastAccess = fastAccess; } + public void setPosition(boolean position) { this.isPosition = position; } + public void setArity(int arity) { this.arity = arity; } + public void setLowerBound(long lowerBound) { this.lowerBound = lowerBound; } + public void setUpperBound(long upperBound) { this.upperBound = upperBound; } + public void setDensePostingListThreshold(double threshold) { this.densePostingListThreshold = threshold; } + public void setTensorType(TensorType tensorType) { this.tensorType = Optional.of(tensorType); } + + public String getName() { return name; } + public Type getType() { return type; } + public CollectionType getCollectionType() { return collectionType; } + + public void setName(String name) { this.name=name; } + private void setType(Type type) { this.type=type; } + public void setCollectionType(CollectionType type) { this.collectionType=type; } + + /** Converts to the right attribute type from a field datatype */ + public static Type convertDataType(DataType fieldType) { + FieldValue fval = fieldType.createFieldValue(); + if (fval instanceof StringFieldValue) { + return Type.STRING; + } else if (fval instanceof IntegerFieldValue) { + return Type.INTEGER; + } else if (fval instanceof LongFieldValue) { + return Type.LONG; + } else if (fval instanceof FloatFieldValue) { + return Type.FLOAT; + } else if (fval instanceof DoubleFieldValue) { + return Type.DOUBLE; + } else if (fval instanceof ByteFieldValue) { + return Type.BYTE; + } else if (fval instanceof Raw) { + return Type.BYTE; + } else if (fval instanceof PredicateFieldValue) { + return Type.PREDICATE; + } else if (fval instanceof TensorFieldValue) { + return Type.TENSOR; + } else if (fieldType instanceof CollectionDataType) { + return convertDataType(((CollectionDataType) fieldType).getNestedType()); + } else { + throw new IllegalArgumentException("Don't know which attribute type to " + + "convert " + fieldType + " to"); + } + } + + /** Converts to the right attribute type from a field datatype */ + public static CollectionType convertCollectionType(DataType fieldType) { + if (fieldType instanceof ArrayDataType) { + return CollectionType.ARRAY; + } else if (fieldType instanceof WeightedSetDataType) { + return CollectionType.WEIGHTEDSET; + } else if (fieldType instanceof PrimitiveDataType) { + return CollectionType.SINGLE; + } else { + throw new IllegalArgumentException("Field " + fieldType + " not supported in convertCollectionType"); + } + } + + /** Converts to the right field type from an attribute type */ + public static DataType convertAttrType(Type attrType) { + if (attrType== Type.STRING) { + return DataType.STRING; + } else if (attrType== Type.INTEGER) { + return DataType.INT; + } else if (attrType== Type.LONG) { + return DataType.LONG; + } else if (attrType== Type.FLOAT) { + return DataType.FLOAT; + } else if (attrType== Type.DOUBLE) { + return DataType.DOUBLE; + } else if (attrType == Type.BYTE) { + return DataType.BYTE; + } else if (attrType == Type.PREDICATE) { + return DataType.PREDICATE; + } else if (attrType == Type.TENSOR) { + return DataType.TENSOR; + } else { + throw new IllegalArgumentException("Don't know which attribute type to " + + "convert " + attrType + " to"); + } + } + + public DataType getDataType() { + DataType dataType = Attribute.convertAttrType(type); + if (collectionType.equals(Attribute.CollectionType.ARRAY)) { + return DataType.getArray(dataType); + } else if (collectionType.equals(Attribute.CollectionType.WEIGHTEDSET)) { + return DataType.getWeightedSet(dataType, createIfNonExistent, removeIfZero); + } else { + return dataType; + } + } + + public int hashCode() { + return name.hashCode() + + type.hashCode() + + collectionType.hashCode() + + sorting.hashCode() + + (isPrefetch() ? 13 : 0) + + (fastSearch ? 17 : 0) + + (removeIfZero ? 47 : 0) + + (createIfNonExistent ? 53 : 0) + + (isPosition ? 61 : 0) + + (huge ? 67 : 0) + + (enableBitVectors ? 71 : 0) + + (enableOnlyBitVector ? 73 : 0) + + tensorType.hashCode(); + } + + public boolean equals(Object object) { + if (! (object instanceof Attribute)) return false; + + Attribute other=(Attribute)object; + if (!this.name.equals(other.name)) return false; + return isCompatible(other); + } + + /** Returns whether these attributes describes the same entity, even if they have different names */ + public boolean isCompatible(Attribute other) { + if ( ! this.type.equals(other.type)) return false; + if ( ! this.collectionType.equals(other.collectionType)) return false; + if (this.isPrefetch() != other.isPrefetch()) return false; + if (this.removeIfZero != other.removeIfZero) return false; + if (this.createIfNonExistent != other.createIfNonExistent) return false; + if (this.enableBitVectors != other.enableBitVectors) return false; + if (this.enableOnlyBitVector != other.enableOnlyBitVector) return false; + // if (this.noSearch != other.noSearch) return false; No backend consequences so compatible for now + if (this.fastSearch != other.fastSearch) return false; + if (this.huge != other.huge) return false; + if ( ! this.sorting.equals(other.sorting)) return false; + if (!this.tensorType.equals(other.tensorType)) return false; + + return true; + } + + public @Override Attribute clone() { + try { + return (Attribute)super.clone(); + } + catch (CloneNotSupportedException e) { + throw new RuntimeException("Programming error"); + } + } + + public String toString() { + return "attribute '" + name + "' (" + type + ")"; + } + + public Set<String> getAliases() { + return aliases; + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/BooleanIndexDefinition.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/BooleanIndexDefinition.java new file mode 100644 index 00000000000..fc5494d0f53 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/BooleanIndexDefinition.java @@ -0,0 +1,73 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document; + +import java.util.OptionalDouble; +import java.util.OptionalInt; +import java.util.OptionalLong; + +/** + * Encapsulates values required for native implementation of boolean search. + * + * @author <a href="mailto:lesters@yahoo-inc.com">Lester Solbakken</a> + * @since 5.2 + */ +public final class BooleanIndexDefinition +{ + public static final int DEFAULT_ARITY = 8; + public static final long DEFAULT_UPPER_BOUND = Long.MAX_VALUE; + public static final long DEFAULT_LOWER_BOUND = Long.MIN_VALUE; + public static final double DEFAULT_DENSE_POSTING_LIST_THRESHOLD = 0.4; + + private final OptionalInt arity; // mandatory field value + private final OptionalLong lowerBound; + private final OptionalLong upperBound; + private final OptionalDouble densePostingListThreshold; + + public BooleanIndexDefinition(OptionalInt arity, OptionalLong lowerBound, + OptionalLong upperBound, OptionalDouble densePostingListThreshold) { + this.arity = arity; + this.lowerBound = lowerBound; + this.upperBound = upperBound; + this.densePostingListThreshold = densePostingListThreshold; + } + + public int getArity() { + return arity.getAsInt(); + } + + public boolean hasArity() { + return arity.isPresent(); + } + + public long getLowerBound() { + return lowerBound.orElse(DEFAULT_LOWER_BOUND); + } + + public boolean hasLowerBound() { + return lowerBound.isPresent(); + } + + public long getUpperBound() { + return upperBound.orElse(DEFAULT_UPPER_BOUND); + } + + public boolean hasUpperBound() { + return upperBound.isPresent(); + } + + public double getDensePostingListThreshold() { + return densePostingListThreshold.orElse(DEFAULT_DENSE_POSTING_LIST_THRESHOLD); + } + + public boolean hasDensePostingListThreshold() { + return densePostingListThreshold.isPresent(); + } + + @Override + public String toString() { + return "BooleanIndexDefinition [arity=" + arity + ", lowerBound=" + + lowerBound + ", upperBound=" + upperBound + ", densePostingListThreshold=" + + densePostingListThreshold + "]"; + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java new file mode 100644 index 00000000000..040a798d3b3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document; + +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.TreeSet; + +/** + * Searchable collection of fields. + * + * @author balder + */ +public class FieldSet { + + private final String name; + private final Set<String> queryCommands = new LinkedHashSet<>(); + private final Set<String> fieldNames = new TreeSet<>(); + private final Set<SDField> fields = new TreeSet<>(); + private Matching matching = null; + + public FieldSet(String name) { this.name = name; } + public String getName() { return name; } + public FieldSet addFieldName(String field) { fieldNames.add(field); return this; } + public Set<String> getFieldNames() { return fieldNames; } + public Set<SDField> fields() { return fields; } + + public Set<String> queryCommands() { + return queryCommands; + } + + public void setMatching(Matching matching) { + this.matching = matching; + } + + public Matching getMatching() { + return matching; + } + + @Override + public String toString() { return "fieldset '" + name + "'"; } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java new file mode 100644 index 00000000000..3c90bcef513 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java @@ -0,0 +1,151 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document; + +import java.io.Serializable; + +/** + * Defines how a field should be matched. + * Matching objects can be compared based on their content, but they are <i>not</i> immutable. + * + * @author bratseth + */ +public class Matching implements Cloneable, Serializable { + + public static final Type defaultType = Type.TEXT; + + public static enum Type { + TEXT("text"), + WORD("word"), + EXACT("exact"), + GRAM("gram"); + private String name; + Type(String name) { this.name = name; } + public String getName() { return name; } + } + + /** Which match algorithm is used by this matching setup */ + public enum Algorithm { + NORMAL("normal"), + PREFIX("prefix"), + SUBSTRING("substring"), + SUFFIX("suffix"); + private String name; + Algorithm(String name) { this.name = name; } + public String getName() { return name; } + } + + private Type type = Type.TEXT; + + /** The basic match algorithm */ + private Algorithm algorithm = Algorithm.NORMAL; + + private boolean typeUserSet = false; + + private boolean algorithmUserSet = false; + + /** The gram size is the n in n-gram, or -1 if not set. Should only be set with gram matching. */ + private int gramSize=-1; + + /** Maximum number of characters to consider when searching in this field. Used for limiting resources, especially in streaming search. */ + private Integer maxLength; + + private String exactMatchTerminator=null; + + /** Creates a matching of type "text" */ + public Matching() {} + + public Matching(Type type) { + this.type = type; + } + + public Type getType() { return type; } + + public void setType(Type type) { + this.type = type; + typeUserSet = true; + } + + public Integer maxLength() { return maxLength; } + public Matching maxLength(int maxLength) { this.maxLength = maxLength; return this; } + public boolean isTypeUserSet() { return typeUserSet; } + + public Algorithm getAlgorithm() { return algorithm; } + + public void setAlgorithm(Algorithm algorithm) { + this.algorithm = algorithm; + algorithmUserSet = true; + } + + public boolean isAlgorithmUserSet() { return algorithmUserSet; } + + public boolean isPrefix() { return algorithm == Algorithm.PREFIX; } + + public boolean isSubstring() { return algorithm == Algorithm.SUBSTRING; } + + public boolean isSuffix() { return algorithm == Algorithm.SUFFIX; } + + /** Returns the gram size, or -1 if not set. Should only be set with gram matching. */ + public int getGramSize() { return gramSize; } + + public void setGramSize(int gramSize) { this.gramSize=gramSize; } + + /** + * Merge data from another matching object + */ + public void merge(Matching m) { + if (m.isAlgorithmUserSet()) { + this.setAlgorithm(m.getAlgorithm()); + } + if (m.isTypeUserSet()) { + this.setType(m.getType()); + if (m.getType()==Type.GRAM) + gramSize=m.gramSize; + } + if (m.getExactMatchTerminator() != null) { + this.setExactMatchTerminator(m.getExactMatchTerminator()); + } + } + + /** + * If exact matching is used, this returns the terminator string + * which terminates an exact matched sequence in queries. If exact + * matching is not used, or no terminator is set, this is null + */ + public String getExactMatchTerminator() { return exactMatchTerminator; } + + /** + * Sets the terminator string which terminates an exact matched + * sequence in queries (used if type is EXACT). + */ + public void setExactMatchTerminator(String exactMatchTerminator) { + this.exactMatchTerminator = exactMatchTerminator; + } + + public String toString() { + return type + " matching [" + (type==Type.GRAM ? "gram size " + gramSize : "supports " + algorithm) + "], [exact-terminator "+exactMatchTerminator+"]"; + } + + public Matching clone() { + try { + return (Matching)super.clone(); + } + catch (CloneNotSupportedException e) { + throw new RuntimeException("Programming error"); + } + } + + public boolean equals(Object o) { + if (! (o instanceof Matching)) return false; + + Matching other=(Matching)o; + if ( ! other.type.equals(this.type)) return false; + if ( ! other.algorithm.equals(this.algorithm)) return false; + if ( this.exactMatchTerminator==null && other.exactMatchTerminator!=null) return false; + if ( this.exactMatchTerminator!=null && ( ! this.exactMatchTerminator.equals(other.exactMatchTerminator)) ) + return false; + if ( gramSize!=other.gramSize) return false; + return true; + } + + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/NormalizeLevel.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/NormalizeLevel.java new file mode 100644 index 00000000000..945593d550b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/NormalizeLevel.java @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document; + +/** + * class representing the character normalization + * we want to do on query and indexed text. + * Levels are strict subsets, so doing accent + * removal means doing codepoint normalizing + * and case normalizing also. + */ +// TODO: Missing author +public class NormalizeLevel { + + /** + * The current levels are as follows: + * NONE: no changes to input text + * CODEPOINT: convert text into Unicode + * Normalization Form Compatibility Composition + * LOWERCASE: also convert text into lowercase letters + * ACCENT: do both above and remove accents on characters + */ + public enum Level { + NONE, CODEPOINT, LOWERCASE, ACCENT + } + + private boolean userSpecified = false; + private Level level = Level.ACCENT; + + /** + * Returns whether accents should be removed from text + */ + public boolean doRemoveAccents() { return level == Level.ACCENT; } + + /** + * Construct a default (full) normalizelevel, + */ + public NormalizeLevel() {} + + /** + * Construct for a specific level, possibly user specified + * + * @param level which level to use + * @param fromUser whether this was specified by the user + */ + public NormalizeLevel(Level level, boolean fromUser) { + this.level = level; + this.userSpecified = fromUser; + } + + /** + * Change the current level to CODEPOINT as inferred + * by other features' needs. If the current level + * was user specified it will not change; also this + * will not increase the level. + */ + public void inferCodepoint() { + if (userSpecified) { + // ignore inferred changes if user specified something + return; + } + // do not increase level + if (level != Level.NONE) level = Level.CODEPOINT; + } + + /** + * Change the current level to LOWERCASE as inferred + * by other features' needs. If the current level + * was user specified it will not change; also this + * will not increase the level. + */ + public void inferLowercase() { + if (userSpecified) { + // ignore inferred changes if user specified something + return; + } + // do not increase level + if (level == Level.NONE) return; + if (level == Level.CODEPOINT) return; + + level = Level.LOWERCASE; + } + + public Level getLevel() { + return level; + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/RankType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/RankType.java new file mode 100644 index 00000000000..f2b0b6a9164 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/RankType.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.Collections; +import java.util.Set; + +/** + * The rank type of a field. For now this is just a container of a string name. + * This class is immutable. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public enum RankType { + + /** *implicit* default: No type has been set. */ + DEFAULT, + + // Rank types which can be set explicitly. These are defined for Vespa in NativeRankTypeDefinitionSet + IDENTITY, ABOUT, TAGS, EMPTY; + + @Override + public String toString() { + return "rank type " + name().toLowerCase(); + } + + /** + * Returns the rank type from a string, regardless of its case. + * + * @param rankTypeName a rank type name in any casing + * @return the rank type found + * @throws IllegalArgumentException if not found + */ + public static RankType fromString(String rankTypeName) { + try { + return RankType.valueOf(rankTypeName.toUpperCase()); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Unknown rank type '" + rankTypeName + "'. Supported rank types are " + + "'identity', 'about', 'tags' and 'empty'."); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java new file mode 100644 index 00000000000..9e0bda7fd5a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document; + +import java.io.Serializable; + +/** + * The rank settings given in a rank clause in the search definition. + * + * @author <a href="mailto:vehardh@yahoo-inc.com">Vegard Havdal</a> + */ +public class Ranking implements Cloneable, Serializable { + + private boolean literal = false; + private boolean filter = false; + private boolean normal = false; + + /** + * <p>Returns whether literal (non-stemmed, non-normalized) forms of the words should + * be indexed in a separate index which is searched by a automatically added rank term + * during searches.</p> + * + * <p>Default is false.</p> + */ + public boolean isLiteral() { return literal; } + + public void setLiteral(boolean literal) { this.literal = literal; } + + /** + * <p>Returns whether this is a filter. Filters will only tell if they are matched or not, + * no detailed relevance information will be available about the match.</p> + * + * <p>Matching a filter is much cheaper for the search engine than matching a regular field.</p> + * + * <p>Default is false.</p> + */ + public boolean isFilter() { return filter && !normal; } + + public void setFilter(boolean filter) { this.filter = filter; } + + /** Whether user has explicitly requested normal (non-filter) behavior */ + public boolean isNormal() { return normal; } + public void setNormal(boolean n) { this.normal = n; } + + /** Returns true if the given rank settings are the same */ + public @Override boolean equals(Object o) { + if ( ! (o instanceof Ranking)) return false; + + Ranking other=(Ranking)o; + if (this.filter != other.filter) return false; + if (this.literal != other.literal) return false; + if (this.normal != other.normal) return false; + return true; + } + + public @Override String toString() { + return "rank settings [filter: " + filter + ", literal: " + literal + ", normal: "+normal+"]"; + } + + public @Override Ranking clone() { + try { + return (Ranking)super.clone(); + } + catch (CloneNotSupportedException e) { + throw new RuntimeException("Programming error",e); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java new file mode 100644 index 00000000000..6292dc9ef72 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java @@ -0,0 +1,316 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document; + +import com.yahoo.document.*; +import com.yahoo.document.annotation.AnnotationType; +import com.yahoo.document.annotation.AnnotationTypeRegistry; +import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.documentmodel.VespaDocumentType; +import com.yahoo.searchdefinition.FieldSets; +import com.yahoo.searchdefinition.Search; + +import java.io.Serializable; +import java.util.*; +import java.util.logging.Logger; + +/** + <p>A document definition is a list of fields. Documents may inherit other documents, + implicitly acquiring their fields as it's own. If a document is not set to inherit + any document, it will always inherit the document "document.0".</p> + + @author <a href="thomasg@yahoo-inc.com">Thomas Gundersen</a> + @author <a href="bratseth@yahoo-inc.com">Jon S Bratseth</a> +*/ +public class SDDocumentType implements Cloneable, Serializable { + public static final SDDocumentType VESPA_DOCUMENT; + private Map<DataTypeName, SDDocumentType> inheritedTypes = new HashMap<>(); + private Map<NewDocumentType.Name, SDDocumentType> ownedTypes = new HashMap<>(); + private AnnotationTypeRegistry annotationTypes = new AnnotationTypeRegistry(); + private DocumentType docType; + private DataType structType; + // The field sets here are set from the processing step in SD, to ensure that the full Search and this SDDocumentType is built first. + private FieldSets fieldSets; + + static { + VESPA_DOCUMENT = new SDDocumentType(VespaDocumentType.INSTANCE.getFullName().getName()); + VESPA_DOCUMENT.addType(createSDDocumentType(PositionDataType.INSTANCE)); + } + + public SDDocumentType clone() throws CloneNotSupportedException { + SDDocumentType type = (SDDocumentType) super.clone(); + type.docType = docType.clone(); + type.inheritedTypes.putAll(inheritedTypes); + type.structType = structType; + return type; + } + + /** + * For adding structs defined in document scope + * + * @param dt The struct to add. + * @return self, for chaining + */ + public SDDocumentType addType(SDDocumentType dt) { + NewDocumentType.Name name = new NewDocumentType.Name(dt.getName()); + if (getType(name) != null) { + throw new IllegalArgumentException("Data type '" + name.toString() + "' has already been used."); + } + if (name.getName() == docType.getName()) { + throw new IllegalArgumentException("Data type '" + name.toString() + "' can not have same name as its defining document."); + } + ownedTypes.put(name, dt); + return this; + } + public final SDDocumentType getOwnedType(String name) { + return getOwnedType(new NewDocumentType.Name(name)); + } + public SDDocumentType getOwnedType(DataTypeName name) { + return getOwnedType(name.getName()); + } + + public SDDocumentType getOwnedType(NewDocumentType.Name name) { + return ownedTypes.get(name); + } + + public final SDDocumentType getType(String name) { + return getType(new NewDocumentType.Name(name)); + } + + public SDDocumentType getType(NewDocumentType.Name name) { + SDDocumentType type = ownedTypes.get(name); + if (type == null) { + for (SDDocumentType inherited : inheritedTypes.values()) { + type = inherited.getType(name); + if (type != null) { + return type; + } + } + } + return type; + } + + public SDDocumentType addAnnotation(AnnotationType annotation) { + annotationTypes.register(annotation); + return this; + } + + /** + * Access to all owned datatypes. + * @return all types + */ + public Collection<SDDocumentType> getTypes() { return ownedTypes.values(); } + public Collection<AnnotationType> getAnnotations() { return annotationTypes.getTypes().values(); } + public AnnotationType findAnnotation(String name) { return annotationTypes.getType(name); } + + public Collection<SDDocumentType> getAllTypes() { + Collection<SDDocumentType> list = new ArrayList<>(); + list.addAll(getTypes()); + for (SDDocumentType inherited : inheritedTypes.values()) { + list.addAll(inherited.getAllTypes()); + } + return list; + } + + /** + * Creates a new document type. + * The document type id will be generated as a hash from the document type name. + * + * @param name The name of the new document type + */ + public SDDocumentType(String name) { + this(name,null); + } + + public SDDocumentType(DataTypeName name) { + this(name.getName()); + } + + /** + * Creates a new document type. + * The document type id will be generated as a hash from the document type name. + * + * @param name The name of the new document type + * @param search check for type ID collisions in this search definition + */ + public SDDocumentType(String name, Search search) { + docType = new DocumentType(name); + docType.getHeaderType().setCompressionConfig(new CompressionConfig()); + docType.getBodyType().setCompressionConfig(new CompressionConfig()); + validateId(search); + inherit(VESPA_DOCUMENT); + } + + public boolean isStruct() { return getStruct() != null; } + public DataType getStruct() { return structType; } + public SDDocumentType setStruct(DataType structType) { + if (structType != null) { + this.structType = structType; + inheritedTypes.clear(); + } else { + if (docType.getHeaderType() != null) { + this.structType = docType.getHeaderType(); + inheritedTypes.clear(); + } else { + throw new IllegalArgumentException("You can not set a null struct"); + } + } + return this; + } + + public String getName() { return docType.getName(); } + public DataTypeName getDocumentName() { return docType.getDataTypeName(); } + public DocumentType getDocumentType() { return docType; } + + public void inherit(DataTypeName name) { + if ( ! inheritedTypes.containsKey(name)) { + inheritedTypes.put(name, new TemporarySDDocumentType(name)); + } + } + + public void inherit(SDDocumentType type) { + if (type != null) { + if (!inheritedTypes.containsKey(type.getDocumentName()) || (inheritedTypes.get(type.getDocumentName()) instanceof TemporarySDDocumentType)) { + inheritedTypes.put(type.getDocumentName(), type); + } + } + } + + public Collection<SDDocumentType> getInheritedTypes() { return inheritedTypes.values(); } + + protected void validateId(Search search) { + if (search == null) return; + if (search.getDocument(getName()) == null) return; + SDDocumentType doc = search.getDocument(); + throw new IllegalArgumentException("Failed creating document type '" + getName() + "', " + + "document type '" + doc.getName() + "' already uses ID '" + doc.getName() + "'"); + } + + public void setFieldId(SDField field, int id) { + field.setId(id, docType); + } + + /** Override getField, as it may need to ask inherited types that isn't registered in document type. + * @param name Name of field to get. + * @return The field found. + */ + public Field getField(String name) { + if (name.contains(".")) { + String superFieldName = name.substring(0,name.indexOf(".")); + String subFieldName = name.substring(name.indexOf(".")+1); + Field f = docType.getField(superFieldName); + if (f != null) { + if (f instanceof SDField) { + SDField superField = (SDField)f; + return superField.getStructField(subFieldName); + } else { + throw new IllegalArgumentException("Field "+f.getName()+" is not SDField"); + } + } + } + Field f = docType.getField(name); + if (f == null) { + for(SDDocumentType parent : inheritedTypes.values()) { + f = parent.getField(name); + if (f != null) return f; + } + } + return f; + } + + public void addField(Field field) { + verifyInheritance(field); + for (Iterator<Field> i = docType.fieldIteratorThisTypeOnly(); i.hasNext(); ) { + if (field.getName().equalsIgnoreCase((i.next()).getName())) { + throw new IllegalArgumentException("Duplicate (case insensitively) " + field + " in " + this); + } + } + docType.addField(field); + } + + /** + * This is SD parse-time inheritance check. + * + * @param field The field being verified. + */ + private void verifyInheritance(Field field) { + for (SDDocumentType parent : inheritedTypes.values()) { + for (Field pField : parent.fieldSet()) { + if (pField.getName().equals(field.getName())) { + if (!pField.getDataType().equals(field.getDataType())) { + throw new IllegalArgumentException("For search '"+getName()+"', field '"+field.getName()+"': " + + "datatype can't be different from that of same field in supertype '"+parent.getName()+"'."); + } + } + } + } + } + + public SDField addField(String string, DataType dataType) { + SDField field = new SDField(this, string, dataType); + addField(field); + return field; + } + + public Field addField(String string, DataType dataType, boolean header, int code) { + SDField field = new SDField(this, string, code, dataType, header); + addField(field); + return field; + } + + private Map<String, Field> fieldsInherited() { + Map<String, Field> map = new LinkedHashMap<>(); + for (SDDocumentType parent : inheritedTypes.values()) { + for (Field field : parent.fieldSet()) { + map.put(field.getName(), field); + } + } + return map; + } + + public Set<Field> fieldSet() { + Map<String, Field> map = fieldsInherited(); + Iterator<Field> it = docType.fieldIteratorThisTypeOnly(); + while (it.hasNext()) { + Field field = it.next(); + map.put(field.getName(), field); + } + return new LinkedHashSet<>(map.values()); + } + + public Iterator<Field> fieldIterator() { + return fieldSet().iterator(); + } + + public int getFieldCount() { + return docType.getFieldCount(); + } + + public String toString() { + return "SD document type '" + docType.getName() + "'"; + } + + private static SDDocumentType createSDDocumentType(StructDataType structType) { + SDDocumentType docType = new SDDocumentType(structType.getName()); + for (Field field : structType.getFields()) { + docType.addField(new SDField(docType, field.getName(), field.getDataType())); + } + docType.setStruct(structType); + return docType; + } + + /** + * The field sets defined for this type and its {@link Search} + * @return fieldsets + */ + public FieldSets getFieldSets() { + return fieldSets; + } + + /** + * Sets the field sets for this + * @param fieldSets field sets to set for this object + */ + public void setFieldSets(FieldSets fieldSets) { + this.fieldSets = fieldSets; + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java new file mode 100644 index 00000000000..b379abd63ec --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -0,0 +1,770 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document; + +import com.yahoo.document.*; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.searchdefinition.Index; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.fieldoperation.FieldOperation; +import com.yahoo.searchdefinition.fieldoperation.FieldOperationContainer; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.indexinglanguage.ExpressionSearcher; +import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; +import com.yahoo.vespa.indexinglanguage.ScriptParserContext; +import com.yahoo.vespa.indexinglanguage.expressions.*; +import com.yahoo.vespa.indexinglanguage.parser.IndexingInput; +import com.yahoo.vespa.indexinglanguage.parser.ParseException; + +import java.io.Serializable; +import java.util.*; + +/** + * The field class represents a document field. It is used in + * the Document class to get and set fields. Each SDField has + * a name, a numeric ID, a data type, and a boolean that says whether it's + * a header field. The numeric ID is used when the fields are stored + * in serialized form. + * + * @author bratseth + */ +public class SDField extends Field implements TypedKey, FieldOperationContainer, Serializable { + + /** Use this field for modifying index-structure, even if it doesn't + have any indexing code */ + private boolean indexStructureField = false; + + /** The indexing statements to be applied to this value during indexing */ + private ScriptExpression indexingScript = new ScriptExpression(); + + /** The default rank type for indices of this field */ + private RankType rankType = RankType.DEFAULT; + + /** Rank settings in a "rank" block for the field. */ + private Ranking ranking = new Ranking(); + + /** + * The literal boost of this field. This boost is added to a rank score + * when a query term matched as query term exactly (unnormalized and unstemmed). + * Non-positive boosts causes no boosting, 0 allows boosts + * to be specified in other rank profiles, while negative values + * turns the capability off. + */ + private int literalBoost=-1; + + /** The weight of this field. This is a percentage, + * so 100 is default to provide the identity transform. */ + private int weight=100; + + /** + * Indicates what kind of matching should be done on this field + */ + private Matching matching=new Matching(); + + /** Attribute settings, or null if there are none */ + private Map<String, Attribute> attributes = new TreeMap<>(); + + /** + * The stemming setting of this field, or null to use the default. + * Default is determined by the owning search definition. + */ + private Stemming stemming=null; + + /** How content of this field should be accent normalized etc. */ + private NormalizeLevel normalizing = new NormalizeLevel(); + + /** Extra query commands of this field */ + private List<String> queryCommands=new java.util.ArrayList<>(0); + + /** Summary fields defined in this field */ + private Map<String,SummaryField> summaryFields = new java.util.LinkedHashMap<>(0); + + /** The explicitly index settings on this field */ + private Map<String, Index> indices=new java.util.LinkedHashMap<>(); + + /** True if body or header is set explicitly for this field */ + private boolean headerOrBodyDefined = false; + + private boolean idOverride = false; + + /** Struct fields defined in this field */ + private Map<String,SDField> structFields = new java.util.LinkedHashMap<>(0); + + /** The document that this field was declared in, or null*/ + protected SDDocumentType ownerDocType = null; + + /** The aliases declared for this field. May pertain to indexes or attributes */ + private Map<String, String> aliasToName = new HashMap<>(); + + /** Pending operations that must be applied after parsing, due to use of not-yet-defined structs. */ + private List<FieldOperation> pendingOperations = new LinkedList<>(); + + private boolean isExtraField = false; + + /** + Creates a new field. This method is only used to create reserved fields + @param name The name of the field + @param dataType The datatype of the field + @param isHeader Whether this is a "header" field or a "content" field + (true = "header"). + */ + protected SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean isHeader, boolean populate) { + super(name, id, dataType, isHeader); + populate(populate, repo, name, dataType, isHeader); + } + + public SDField(SDDocumentType repo, String name, int id, DataType dataType, boolean isHeader) { + this(repo, name, id, dataType, isHeader, true); + } + + /** + Creates a new field. + + @param name The name of the field + @param dataType The datatype of the field + @param isHeader Whether this is a "header" field or a "content" field + (true = "header"). + */ + public SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, boolean populate) { + super(name,dataType,isHeader); + populate(populate, repo, name, dataType, isHeader); + } + + private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, boolean isHeader) { + populate(populate,repo, name, dataType, isHeader, null, 0); + } + + private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType, boolean isHeader, Matching fieldMatching, int recursion) { + if (populate || (dataType instanceof MapDataType)) { + populateWithStructFields(repo, name, dataType, isHeader, recursion); + populateWithStructMatching(repo, name, dataType, fieldMatching); + } + } + + public SDField(String name, DataType dataType, boolean isHeader) { + this(null, name, dataType, isHeader, true); + } + + /** + Creates a new field. + + @param name The name of the field + @param dataType The datatype of the field + @param isHeader Whether this is a "header" field or a "content" field + (true = "header"). + @param owner the owning document (used to check for id collisions) + */ + protected SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, SDDocumentType owner, boolean populate) { + super(name, dataType, isHeader, owner == null ? null : owner.getDocumentType()); + this.ownerDocType=owner; + populate(populate, repo, name, dataType, isHeader); + } + + /** + Creates a new field. + + @param name The name of the field + @param dataType The datatype of the field + @param isHeader Whether this is a "header" field or a "content" field + (true = "header"). + @param owner The owning document (used to check for id collisions) + @param fieldMatching The matching object to set for the field + */ + protected SDField(SDDocumentType repo, String name, DataType dataType, boolean isHeader, SDDocumentType owner, Matching fieldMatching, boolean populate, int recursion) { + super(name, dataType, isHeader, owner == null ? null : owner.getDocumentType()); + this.ownerDocType=owner; + if (fieldMatching != null) { + this.setMatching(fieldMatching); + } + populate(populate, repo, name, dataType, isHeader, fieldMatching, recursion); + } + + /** + Constructor for <b>header</b> fields + + @param name The name of the field + @param dataType The datatype of the field + */ + public SDField(SDDocumentType repo, String name, DataType dataType) { + this(repo, name,dataType,true, true); + } + public SDField(String name, DataType dataType) { + this(null, name,dataType); + } + + public void setIsExtraField(boolean isExtra) { + isExtraField = isExtra; + } + + public boolean isExtraField() { + return isExtraField; + } + + public boolean doesAttributing() { + return containsExpression(AttributeExpression.class); + } + + public boolean doesIndexing() { + return containsExpression(IndexExpression.class); + } + + public boolean doesSummarying() { + if (usesStruct()) { + for (SDField structField : getStructFields()) { + if (structField.doesSummarying()) { + return true; + } + } + } + return containsExpression(SummaryExpression.class); + } + + public boolean doesLowerCasing() { + return containsExpression(LowerCaseExpression.class); + } + + public <T extends Expression> boolean containsExpression(Class<T> searchFor) { + return findExpression(searchFor) != null; + } + + public <T extends Expression> T findExpression(Class<T> searchFor) { + return new ExpressionSearcher<>(searchFor).searchIn(indexingScript); + } + + public void addSummaryFieldSources(SummaryField summaryField) { + if (usesStruct()) { + /* + * How this works for structs: When at least one sub-field in a struct is to + * be used for summary, that whole struct field is included in summary.cfg. Then, + * vsmsummary.cfg specifies the sub-fields used for each struct field. + * So we recurse into each struct, adding the destination classes set for each sub-field + * to the main summary-field for the struct field. + */ + for (SDField structField : getStructFields()) { + for (SummaryField sumF : structField.getSummaryFields()) { + for (String dest : sumF.getDestinations()) { + summaryField.addDestination(dest); + } + } + structField.addSummaryFieldSources(summaryField); + } + } else { + if (doesSummarying()) { + summaryField.addSource(getName()); + } + } + } + + public void populateWithStructFields(SDDocumentType sdoc, String name, DataType dataType, boolean isHeader, int recursion) { + DataType dt = getFirstStructOrMapRecursive(); + if (dt == null) { + return; + } + if (dataType instanceof MapDataType) { + MapDataType mdt = (MapDataType) dataType; + + SDField keyField = new SDField(sdoc, name.concat(".key"), mdt.getKeyType(), + isHeader, getOwnerDocType(), new Matching(), true, recursion + 1); + structFields.put("key", keyField); + + SDField valueField = new SDField(sdoc, name.concat(".value"), mdt.getValueType(), + isHeader, getOwnerDocType(), new Matching(), true, recursion + 1); + structFields.put("value", valueField); + } else { + if (recursion >= 10) { + return; + } + if (dataType instanceof CollectionDataType) { + dataType = ((CollectionDataType)dataType).getNestedType(); + } + SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null; + if (subType == null) { + throw new IllegalArgumentException("Could not find struct '" + dataType.getName() + "'."); + } + for (Field field : subType.fieldSet()) { + SDField subField = new SDField(sdoc, name.concat(".").concat(field.getName()), field.getDataType(), + isHeader, subType, new Matching(), true, recursion + 1); + structFields.put(field.getName(), subField); + } + } + } + + public void populateWithStructMatching(SDDocumentType sdoc, String name, DataType dataType, + Matching superFieldMatching) { + DataType dt = getFirstStructOrMapRecursive(); + if (dt != null) { + if (dataType instanceof MapDataType) { + MapDataType mdt = (MapDataType) dataType; + + Matching keyFieldMatching = new Matching(); + if (superFieldMatching != null) { + keyFieldMatching.merge(superFieldMatching); + } + SDField keyField = structFields.get(name.concat(".key")); + if (keyField != null) { + keyField.populateWithStructMatching(sdoc, name.concat(".key"), mdt.getKeyType(), keyFieldMatching); + keyField.setMatching(keyFieldMatching); + } + + Matching valueFieldMatching = new Matching(); + if (superFieldMatching != null) { + valueFieldMatching.merge(superFieldMatching); + } + SDField valueField = structFields.get(name.concat(".value")); + if (valueField != null) { + valueField.populateWithStructMatching(sdoc, name.concat(".value"), mdt.getValueType(), + valueFieldMatching); + valueField.setMatching(valueFieldMatching); + } + + } else { + + if (dataType instanceof CollectionDataType) { + dataType = ((CollectionDataType)dataType).getNestedType(); + } + SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null; + if (subType != null) { + for (Field f : subType.fieldSet()) { + if (f instanceof SDField) { + SDField field = (SDField)f; + Matching subFieldMatching = new Matching(); + if (superFieldMatching != null) { + subFieldMatching.merge(superFieldMatching); + } + subFieldMatching.merge(field.getMatching()); + SDField subField = structFields.get(field.getName()); + if (subField != null) { + subField.populateWithStructMatching(sdoc, name.concat(".").concat(field.getName()), field.getDataType(), + subFieldMatching); + subField.setMatching(subFieldMatching); + } + } else { + throw new IllegalArgumentException("Field in struct is not SDField " + f.getName()); + } + } + } else { + throw new IllegalArgumentException("Could not find struct " + dataType.getName()); + } + } + } + } + + public void addOperation(FieldOperation op) { + pendingOperations.add(op); + } + + public void applyOperations(SDField field) { + if (pendingOperations.isEmpty()) { + return; + } + ListIterator<FieldOperation> ops = pendingOperations.listIterator(); + while (ops.hasNext()) { + FieldOperation op = ops.next(); + ops.remove(); + op.apply(field); + } + } + + public void applyOperations() { + applyOperations(this); + } + + public void setId(int fieldId, DocumentType owner) { + super.setId(fieldId, owner); + idOverride = true; + } + + public StructDataType getFirstStructRecursive() { + DataType dataType = getDataType(); + while (true) { // Currently no nesting of collections + if (dataType instanceof CollectionDataType) { + dataType = ((CollectionDataType)dataType).getNestedType(); + } else if (dataType instanceof MapDataType) { + dataType = ((MapDataType)dataType).getValueType(); + } else { + break; + } + } + return (dataType instanceof StructDataType) ? (StructDataType)dataType : null; + } + + public DataType getFirstStructOrMapRecursive() { + DataType dataType = getDataType(); + while (dataType instanceof CollectionDataType) { // Currently no nesting of collections + dataType = ((CollectionDataType)dataType).getNestedType(); + } + return (dataType instanceof StructDataType || dataType instanceof MapDataType) ? dataType : null; + } + + public boolean usesStruct() { + DataType dt = getFirstStructRecursive(); + return (dt != null); + } + + public boolean usesStructOrMap() { + DataType dt = getFirstStructOrMapRecursive(); + return (dt != null); + } + + /** Parse an indexing expression which will use the simple linguistics implementatino suitable for testing */ + public void parseIndexingScript(String script) { + parseIndexingScript(script, new SimpleLinguistics()); + } + + public void parseIndexingScript(String script, Linguistics linguistics) { + try { + ScriptParserContext config = new ScriptParserContext(linguistics); + config.setInputStream(new IndexingInput(script)); + setIndexingScript(ScriptExpression.newInstance(config)); + } catch (ParseException e) { + throw new RuntimeException("Failed to parser script '" + script + "'.", e); + } + } + + /** Sets the indexing script of this, or null to not use a script */ + public void setIndexingScript(ScriptExpression exp) { + if (exp == null) { + exp = new ScriptExpression(); + } + indexingScript = exp; + if (indexingScript.isEmpty()) { + return; // TODO: This causes empty expressions not to be propagate to struct fields!! BAD BAD BAD!! + } + if (!usesStructOrMap()) { + new ExpressionVisitor() { + + @Override + protected void doVisit(Expression exp) { + if (!(exp instanceof AttributeExpression)) { + return; + } + String fieldName = ((AttributeExpression)exp).getFieldName(); + if (fieldName == null) { + fieldName = getName(); + } + Attribute attribute = attributes.get(fieldName); + if (attribute == null) { + addAttribute(new Attribute(fieldName, getDataType())); + } + } + }.visit(indexingScript); + } + for (SDField structField : getStructFields()) { + structField.setIndexingScript(exp); + } + } + + public ScriptExpression getIndexingScript() { + return indexingScript; + } + + @SuppressWarnings("deprecation") + @Override + public void setDataType(DataType type) { + if (type.equals(DataType.URI)) { // Different defaults, naturally + normalizing.inferLowercase(); + stemming=Stemming.NONE; + } + this.dataType = type; + if (!idOverride) { + this.fieldId = calculateIdV7(null); + } + } + + public boolean isIndexStructureField() { + return indexStructureField; + } + + public void setIndexStructureField(boolean indexStructureField) { + this.indexStructureField = indexStructureField; + } + + /** + * Returns an iterator of the index names this should index to + * (whether set explicitly or not) + */ + public Iterator<String> getFieldNameAsIterator() { // TODO: Replace usage by getName + return Collections.singletonList(getName()).iterator(); + } + + /** Returns 1 if this is indexed, 0 if it is not indexed */ // TODO: Replace by a boolean method, or something, see hasIndex + public int getIndexToCount() { + if (getIndexingScript() == null) return 0; + if (!doesIndexing()) return 0; + + return 1; + } + + /** Sets the literal boost of this field */ + public void setLiteralBoost(int literalBoost) { this.literalBoost=literalBoost; } + + /** + * Returns the literal boost of this field. This boost is added to a literal score + * when a query term matched as query term exactly (unnormalized and unstemmed). + * Default is non-positive. + */ + public int getLiteralBoost() { return literalBoost; } + + /** Sets the weight of this field */ + public void setWeight(int weight) { this.weight=weight; } + + /** Returns the weight of this field, or 0 if nothing is set */ + public int getWeight() { return weight; } + + /** + * Returns what kind of matching type should be applied. + */ + public Matching getMatching() { return matching; } + + /** + * Sets what kind of matching type should be applied. + * (Token matching is default, PREFIX, SUBSTRING, SUFFIX are alternatives) + */ + public void setMatching(Matching matching) { this.matching=matching; } + + /** + * Set the matching type for this field and all subfields. + */ + // TODO: When this is not the same as getMatching().setthis we have a potential for inconsistency. Find the right + // Matching object for struct fields as lookup time instead. + public void setMatchingType(Matching.Type type) { + this.getMatching().setType(type); + for (SDField structField : getStructFields()) { + structField.setMatchingType(type); + } + } + + /** + * Set matching algorithm for this field and all subfields. + */ + // TODO: When this is not the same as getMatching().setthis we have a potential for inconsistency. Find the right + // Matching object for struct fields as lookup time instead. + public void setMatchingAlgorithm(Matching.Algorithm algorithm) { + this.getMatching().setAlgorithm(algorithm); + for (SDField structField : getStructFields()) { + structField.getMatching().setAlgorithm(algorithm); + } + } + + /** Adds an explicit index defined in this field */ + public void addIndex(Index index) { + indices.put(index.getName(),index); + } + + /** + * Returns an index, or null if no index with this name has had + * some <b>explicit settings</b> applied in this field (even if this returns null, + * the index may be implicitly defined by an indexing statement) + */ + public Index getIndex(String name) { + return indices.get(name); + } + + /** + * Returns an index if this field has one (implicitly or + * explicitly) targeting the given name. + */ + public boolean existsIndex(String name) { + if (indices.get(name) != null) return true; + if (name.equals(getName())) { + if (doesIndexing()) { + return true; + } + } + return false; + } + + /** + * Defined indices on this field + * @return defined indices on this + */ + public Map<String, Index> getIndices() { + return indices; + } + + /** + * Sets the default rank type of this fields indices, and sets this rank type + * to all indices explicitly defined here which has no index set. + * (This complex behavior is dues to the fact than we would prefer to have rank types + * per field, not per index) + */ + public void setRankType(RankType rankType) { + this.rankType=rankType; + for (Index index : getIndices().values()) { + if (index.getRankType()==null) + index.setRankType(rankType); + } + + } + + /** Returns the rank settings set in a "rank" block for this field. This is never null. */ + public Ranking getRanking() { return ranking; } + + /** Returns the default rank type of indices of this field, or null if nothing is set */ + public RankType getRankType() { return this.rankType; } + + /** + * <p>Returns the search-time attribute settings of this field + * or null if none is set.</p> + * + * <p>TODO: Make unmodifiable.</p> + */ + public Map<String, Attribute> getAttributes() { return attributes; } + + public void addAttribute(Attribute attribute) { + String name = attribute.getName(); + if (name == null || "".equals(name)) { + name = getName(); + attribute.setName(name); + } + attributes.put(attribute.getName(),attribute); + } + + /** + * Returns the stemming setting of this field. + * Default is determined by the owning search definition. + * + * @return the stemming setting of this, or null, to use the default + */ + public Stemming getStemming() { return stemming; } + + /** + * Whether this field should be stemmed in this search definition + */ + public Stemming getStemming(Search search) { + if (stemming!=null) + return stemming; + else + return search.getStemming(); + } + + /** + * Sets how this field should be stemmed, or set to null to use the default. + */ + public void setStemming(Stemming stemming) { + this.stemming=stemming; + } + + /** + * List of static summary fields + * @return list of static summary fields + */ + public Collection<SummaryField> getSummaryFields() { return summaryFields.values(); } + + /** + * Add summary field + */ + public void addSummaryField(SummaryField summaryField) { + summaryFields.put(summaryField.getName(),summaryField); + } + + /** + * Returns a summary field defined (implicitly or explicitly) by this field. + * Returns null if there is no such summary field defined. + */ + public SummaryField getSummaryField(String name) { + return summaryFields.get(name); + } + + /** + * Returns a summary field defined (implicitly or explicitly) by this field. + * + * @param create true to create the summary field and add it to this field before returning if it is missing + * @return the summary field, or null if not present and create is false + */ + public SummaryField getSummaryField(String name,boolean create) { + SummaryField summaryField=summaryFields.get(name); + if (summaryField==null && create) { + summaryField=new SummaryField(name, getDataType()); + addSummaryField(summaryField); + } + return summaryFields.get(name); + } + + /** Returns list of static struct fields */ + public Collection<SDField> getStructFields() { return structFields.values(); } + + /** + * Returns a struct field defined in this field, + * potentially traversing into nested structs. + * Returns null if there is no such struct field defined. + */ + public SDField getStructField(String name) { + if (name.contains(".")) { + String superFieldName = name.substring(0,name.indexOf(".")); + String subFieldName = name.substring(name.indexOf(".")+1); + SDField superField = structFields.get(superFieldName); + if (superField != null) { + return superField.getStructField(subFieldName); + } + return null; + } + return structFields.get(name); + } + + /** + * Returns how the content of this field should be accent normalized etc + */ + public NormalizeLevel getNormalizing() { return normalizing; } + + /** + * Change how the content of this field should be accent normalized etc + */ + public void setNormalizing(NormalizeLevel level) { normalizing = level; } + + public void addQueryCommand(String name) { + queryCommands.add(name); + } + + public boolean hasQueryCommand(String name) { + return queryCommands.contains(name); + } + + /** + * A list of query commands + * @return a list of strings with query commands. + */ + public List<String> getQueryCommands() { + return queryCommands; + } + + + public boolean isHeaderOrBodyDefined() { + return headerOrBodyDefined; + } + + public void setHeaderOrBodyDefined(boolean headerOrBodySetExplicitly) { + this.headerOrBodyDefined = headerOrBodySetExplicitly; + } + + /** + * The document that this field was declared in, or null + * + */ + public SDDocumentType getOwnerDocType() { + return ownerDocType; + } + + /** + * Two fields are equal if they have the same name + * No, they are not. + */ + public boolean equals(Object other) { + if ( ! (other instanceof SDField)) return false; + return super.equals(other); + } + + public int hashCode() { + return getName().hashCode(); + } + + public String toString() { + return "field '" + getName() + "'"; + } + + /** The aliases declared for this field */ + public Map<String, String> getAliasToName() { + return aliasToName; + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Sorting.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Sorting.java new file mode 100644 index 00000000000..a0511dec4b0 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Sorting.java @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document; + +import java.io.Serializable; + +/** + * A search-time document attribute sort specification(per-document in-memory value). + * This belongs to the attribute or field(implicitt attribute). + * + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public final class Sorting implements Cloneable, Serializable { + + // Remember to change hashCode and equals when you add new fields + public enum Function {UCA, RAW, LOWERCASE} + public enum Strength {PRIMARY, SECONDARY, TERTIARY, QUATERNARY, IDENTICAL} + private boolean ascending = true; + private Function function = Function.UCA; + private String locale = ""; + private Strength strength = Strength.PRIMARY; + + public boolean isAscending() { return ascending; } + public boolean isDescending() { return ! ascending; } + public String getLocale() { return locale; } + public Function getFunction() { return function; } + public Strength getStrength() { return strength; } + + public void setAscending() { ascending = true; } + public void setDescending() { ascending = false; } + public void setFunction(Function function) { this.function = function; } + public void setLocale(String locale) { this.locale = locale; } + public void setStrength(Strength strength) { this.strength = strength; } + + public int hashCode() { + return locale.hashCode() + + strength.hashCode() + + function.hashCode() + + (isDescending() ? 13 : 0); + } + + public boolean equals(Object object) { + if (! (object instanceof Sorting)) return false; + + Sorting other=(Sorting)object; + return this.locale.equals(other.locale) && + (ascending == other.ascending) && + (function == other.function) && + (strength == other.strength); + } + + public @Override Sorting clone() { + try { + return (Sorting)super.clone(); + } + catch (CloneNotSupportedException e) { + throw new RuntimeException("Programming error"); + } + } + + public String toString() { + return "sorting '" + (isAscending() ? '+' : '-') + function.toString() + "(" + strength.toString() + ", " + locale + ")"; + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java new file mode 100644 index 00000000000..f471201f55e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document; + +import com.yahoo.language.process.StemMode; + +import java.util.logging.Logger; + +/** + * <p>The stemming setting of a field. This describes how the search engine + * should transform content of this field into base forms (stems) to increase + * recall (find "car" when you search for "cars" etc.).</p> + * + * @author bratseth + */ +public enum Stemming { + + /** No stemming */ + NONE("none"), + + /** Stem as much as possible */ + ALL("all"), + + /** select shortest possible stem */ + SHORTEST("shortest"), + + /** index (and query?) multiple stems */ + MULTIPLE("multiple"); + + private static Logger log=Logger.getLogger(Stemming.class.getName()); + + private final String name; + + /** + * Returns the stemming object for the given string. + * The legal stemming names are the stemming constants in any capitalization. + * + * @throws IllegalArgumentException if there is no stemming type with the given name + */ + public static Stemming get(String stemmingName) { + try { + Stemming stemming = Stemming.valueOf(stemmingName.toUpperCase()); + if (stemming.equals(ALL)) { + log.warning("note: stemming ALL is the same as stemming mode SHORTEST"); + stemming = SHORTEST; + } + return stemming; + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("'" + stemmingName + "' is not a valid stemming setting"); + } + } + + private Stemming(String name) { + this.name = name; + } + + public String getName() { return name; } + + public String toString() { + return "stemming " + name; + } + + public StemMode toStemMode() { + if (this == Stemming.SHORTEST) { + return StemMode.SHORTEST; + } + if (this == Stemming.MULTIPLE) { + return StemMode.ALL; + } + return StemMode.NONE; + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDDocumentType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDDocumentType.java new file mode 100644 index 00000000000..193ec75acd0 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDDocumentType.java @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document; + +import com.yahoo.document.DataTypeName; + +/** + * @author balder + */ +public class TemporarySDDocumentType extends SDDocumentType { + public TemporarySDDocumentType(DataTypeName name) { + super(name); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java new file mode 100644 index 00000000000..00012d65434 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document; + +import com.yahoo.document.DataType; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class TemporarySDField extends SDField { + + public TemporarySDField(String name, DataType dataType, boolean isHeader, SDDocumentType owner) { + super(owner, name, dataType, isHeader, owner, false); + } + + public TemporarySDField(String name, DataType dataType) { + super(null, name, dataType, true, false); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TypedKey.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/TypedKey.java new file mode 100644 index 00000000000..6a10ebe642d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/TypedKey.java @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document; + +import com.yahoo.document.DataType; + +/** + * Common interface for various typed key (or field definitions). + * Used by code which wants to use common algorithms for dealing with typed keys, like the logical mapping + * + * @author bratseth + */ +public interface TypedKey { + + String getName(); + + void setDataType(DataType type); + + DataType getDataType(); + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/SDAnnotationType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/SDAnnotationType.java new file mode 100644 index 00000000000..0f8e200c4c6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/SDAnnotationType.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document.annotation; + +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.document.annotation.AnnotationType; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class SDAnnotationType extends AnnotationType { + private SDDocumentType sdDocType; + private String inherits; + + public SDAnnotationType(String name) { + super(name); + } + + public SDAnnotationType(String name, SDDocumentType dataType, String inherits) { + super(name); + this.sdDocType = dataType; + this.inherits = inherits; + } + + public SDDocumentType getSdDocType() { + return sdDocType; + } + + public String getInherits() { + return inherits; + } + + public void inherit(String inherits) { + this.inherits = inherits; + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/TemporaryAnnotationReferenceDataType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/TemporaryAnnotationReferenceDataType.java new file mode 100644 index 00000000000..5b82b068908 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/TemporaryAnnotationReferenceDataType.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document.annotation; + +import com.yahoo.document.annotation.AnnotationReferenceDataType; +import com.yahoo.document.annotation.AnnotationType; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class TemporaryAnnotationReferenceDataType extends AnnotationReferenceDataType { + private final String target; + + public TemporaryAnnotationReferenceDataType(String target) { + this.target = target; + } + + public String getTarget() { + return target; + } + + @Override + public void setAnnotationType(AnnotationType type) { + super.setName("annotationreference<" + type.getName() + ">"); + super.setAnnotationType(type); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AliasOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AliasOperation.java new file mode 100644 index 00000000000..60aeb3d50cc --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AliasOperation.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.searchdefinition.document.SDField; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class AliasOperation implements FieldOperation { + private String aliasedName; + private String alias; + + public AliasOperation(String aliasedName, String alias) { + this.aliasedName = aliasedName; + this.alias = alias; + } + + public String getAliasedName() { + return aliasedName; + } + + public void setAliasedName(String aliasedName) { + this.aliasedName = aliasedName; + } + + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } + + public void apply(SDField field) { + if (aliasedName == null) { + aliasedName = field.getName(); + } + field.getAliasToName().put(alias, aliasedName); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java new file mode 100644 index 00000000000..19ee857c627 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java @@ -0,0 +1,153 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.tensor.TensorType; + +import java.util.Optional; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class AttributeOperation implements FieldOperation, FieldOperationContainer { + private final String name; + private Boolean huge; + private Boolean fastSearch; + private Boolean fastAccess; + private Boolean prefetch; + private Boolean enableBitVectors; + private Boolean enableOnlyBitVector; + //TODO: Husk sorting!! + private boolean doAlias = false; + private String alias; + private String aliasedName; + private Optional<TensorType> tensorType = Optional.empty(); + + public AttributeOperation(String name) { + this.name = name; + } + + public void addOperation(FieldOperation op) { + //TODO: Implement this method: + + } + + public void applyOperations(SDField field) { + //TODO: Implement this method: + + } + + public String getName() { + return name; + } + + public Boolean getHuge() { + return huge; + } + + public void setHuge(Boolean huge) { + this.huge = huge; + } + + public Boolean getFastSearch() { + return fastSearch; + } + + public void setFastSearch(Boolean fastSearch) { + this.fastSearch = fastSearch; + } + + public Boolean getFastAccess() { + return fastAccess; + } + + public void setFastAccess(Boolean fastAccess) { + this.fastAccess = fastAccess; + } + + public Boolean getPrefetch() { + return prefetch; + } + + public void setPrefetch(Boolean prefetch) { + this.prefetch = prefetch; + } + + public Boolean getEnableBitVectors() { + return enableBitVectors; + } + + public void setEnableBitVectors(Boolean enableBitVectors) { + this.enableBitVectors = enableBitVectors; + } + + public Boolean getEnableOnlyBitVector() { + return enableOnlyBitVector; + } + + public void setEnableOnlyBitVector(Boolean enableOnlyBitVector) { + this.enableOnlyBitVector = enableOnlyBitVector; + } + + public boolean isDoAlias() { + return doAlias; + } + + public void setDoAlias(boolean doAlias) { + this.doAlias = doAlias; + } + + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } + + public String getAliasedName() { + return aliasedName; + } + + public void setAliasedName(String aliasedName) { + this.aliasedName = aliasedName; + } + + public void setTensorType(TensorType tensorType) { + this.tensorType = Optional.of(tensorType); + } + + public void apply(SDField field) { + Attribute attribute = field.getAttributes().get(name); + if (attribute == null) { + attribute = new Attribute(name, field.getDataType()); + field.addAttribute(attribute); + } + + if (huge != null) { + attribute.setHuge(huge); + } + if (fastSearch != null) { + attribute.setFastSearch(fastSearch); + } + if (fastAccess != null) { + attribute.setFastAccess(fastAccess); + } + if (prefetch != null) { + attribute.setPrefetch(prefetch); + } + if (enableBitVectors != null) { + attribute.setEnableBitVectors(enableBitVectors); + } + if (enableOnlyBitVector != null) { + attribute.setEnableOnlyBitVector(enableOnlyBitVector); + } + if (doAlias) { + field.getAliasToName().put(alias, aliasedName); + } + if (tensorType.isPresent()) { + attribute.setTensorType(tensorType.get()); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BodyOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BodyOperation.java new file mode 100644 index 00000000000..b830407ea2e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BodyOperation.java @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.searchdefinition.document.SDField; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class BodyOperation implements FieldOperation { + public void apply(SDField field) { + field.setHeader(false); + field.setHeaderOrBodyDefined(true); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BoldingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BoldingOperation.java new file mode 100644 index 00000000000..f6c7277869e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BoldingOperation.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.documentmodel.SummaryField; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class BoldingOperation implements FieldOperation { + + private final boolean bold; + + public BoldingOperation(boolean bold) { + this.bold = bold; + } + + public void apply(SDField field) { + SummaryField summaryField = field.getSummaryField(field.getName(), true); + summaryField.addSource(field.getName()); + summaryField.addDestination("default"); + summaryField.setTransform(bold ? summaryField.getTransform().bold() : summaryField.getTransform().unbold()); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java new file mode 100644 index 00000000000..20caf0b13c7 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.searchdefinition.document.SDField; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public interface FieldOperation { + public void apply(SDField field); +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java new file mode 100644 index 00000000000..30d16e369b1 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.searchdefinition.document.SDField; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public interface FieldOperationContainer { + public void addOperation(FieldOperation op); + public void applyOperations(SDField field); + public String getName(); +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/HeaderOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/HeaderOperation.java new file mode 100644 index 00000000000..48d2711b39e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/HeaderOperation.java @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.searchdefinition.document.SDField; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class HeaderOperation implements FieldOperation { + public void apply(SDField field) { + field.setHeader(true); + field.setHeaderOrBodyDefined(true); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IdOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IdOperation.java new file mode 100644 index 00000000000..25318e2db85 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IdOperation.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class IdOperation implements FieldOperation { + private SDDocumentType document; + private int fieldId; + + public SDDocumentType getDocument() { + return document; + } + + public void setDocument(SDDocumentType document) { + this.document = document; + } + + public int getFieldId() { + return fieldId; + } + + public void setFieldId(int fieldId) { + this.fieldId = fieldId; + } + + public void apply(SDField field) { + document.setFieldId(field, fieldId); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java new file mode 100644 index 00000000000..8ab065ce419 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java @@ -0,0 +1,114 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.searchdefinition.Index; +import com.yahoo.searchdefinition.Index.Type; +import com.yahoo.searchdefinition.document.BooleanIndexDefinition; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.Stemming; + +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; +import java.util.OptionalDouble; +import java.util.OptionalInt; +import java.util.OptionalLong; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class IndexOperation implements FieldOperation { + private String indexName; + private Optional<Boolean> prefix = Optional.empty(); + private List<String> aliases = new LinkedList<>(); + private Optional<String> stemming = Optional.empty(); + private Optional<Type> type = Optional.empty(); + + private OptionalInt arity = OptionalInt.empty(); // For predicate data type in boolean search + private OptionalLong lowerBound = OptionalLong.empty(); + private OptionalLong upperBound = OptionalLong.empty(); + private OptionalDouble densePostingListThreshold = OptionalDouble.empty(); + + public String getIndexName() { + return indexName; + } + + public void setIndexName(String indexName) { + this.indexName = indexName; + } + + public boolean getPrefix() { + return prefix.get(); + } + + public void setPrefix(Boolean prefix) { + this.prefix = Optional.of(prefix); + } + + public void addAlias(String alias) { + aliases.add(alias); + } + + public String getStemming() { + return stemming.get(); + } + + public void setStemming(String stemming) { + this.stemming = Optional.of(stemming); + } + + public void apply(SDField field) { + Index index = field.getIndex(indexName); + + if (index == null) { + index = new Index(indexName); + field.addIndex(index); + } + + applyToIndex(index); + } + + public void applyToIndex(Index index) { + if (prefix.isPresent()) { + index.setPrefix(prefix.get()); + } + for (String alias : aliases) { + index.addAlias(alias); + } + if (stemming.isPresent()) { + index.setStemming(Stemming.get(stemming.get())); + } + if (type.isPresent()) { + index.setType(type.get()); + } + if (arity.isPresent() || lowerBound.isPresent() || + upperBound.isPresent() || densePostingListThreshold.isPresent()) { + index.setBooleanIndexDefiniton( + new BooleanIndexDefinition(arity, lowerBound, upperBound, densePostingListThreshold)); + } + } + + public Type getType() { + return type.get(); + } + + public void setType(Type type) { + this.type = Optional.of(type); + } + + public void setArity(int arity) { + this.arity = OptionalInt.of(arity); + } + + public void setLowerBound(long value) { + this.lowerBound = OptionalLong.of(value); + } + + public void setUpperBound(long value) { + this.upperBound = OptionalLong.of(value); + } + + public void setDensePostingListThreshold(double densePostingListThreshold) { + this.densePostingListThreshold = OptionalDouble.of(densePostingListThreshold); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java new file mode 100644 index 00000000000..af3852bffb8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.searchdefinition.parser.SimpleCharStream; +import com.yahoo.vespa.indexinglanguage.ScriptParserContext; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; +import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class IndexingOperation implements FieldOperation { + + private final ScriptExpression script; + + public IndexingOperation(ScriptExpression script) { + this.script = script; + } + + public void apply(SDField field) { + field.setIndexingScript(script); + } + + /** Creates an indexing operation which will use the simple linguistics implementation suitable for testing */ + public static IndexingOperation fromStream(SimpleCharStream input, boolean multiLine) throws ParseException { + return fromStream(input, multiLine, new SimpleLinguistics()); + } + + public static IndexingOperation fromStream(SimpleCharStream input, boolean multiLine, Linguistics linguistics) + throws ParseException { + ScriptParserContext config = new ScriptParserContext(linguistics); + config.setAnnotatorConfig(new AnnotatorConfig()); + config.setInputStream(input); + ScriptExpression exp; + try { + if (multiLine) { + exp = ScriptExpression.newInstance(config); + } else { + exp = new ScriptExpression(StatementExpression.newInstance(config)); + } + } catch (com.yahoo.vespa.indexinglanguage.parser.ParseException e) { + ParseException t = new ParseException("Error reported by IL parser: " + e.getMessage()); + t.initCause(e); + throw t; + } + return new IndexingOperation(exp); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java new file mode 100644 index 00000000000..a416895a6a7 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingRewriteOperation.java @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.searchdefinition.document.SDField; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class IndexingRewriteOperation implements FieldOperation { + public void apply(SDField field) { + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/MatchOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/MatchOperation.java new file mode 100644 index 00000000000..9128778b7ea --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/MatchOperation.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.SDField; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class MatchOperation implements FieldOperation { + private Matching.Type matchingType; + private Integer gramSize; + private Integer maxLength; + private Matching.Algorithm matchingAlgorithm; + private String exactMatchTerminator; + + public void setMatchingType(Matching.Type matchingType) { + this.matchingType = matchingType; + } + + public void setGramSize(Integer gramSize) { + this.gramSize = gramSize; + } + public void setMaxLength(Integer maxLength) { + this.maxLength = maxLength; + } + + public void setMatchingAlgorithm(Matching.Algorithm matchingAlgorithm) { + this.matchingAlgorithm = matchingAlgorithm; + } + + public void setExactMatchTerminator(String exactMatchTerminator) { + this.exactMatchTerminator = exactMatchTerminator; + } + + public void apply(SDField field) { + if (matchingType != null) { + field.setMatchingType(matchingType); + } + if (gramSize != null) { + field.getMatching().setGramSize(gramSize); + } + if (maxLength != null) { + field.getMatching().maxLength(maxLength); + } + if (matchingAlgorithm != null) { + field.setMatchingAlgorithm(matchingAlgorithm); + } + if (exactMatchTerminator != null) { + field.getMatching().setExactMatchTerminator(exactMatchTerminator); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/NormalizingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/NormalizingOperation.java new file mode 100644 index 00000000000..e0d15cabbe0 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/NormalizingOperation.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.searchdefinition.document.NormalizeLevel; +import com.yahoo.searchdefinition.document.SDField; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class NormalizingOperation implements FieldOperation { + private NormalizeLevel.Level level; + + public NormalizingOperation(String setting) { + if ("none".equals(setting)) { + this.level = NormalizeLevel.Level.NONE; + } else if ("codepoint".equals(setting)) { + this.level = NormalizeLevel.Level.CODEPOINT; + } else if ("lowercase".equals(setting)) { + this.level = NormalizeLevel.Level.LOWERCASE; + } else if ("accent".equals(setting)) { + this.level = NormalizeLevel.Level.ACCENT; + } else if ("all".equals(setting)) { + this.level = NormalizeLevel.Level.ACCENT; + } else { + throw new IllegalArgumentException("invalid normalizing setting: "+setting); + } + } + + public void apply(SDField field) { + field.setNormalizing(new NormalizeLevel(level, true)); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/QueryCommandOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/QueryCommandOperation.java new file mode 100644 index 00000000000..e4af5660e7b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/QueryCommandOperation.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.searchdefinition.document.SDField; + +import java.util.List; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class QueryCommandOperation implements FieldOperation { + private List<String> queryCommands = new java.util.ArrayList<>(0); + + public void addQueryCommand(String name) { + queryCommands.add(name); + } + + public void apply(SDField field) { + for (String command : queryCommands) { + field.addQueryCommand(command); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankOperation.java new file mode 100644 index 00000000000..42b1e5514a1 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankOperation.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.searchdefinition.document.SDField; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class RankOperation implements FieldOperation { + + private Boolean literal = null; + private Boolean filter = null; + private Boolean normal = null; + + public Boolean getLiteral() { return literal; } + public void setLiteral(Boolean literal) { this.literal = literal; } + + public Boolean getFilter() { return filter; } + public void setFilter(Boolean filter) { this.filter = filter; } + + public Boolean getNormal() { return normal; } + public void setNormal(Boolean n) { this.normal = n; } + + public void apply(SDField field) { + if (literal != null) { + field.getRanking().setLiteral(literal); + } + if (filter != null) { + field.getRanking().setFilter(filter); + } + if (normal != null) { + field.getRanking().setNormal(normal); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankTypeOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankTypeOperation.java new file mode 100644 index 00000000000..640ccc48818 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankTypeOperation.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.searchdefinition.document.RankType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.Index; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class RankTypeOperation implements FieldOperation { + private String indexName; + private RankType type; + + public String getIndexName() { + return indexName; + } + public void setIndexName(String indexName) { + this.indexName = indexName; + } + + public RankType getType() { + return type; + } + public void setType(RankType type) { + this.type = type; + } + + public void apply(SDField field) { + if (indexName == null) { + field.setRankType(type); // Set default if the index is not specified. + } else { + Index index = field.getIndex(indexName); + if (index == null) { + index = new Index(indexName); + field.addIndex(index); + } + index.setRankType(type); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SortingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SortingOperation.java new file mode 100644 index 00000000000..e9225f0def5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SortingOperation.java @@ -0,0 +1,91 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.Sorting; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class SortingOperation implements FieldOperation { + private String attributeName; + private Boolean ascending; + private Boolean descending; + private Sorting.Function function; + private Sorting.Strength strength; + private String locale; + + public SortingOperation(String attributeName) { + this.attributeName = attributeName; + } + + public String getAttributeName() { + return attributeName; + } + + public Boolean getAscending() { + return ascending; + } + + public void setAscending() { + this.ascending = true; + } + + public Boolean getDescending() { + return descending; + } + + public void setDescending() { + this.descending = true; + } + + public Sorting.Function getFunction() { + return function; + } + + public void setFunction(Sorting.Function function) { + this.function = function; + } + + public Sorting.Strength getStrength() { + return strength; + } + + public void setStrength(Sorting.Strength strength) { + this.strength = strength; + } + + public String getLocale() { + return locale; + } + + public void setLocale(String locale) { + this.locale = locale; + } + + public void apply(SDField field) { + Attribute attribute = field.getAttributes().get(attributeName); + if (attribute == null) { + attribute = new Attribute(attributeName, field.getDataType()); + field.addAttribute(attribute); + } + Sorting sorting = attribute.getSorting(); + + if (ascending != null) { + sorting.setAscending(); + } + if (descending != null) { + sorting.setDescending(); + } + if (function != null) { + sorting.setFunction(function); + } + if (strength != null) { + sorting.setStrength(strength); + } + if (locale != null) { + sorting.setLocale(locale); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StemmingOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StemmingOperation.java new file mode 100644 index 00000000000..19b760386a1 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StemmingOperation.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.Stemming; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class StemmingOperation implements FieldOperation { + private String setting; + + public String getSetting() { + return setting; + } + + public void setSetting(String setting) { + this.setting = setting; + } + + public void apply(SDField field) { + field.setStemming(Stemming.get(setting)); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StructFieldOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StructFieldOperation.java new file mode 100644 index 00000000000..f0c3d964934 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StructFieldOperation.java @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.searchdefinition.document.SDField; + +import java.util.LinkedList; +import java.util.List; +import java.util.ListIterator; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class StructFieldOperation implements FieldOperation, FieldOperationContainer { + private String structFieldName; + private List<FieldOperation> pendingOperations = new LinkedList<>(); + + public StructFieldOperation(String structFieldName) { + this.structFieldName = structFieldName; + } + + public void apply(SDField field) { + SDField structField = field.getStructField(structFieldName); + if (structField == null ) { + throw new IllegalArgumentException("Struct field '" + structFieldName + "' has not been defined in struct " + + "for field '" + field.getName() + "'."); + } + + applyOperations(structField); + } + + public void addOperation(FieldOperation op) { + pendingOperations.add(op); + } + + public void applyOperations(SDField field) { + if (pendingOperations.isEmpty()) { + return; + } + ListIterator<FieldOperation> ops = pendingOperations.listIterator(); + while (ops.hasNext()) { + FieldOperation op = ops.next(); + ops.remove(); + op.apply(field); + } + } + + public String getName() { + return structFieldName; + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldLongOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldLongOperation.java new file mode 100644 index 00000000000..f320ef649b4 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldLongOperation.java @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.document.DataType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.documentmodel.SummaryField; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class SummaryInFieldLongOperation extends SummaryInFieldOperation { + private DataType type; + private Boolean bold; + private Set<String> destinations = new java.util.LinkedHashSet<>(); + private List<SummaryField.Property> properties = new ArrayList<>(); + + public SummaryInFieldLongOperation(String name) { + super(name); + } + + public SummaryInFieldLongOperation() { + super(null); + } + + public void setType(DataType type) { + this.type = type; + } + + public void setBold(Boolean bold) { + this.bold = bold; + } + + public void addDestination(String destination) { + destinations.add(destination); + } + + public Iterator<String> destinationIterator() { + return destinations.iterator(); + } + + + public void addProperty(SummaryField.Property property) { + properties.add(property); + } + + public void apply(SDField field) { + if (type == null) { + type = field.getDataType(); + } + SummaryField summary = new SummaryField(name, type); + applyToSummary(summary); + field.addSummaryField(summary); + } + + public void applyToSummary(SummaryField summary) { + if (transform != null) { + summary.setTransform(transform); + } + + if (bold != null) { + summary.setTransform(bold ? summary.getTransform().bold() : summary.getTransform().unbold()); + } + + for (SummaryField.Source source : sources) { + summary.addSource(source); + } + + for (String destination : destinations) { + summary.addDestination(destination); + } + + for (SummaryField.Property prop : properties) { + summary.addProperty(prop.getName(), prop.getValue()); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldOperation.java new file mode 100644 index 00000000000..e1abbcc93b5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldOperation.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; + +import java.util.Set; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public abstract class SummaryInFieldOperation implements FieldOperation { + protected String name; + protected SummaryTransform transform; + protected Set<SummaryField.Source> sources = new java.util.LinkedHashSet<>(); + + public SummaryInFieldOperation(String name) { + this.name = name; + } + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public void setTransform(SummaryTransform transform) { + this.transform = transform; + } + + public SummaryTransform getTransform() { + return transform; + } + + public void addSource(String name) { + sources.add(new SummaryField.Source(name)); + } + + public void addSource(SummaryField.Source source) { + sources.add(source); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldShortOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldShortOperation.java new file mode 100644 index 00000000000..1ffb8d9a83e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldShortOperation.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.documentmodel.SummaryField; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class SummaryInFieldShortOperation extends SummaryInFieldOperation { + public SummaryInFieldShortOperation(String name) { + super(name); + } + + public void apply(SDField field) { + SummaryField ret = field.getSummaryField(name); + if (ret == null) { + ret = new SummaryField(name, field.getDataType()); + ret.addSource(field.getName()); + ret.addDestination("default"); + } + ret.setImplicit(false); + + ret.setTransform(transform); + for (SummaryField.Source source : sources) { + ret.addSource(source); + } + field.addSummaryField(ret); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryToOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryToOperation.java new file mode 100644 index 00000000000..f392b2fdcc8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryToOperation.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.documentmodel.SummaryField; + +import java.util.Set; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class SummaryToOperation implements FieldOperation { + private Set<String> destinations = new java.util.LinkedHashSet<>(); + private String name; + + public void setName(String name) { + this.name = name; + } + + public void addDestination(String destination) { + destinations.add(destination); + } + + public void apply(SDField field) { + SummaryField summary; + summary = field.getSummaryField(name); + if (summary == null) { + summary = new SummaryField(field); + summary.addSource(field.getName()); + summary.addDestination("default"); + field.addSummaryField(summary); + } + summary.setImplicit(false); + + for (String destination : destinations) { + summary.addDestination(destination); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightOperation.java new file mode 100644 index 00000000000..3cbd557b850 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightOperation.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.searchdefinition.document.SDField; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class WeightOperation implements FieldOperation { + private int weight; + + public int getWeight() { + return weight; + } + + public void setWeight(int weight) { + this.weight = weight; + } + + public void apply(SDField field) { + field.setWeight(weight); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightedSetOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightedSetOperation.java new file mode 100644 index 00000000000..c1721a4dbb5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightedSetOperation.java @@ -0,0 +1,63 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.fieldoperation; + +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.document.DataType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.document.WeightedSetDataType; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class WeightedSetOperation implements FieldOperation { + + private Boolean createIfNonExistent; + private Boolean removeIfZero; + + public Boolean getCreateIfNonExistent() { + return createIfNonExistent; + } + + public void setCreateIfNonExistent(Boolean createIfNonExistent) { + this.createIfNonExistent = createIfNonExistent; + } + + public Boolean getRemoveIfZero() { + return removeIfZero; + } + + public void setRemoveIfZero(Boolean removeIfZero) { + this.removeIfZero = removeIfZero; + } + + public void apply(SDField field) { + WeightedSetDataType ctype = (WeightedSetDataType) field.getDataType(); + + if (createIfNonExistent != null) { + field.setDataType(DataType.getWeightedSet(ctype.getNestedType(), createIfNonExistent, + ctype.removeIfZero())); + } + + ctype = (WeightedSetDataType) field.getDataType(); + if (removeIfZero != null) { + field.setDataType(DataType.getWeightedSet(ctype.getNestedType(), + ctype.createIfNonExistent(), removeIfZero)); + } + + ctype = (WeightedSetDataType) field.getDataType(); + for (Object o : field.getAttributes().values()) { + Attribute attribute = (Attribute) o; + attribute.setRemoveIfZero(ctype.removeIfZero()); + attribute.setCreateIfNonExistent(ctype.createIfNonExistent()); + } + } + + @Override + public String toString() { + return "WeightedSetOperation{" + + "createIfNonExistent=" + createIfNonExistent + + ", removeIfZero=" + removeIfZero + + '}'; + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java new file mode 100644 index 00000000000..b925b4f2caa --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.parser; + +import com.yahoo.javacc.FastCharStream; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +@SuppressWarnings("deprecation") +public class SimpleCharStream extends FastCharStream implements com.yahoo.searchdefinition.parser.CharStream, + com.yahoo.vespa.indexinglanguage.parser.CharStream +{ + public SimpleCharStream(String input) { + super(input); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java new file mode 100644 index 00000000000..5d39c70e3b9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.document.PositionDataType; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * This processor creates a {@link com.yahoo.searchdefinition.document.SDDocumentType} for each {@link Search} object which holds all the data that search + * associates with a document described in a search definition file. This includes all extra fields, summary fields and + * implicit fields. All non-indexed and non-summary fields are discarded. + */ +public class AddExtraFieldsToDocument extends Processor { + + public AddExtraFieldsToDocument(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + SDDocumentType document = search.getDocument(); + if (document != null) { + for (Field field : search.extraFieldList()) { + addSdField(search, document, (SDField)field); + } + for (SummaryField field : search.getSummary("default").getSummaryFields()) { + addSummaryField(search, document, field); + } + } + } + + private void addSdField(Search search, SDDocumentType document, SDField field) { + if (field.getIndexToCount() == 0 && field.getAttributes().isEmpty()) { + return; + } + for (Attribute atr : field.getAttributes().values()) { + if (atr.getName().equals(field.getName() + "_position")) { + DataType type = PositionDataType.INSTANCE; + if (atr.getCollectionType().equals(Attribute.CollectionType.ARRAY)) { + type = DataType.getArray(type); + } + addField(search, document, new SDField(document, atr.getName(), type)); + } else if (!atr.getName().equals(field.getName())) { + addField(search, document, new SDField(document, atr.getName(), atr.getDataType())); + } + } + addField(search, document, field); + } + + private void addSummaryField(Search search, SDDocumentType document, SummaryField field) { + Field docField = document.getField(field.getName()); + if (docField == null) { + SDField newField = search.getField(field.getName()); + if (newField == null) { + newField = new SDField(document, field.getName(), field.getDataType(), field.isHeader(), true); + newField.setIsExtraField(true); + } + document.addField(newField); + } else if (!docField.getDataType().equals(field.getDataType())) { + throw newProcessException(search, field, "Summary field has conflicting type."); + } + } + + private void addField(Search search, SDDocumentType document, Field field) { + if (document.getField(field.getName()) != null) { + throw newProcessException(search, field, "Field shadows another."); + } + document.addField(field); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java new file mode 100644 index 00000000000..9df77c09373 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Checks that attribute properties only are set for attributes that have data (are created by an indexing statement). + * + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class AttributeProperties extends Processor { + + public AttributeProperties(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField field : search.allFieldsList()) { + String fieldName = field.getName(); + + // For each attribute, check if the attribute has been created + // by an indexing statement. + for (Attribute attribute : field.getAttributes().values()) { + if (attributeCreated(field, attribute.getName())) { + continue; + } + // Check other fields or statements that may have created this attribute. + boolean created = false; + for (SDField f : search.allFieldsList()) { + // Checking against the field we are looking at + if (!f.getName().equals(fieldName)) { + if (attributeCreated(f, attribute.getName())) { + created = true; + break; + } + } + } + if (!created) { + throw new IllegalArgumentException("Attribute '" + attribute.getName() + "' in field '" + + field.getName() + "' is not created by the indexing statement"); + } + } + } + } + + /** + * Checks if the attribute has been created bye an indexing statement in this field. + * + * @param field a searchdefinition field + * @param attributeName name of the attribute + * @return true if the attribute has been created by this field, else false + */ + static boolean attributeCreated(SDField field, String attributeName) { + if (!field.doesAttributing()) { + return false; + } + for (Attribute attribute : field.getAttributes().values()) { + if (attribute.getName().equals(attributeName)) { + return true; + } + } + return false; + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java new file mode 100644 index 00000000000..774160793cb --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.document.DataType; +import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.document.NumericDataType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Fields that derive to attribute(s) and no indices should use the WORD indexing form, + * in a feeble attempt to match the most peoples expectations as closely as possible. + * + * @author vegardh + * + */ +public class AttributesImplicitWord extends Processor { + + public AttributesImplicitWord(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField field : search.allFieldsList()) { + if (fieldImplicitlyWordMatch(field)) { + field.getMatching().setType(Matching.Type.WORD); + } + } + } + + private boolean fieldImplicitlyWordMatch(SDField field) { + // numeric types should not trigger exact-match query parsing + DataType dt = field.getDataType().getPrimitiveType(); + if (dt != null && dt instanceof NumericDataType) { + return false; + } + return (field.getIndexToCount() == 0 + && field.getAttributes().size() > 0 + && field.getIndices().isEmpty() + && !field.getMatching().isTypeUserSet()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java new file mode 100644 index 00000000000..2e2e665eb34 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.document.DataType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Checks that bolding or dynamic summary is turned on only for text fields. Throws exception if it is turned on for any + * other fields (otherwise will cause indexing failure) + * + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class Bolding extends Processor { + + public Bolding(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField field : search.allFieldsList()) { + for (SummaryField summary : field.getSummaryFields()) { + if (summary.getTransform().isBolded() && + !((summary.getDataType() == DataType.STRING) || (summary.getDataType() == DataType.URI))) + { + throw new IllegalArgumentException("'bolding: on' for non-text field " + + "'" + field.getName() + "'" + + " (" + summary.getDataType() + ")" + + " is not allowed"); + } else if (summary.getTransform().isDynamic() && + !((summary.getDataType() == DataType.STRING) || (summary.getDataType() == DataType.URI))) + { + throw new IllegalArgumentException("'summary: dynamic' for non-text field " + + "'" + field.getName() + "'" + + " (" + summary.getDataType() + ")" + + " is not allowed"); + } + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java new file mode 100644 index 00000000000..4ad9b5304a5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.document.Field; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Adds field sets for 1) fields defined inside document type 2) fields inside search but outside document + * @author vegardh + * + */ +public class BuiltInFieldSets extends Processor { + + private static final String DOC_FIELDSET_NAME = "[document]"; + public static final String SEARCH_FIELDSET_NAME = "[search]"; // Public due to oddities in position handling. + public static final String INTERNAL_FIELDSET_NAME = "[internal]"; // This one populated from misc places + + public BuiltInFieldSets(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + addDocumentFieldSet(); + addSearchFieldSet(); + // "Hook" the field sets on search onto the document types, since we will include them + // on the document configs + search.getDocument().setFieldSets(search.fieldSets()); + } + + private void addSearchFieldSet() { + for (SDField searchField : search.extraFieldList()) { + search.fieldSets().addBuiltInFieldSetItem(SEARCH_FIELDSET_NAME, searchField.getName()); + } + } + + private void addDocumentFieldSet() { + for (Field docField : search.getDocument().fieldSet()) { + search.fieldSets().addBuiltInFieldSetItem(DOC_FIELDSET_NAME, docField.getName()); + } + } + + + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java new file mode 100644 index 00000000000..0aabd4dc192 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java @@ -0,0 +1,201 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.document.ArrayDataType; +import com.yahoo.document.DataType; +import com.yahoo.document.PositionDataType; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; +import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; +import java.util.logging.Level; + +/** + * Adds a "fieldName.zcurve" long attribute and a "fieldName.distance" summary field to all position type fields. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class CreatePositionZCurve extends Processor { + + public CreatePositionZCurve(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField field : search.allFieldsList()) { + DataType fieldType = field.getDataType(); + if ( ! isSupportedPositionType(fieldType)) continue; + + if (field.doesIndexing()) { + fail(search, field, "Indexing of data type '" + fieldType.getName() + "' is not supported, " + + "replace 'index' statement with 'attribute'."); + } + + if ( ! field.doesAttributing()) continue; + + boolean doesSummary = field.doesSummarying(); + + String fieldName = field.getName(); + field.getAttributes().remove(fieldName); + + String zName = PositionDataType.getZCurveFieldName(fieldName); + SDField zCurveField = createZCurveField(field, zName); + search.addExtraField(zCurveField); + search.fieldSets().addBuiltInFieldSetItem(BuiltInFieldSets.INTERNAL_FIELDSET_NAME, zCurveField.getName()); + + // configure summary + Collection<String> summaryTo = removeSummaryTo(field); + ensureCompatibleSummary(field, zName, + PositionDataType.getPositionSummaryFieldName(fieldName), + DataType.getArray(DataType.STRING), // will become "xmlstring" + SummaryTransform.POSITIONS, summaryTo); + ensureCompatibleSummary(field, zName, + PositionDataType.getDistanceSummaryFieldName(fieldName), + DataType.INT, + SummaryTransform.DISTANCE, summaryTo); + // clear indexing script + field.setIndexingScript(null); + SDField posX = field.getStructField(PositionDataType.FIELD_X); + if (posX != null) { + posX.setIndexingScript(null); + } + SDField posY = field.getStructField(PositionDataType.FIELD_Y); + if (posY != null) { + posY.setIndexingScript(null); + } + if (doesSummary) ensureCompatibleSummary(field, zName, + field.getName(), + field.getDataType(), + SummaryTransform.GEOPOS, summaryTo); + } + } + + private SDField createZCurveField(SDField inputField, String fieldName) { + if (search.getField(fieldName) != null || search.getAttribute(fieldName) != null) { + throw newProcessException(search, null, "Incompatible position attribute '" + fieldName + + "' already created."); + } + boolean isArray = inputField.getDataType() instanceof ArrayDataType; + SDField field = new SDField(fieldName, isArray ? DataType.getArray(DataType.LONG) : DataType.LONG); + Attribute attribute = new Attribute(fieldName, Attribute.Type.LONG, isArray ? Attribute.CollectionType.ARRAY : + Attribute.CollectionType.SINGLE); + attribute.setPosition(true); + attribute.setFastSearch(true); + field.addAttribute(attribute); + + ScriptExpression script = inputField.getIndexingScript(); + script = (ScriptExpression)new RemoveSummary(inputField.getName()).convert(script); + script = (ScriptExpression)new PerformZCurve(field, fieldName).convert(script); + field.setIndexingScript(script); + return field; + } + + private void ensureCompatibleSummary(SDField field, String sourceName, String summaryName, DataType summaryType, + SummaryTransform summaryTransform, Collection<String> summaryTo) { + SummaryField summary = search.getSummaryField(summaryName); + if (summary == null) { + summary = new SummaryField(summaryName, summaryType, summaryTransform); + summary.addDestination("default"); + summary.addDestinations(summaryTo); + field.addSummaryField(summary); + } else if (!summary.getDataType().equals(summaryType)) { + fail(search, field, "Incompatible summary field '" + summaryName + "' type "+summary.getDataType()+" already created."); + } else if (summary.getTransform() == SummaryTransform.NONE) { + summary.setTransform(summaryTransform); + summary.addDestination("default"); + summary.addDestinations(summaryTo); + } else if (summary.getTransform() != summaryTransform) { + deployLogger.log(Level.WARNING, "Summary field " + summaryName + " has wrong transform: " + summary.getTransform()); + return; + } + SummaryField.Source source = new SummaryField.Source(sourceName); + summary.getSources().clear(); + summary.addSource(source); + } + + private Set<String> removeSummaryTo(SDField field) { + Set<String> summaryTo = new HashSet<>(); + Collection<SummaryField> summaryFields = field.getSummaryFields(); + for (SummaryField summary : summaryFields) { + summaryTo.addAll(summary.getDestinations()); + } + summaryFields.clear(); + return summaryTo; + } + + private static boolean isSupportedPositionType(DataType dataType) { + if (dataType instanceof ArrayDataType) { + dataType = ((ArrayDataType)dataType).getNestedType(); + } + return dataType.equals(PositionDataType.INSTANCE); + } + + private static class RemoveSummary extends ExpressionConverter { + + final String find; + + RemoveSummary(String find) { + this.find = find; + } + + @Override + protected boolean shouldConvert(Expression exp) { + if (!(exp instanceof SummaryExpression)) { + return false; + } + String fieldName = ((SummaryExpression)exp).getFieldName(); + return fieldName == null || fieldName.equals(find); + } + + @Override + protected Expression doConvert(Expression exp) { + return null; + } + } + + private static class PerformZCurve extends ExpressionConverter { + + final String find; + final String replace; + final boolean isArray; + + PerformZCurve(SDField find, String replace) { + this.find = find.getName(); + this.replace = replace; + this.isArray = find.getDataType() instanceof ArrayDataType; + } + + @Override + protected boolean shouldConvert(Expression exp) { + if (!(exp instanceof AttributeExpression)) { + return false; + } + String fieldName = ((AttributeExpression)exp).getFieldName(); + return fieldName == null || fieldName.equals(find); + } + + @Override + protected Expression doConvert(Expression exp) { + return new StatementExpression( + isArray ? new ForEachExpression(new ZCurveExpression()) : + new ZCurveExpression(), new AttributeExpression(replace)); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java new file mode 100644 index 00000000000..6d5b3ff8936 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DeprecateAttributePrefetch.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +public class DeprecateAttributePrefetch extends Processor { + + public DeprecateAttributePrefetch(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField field : search.allFieldsList()) { + for (Attribute a : field.getAttributes().values()) { + if (Boolean.TRUE.equals(a.getPrefetchValue())) { + warn(search, field, "Attribute prefetch is deprecated. Use an explicitly defined document summary with all desired fields defined as attribute."); + } + if (Boolean.FALSE.equals(a.getPrefetchValue())) { + warn(search, field, "Attribute prefetch is deprecated. no-prefetch can be removed."); + } + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java new file mode 100644 index 00000000000..dbe83143189 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.document.DataType; +import com.yahoo.document.MapDataType; +import com.yahoo.document.PrimitiveDataType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Non-primitive key types for map and weighted set forbidden (though OK in document model) + * @author vegardh + * + */ +public class DisallowComplexMapAndWsetKeyTypes extends Processor { + + public DisallowComplexMapAndWsetKeyTypes(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + // TODO also traverse struct types to search for bad map or wset types there. Do this after document manager is fixed, do + // not start using the static stuff on SDDocumentTypes any more. + for (SDField field : search.allFieldsList()) { + if (field.getDataType() instanceof WeightedSetDataType) { + DataType nestedType = ((WeightedSetDataType)field.getDataType()).getNestedType(); + if (!(nestedType instanceof PrimitiveDataType)) { + fail(search, field, "Weighted set must have a primitive key type."); + } + } else if (field.getDataType() instanceof MapDataType) { + if (!(((MapDataType)field.getDataType()).getKeyType() instanceof PrimitiveDataType)) { + fail(search, field, "Map key type must be a primitive type"); + } + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java new file mode 100644 index 00000000000..49701f3e443 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Created by balder on 3/10/15. + */ +public class DiversitySettingsValidator extends Processor { + public DiversitySettingsValidator(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (RankProfile rankProfile : rankProfileRegistry.localRankProfiles(search)) { + if (rankProfile.getMatchPhaseSettings() != null && rankProfile.getMatchPhaseSettings().getDiversity() != null) { + validate(rankProfile, rankProfile.getMatchPhaseSettings().getDiversity()); + } + } + } + private void validate(RankProfile rankProfile, RankProfile.DiversitySettings settings) { + String attributeName = settings.getAttribute(); + new AttributeValidator(search.getName(), rankProfile.getName(), + search.getAttribute(attributeName), attributeName).validate(); + } + + private static class AttributeValidator extends MatchPhaseSettingsValidator.AttributeValidator { + public AttributeValidator(String searchName, String rankProfileName, Attribute attribute, String attributeName) { + super(searchName, rankProfileName, attribute, attributeName); + } + + protected void validateThatAttributeIsSingleAndNotPredicate() { + if (!attribute.getCollectionType().equals(Attribute.CollectionType.SINGLE) || + attribute.getType().equals(Attribute.Type.PREDICATE)) + { + failValidation("must be single value numeric, or enumerated attribute, but it is '" + + attribute.getDataType().getName() + "'"); + } + } + + @Override + public void validate() { + validateThatAttributeExists(); + validateThatAttributeIsSingleAndNotPredicate(); + } + + @Override + public String getValidationType() { + return "diversity"; + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java new file mode 100644 index 00000000000..d53be4bd18d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java @@ -0,0 +1,98 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.document.*; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.vespa.indexinglanguage.ExpressionSearcher; +import com.yahoo.vespa.indexinglanguage.expressions.*; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * The implementation of exact matching + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class ExactMatch extends Processor { + + public static final String DEFAULT_EXACT_TERMINATOR = "@@"; + + public ExactMatch(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField field : search.allFieldsList()) { + Matching.Type matching = field.getMatching().getType(); + if (matching.equals(Matching.Type.EXACT) || + matching.equals(Matching.Type.WORD)) + { + implementExactMatch(field, search); + } else if (field.getMatching().getExactMatchTerminator() != null) { + warn(search, field, "exact-terminator requires 'exact' matching to have any effect."); + } + } + } + + private void implementExactMatch(SDField field, Search search) { + field.setStemming(Stemming.NONE); + field.getNormalizing().inferLowercase(); + + if (field.getMatching().getType().equals(Matching.Type.WORD)) { + field.addQueryCommand("word"); + } else { // exact + String exactTerminator = DEFAULT_EXACT_TERMINATOR; + if (field.getMatching().getExactMatchTerminator() != null && + ! field.getMatching().getExactMatchTerminator().equals("")) + { + exactTerminator = field.getMatching().getExactMatchTerminator(); + } else { + warn(search, field, + "With 'exact' matching, an exact-terminator is needed (using \"" + + exactTerminator +"\" as terminator)"); + } + field.addQueryCommand("exact " + exactTerminator); + + // The following part illustrates how nice it would have been with canonical + // representation of indices + if (field.doesIndexing()) { + exactMatchSettingsForField(field); + } + } + ScriptExpression script = field.getIndexingScript(); + if (new ExpressionSearcher<>(IndexExpression.class).containedIn(script)) { + field.setIndexingScript((ScriptExpression)new MyProvider(search).convert(field.getIndexingScript())); + } + } + + private void exactMatchSettingsForField(SDField field) { + field.getRanking().setFilter(true); + } + + private static class MyProvider extends TypedTransformProvider { + + MyProvider(Search search) { + super(ExactExpression.class, search); + } + + @Override + protected boolean requiresTransform(Expression exp, DataType fieldType) { + return exp instanceof OutputExpression; + } + + @Override + protected Expression newTransform(DataType fieldType) { + Expression exp = new ExactExpression(); + if (fieldType instanceof CollectionDataType) { + exp = new ForEachExpression(exp); + } + return exp; + } + } +} + diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java new file mode 100644 index 00000000000..e809fe90c7f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java @@ -0,0 +1,94 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.document.FieldSet; +import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.NormalizeLevel; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Computes the right "index commands" for each fieldset in a search definition. + * + * + * @author vegardh + * @author bratseth + */ +public class FieldSetValidity extends Processor { + + public FieldSetValidity(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (FieldSet fieldSet : search.fieldSets().userFieldSets().values()) { + checkFieldNames(search, fieldSet); + checkMatching(search, fieldSet); + checkNormalization(search, fieldSet); + checkStemming(search, fieldSet); + } + } + + private void checkFieldNames(Search search, FieldSet fieldSet) { + for (String fld : fieldSet.getFieldNames()) { + SDField field = search.getField(fld); + if (field == null) { + throw new IllegalArgumentException("For search '" + search.getName() + "': Field '"+ fld + "' in " + fieldSet + " does not exist."); + } + } + } + + private void checkMatching(Search search, FieldSet fieldSet) { + Matching fsMatching = null; + for (String fld : fieldSet.getFieldNames()) { + SDField field = search.getField(fld); + Matching fieldMatching = field.getMatching(); + if (fsMatching==null) { + fsMatching = fieldMatching; + } else { + if (fsMatching!=null && !fsMatching.equals(fieldMatching)) { + warn(search, field, "The matching settings for the fields in " + fieldSet + " are inconsistent (explicitly or because of field type). This may lead to recall and ranking issues."); + return; + } + } + } + } + + private void checkNormalization(Search search, FieldSet fieldSet) { + NormalizeLevel.Level fsNorm = null; + for (String fld : fieldSet.getFieldNames()) { + SDField field = search.getField(fld); + NormalizeLevel.Level fieldNorm = field.getNormalizing().getLevel(); + if (fsNorm==null) { + fsNorm = fieldNorm; + } else { + if (fsNorm!=null && !fsNorm.equals(fieldNorm)) { + warn(search, field, "The normalization settings for the fields in " + fieldSet + " are inconsistent (explicitly or because of field type). This may lead to recall and ranking issues."); + return; + } + } + } + } + + private void checkStemming(Search search, FieldSet fieldSet) { + Stemming fsStemming = null; + for (String fld : fieldSet.getFieldNames()) { + SDField field = search.getField(fld); + Stemming fieldStemming = field.getStemming(); + if (fsStemming==null) { + fsStemming = fieldStemming; + } else { + if (fsStemming!=null && !fsStemming.equals(fieldStemming)) { + warn(search, field, "The stemming settings for the fields in the fieldset '"+fieldSet.getName()+"' are inconsistent (explicitly or because of field type). This may lead to recall and ranking issues."); + return; + } + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java new file mode 100644 index 00000000000..1df8c642750 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java @@ -0,0 +1,70 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.document.RankType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.logging.Level; + +/** + * Takes the fields and indexes that are of type rank filter, and stores those names on all rank profiles + * @author vegardh + * + */ +public class FilterFieldNames extends Processor { + + public FilterFieldNames(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField f : search.allFieldsList()) { + if (f.getRanking().isFilter()) { + filterField(f.getName()); + } + } + for (RankProfile profile : rankProfileRegistry.localRankProfiles(search)) { + Set<String> filterFields = new LinkedHashSet<>(); + findFilterFields(search, profile, filterFields); + for (Iterator<String> itr = filterFields.iterator(); itr.hasNext(); ) { + String fieldName = itr.next(); + profile.filterFields().add(fieldName); + profile.addRankSetting(fieldName, RankProfile.RankSetting.Type.RANKTYPE, RankType.EMPTY); + } + } + } + + private void filterField(String f) { + for (RankProfile rp : rankProfileRegistry.localRankProfiles(search)) { + rp.filterFields().add(f); + } + } + + private void findFilterFields(Search search, RankProfile profile, Set<String> filterFields) { + for (Iterator<RankProfile.RankSetting> itr = profile.declaredRankSettingIterator(); itr.hasNext(); ) { + RankProfile.RankSetting setting = itr.next(); + if (setting.getType().equals(RankProfile.RankSetting.Type.PREFERBITVECTOR) && + ((Boolean)setting.getValue()).booleanValue()) + { + String fieldName = setting.getFieldName(); + if (search.getField(fieldName) != null) { + if (!profile.filterFields().contains(fieldName)) { + filterFields.add(fieldName); + } + } else { + deployLogger.log(Level.WARNING, "For rank profile '" + profile.getName() + "': Cannot apply rank filter setting to unexisting field '" + fieldName + "'"); + } + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java new file mode 100644 index 00000000000..fa7725843b1 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java @@ -0,0 +1,222 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import java.util.logging.Level; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.document.PositionDataType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Makes implicitly defined summaries into explicit summaries + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class ImplicitSummaries extends Processor { + + public ImplicitSummaries(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + DocumentSummary defaultSummary=search.getSummary("default"); + if (defaultSummary==null) { + defaultSummary=new DocumentSummary("default"); + search.addSummary(defaultSummary); + } + + for (SDField field : search.allFieldsList()) { + collectSummaries(field,search); + } + + for (DocumentSummary documentSummary : search.getSummaries().values()) { + documentSummary.purgeImplicits(); + } + } + + private void addSummaryFieldSources(SummaryField summaryField, SDField sdField) { + sdField.addSummaryFieldSources(summaryField); + } + + private void collectSummaries(SDField field,Search search) { + SummaryField addedSummaryField=null; + + // Implicit + String fieldName = field.getName(); + SummaryField fieldSummaryField=field.getSummaryField(fieldName); + if (fieldSummaryField == null && field.doesSummarying()) { + fieldSummaryField=new SummaryField(fieldName, field.getDataType()); + fieldSummaryField.setImplicit(true); + addSummaryFieldSources(fieldSummaryField, field); + fieldSummaryField.addDestination("default"); + field.addSummaryField(fieldSummaryField); + addedSummaryField = fieldSummaryField; + } + if (fieldSummaryField != null) { + for (String dest : fieldSummaryField.getDestinations()) { + DocumentSummary summary = search.getSummary(dest); + if (summary != null) { + summary.add(fieldSummaryField); + } + } + } + + // Attribute prefetch + for (Attribute attribute : field.getAttributes().values()) { + if (attribute.getName().equals(fieldName)) { + if (addedSummaryField != null) { + addedSummaryField.setTransform(SummaryTransform.ATTRIBUTE); + } + if (attribute.isPrefetch()) { + addPrefetchAttribute(attribute, field, search); + } + } + } + + // Position attributes + if (field.doesSummarying()) { + for (Attribute attribute : field.getAttributes().values()) { + if (!attribute.isPosition()) { + continue; + } + DocumentSummary attributePrefetchSummary = getOrCreateAttributePrefetchSummary(search); + attributePrefetchSummary.add(field.getSummaryField(PositionDataType.getDistanceSummaryFieldName(fieldName))); + attributePrefetchSummary.add(field.getSummaryField(PositionDataType.getPositionSummaryFieldName(fieldName))); + } + } + + // Explicits + for (SummaryField summaryField : field.getSummaryFields()) { + // Make sure we fetch from attribute here too + Attribute attribute=field.getAttributes().get(fieldName); + if (attribute!=null && summaryField.getTransform()==SummaryTransform.NONE) { + summaryField.setTransform(SummaryTransform.ATTRIBUTE); + } + + if (isValid(summaryField,search)) { + addToDestinations(summaryField, search); + } + } + + } + + private DocumentSummary getOrCreateAttributePrefetchSummary(Search search) { + DocumentSummary summary = search.getSummary("attributeprefetch"); + if (summary == null) { + summary = new DocumentSummary("attributeprefetch"); + search.addSummary(summary); + } + return summary; + } + + + private void addPrefetchAttribute(Attribute attribute,SDField field,Search search) { + if (attribute.getPrefetchValue()==null) { // Prefetch by default - unless any summary makes this dynamic + // Check if there is an implicit dynamic definition + SummaryField fieldSummaryField=field.getSummaryField(attribute.getName()); + if (fieldSummaryField!=null && fieldSummaryField.getTransform().isDynamic()) return; + + // Check if an explicit class makes it dynamic (first is enough, as all must be the same, checked later) + SummaryField explicitSummaryField=search.getExplicitSummaryField(attribute.getName()); + if (explicitSummaryField!=null && explicitSummaryField.getTransform().isDynamic()) return; + } + + DocumentSummary summary = getOrCreateAttributePrefetchSummary(search); + SummaryField attributeSummaryField = new SummaryField(attribute.getName(), attribute.getDataType()); + attributeSummaryField.addSource(attribute.getName()); + attributeSummaryField.addDestination("attributeprefetch"); + attributeSummaryField.setTransform(SummaryTransform.ATTRIBUTE); + summary.add(attributeSummaryField); + } + + // Returns whether this is valid. Warns if invalid and ignorable. Throws if not ignorable. + private boolean isValid(SummaryField summaryField,Search search) { + if (summaryField.getTransform() == SummaryTransform.DISTANCE || + summaryField.getTransform() == SummaryTransform.POSITIONS) + { + int sourceCount = summaryField.getSourceCount(); + if (sourceCount != 1) { + throw newProcessException(search.getName(), summaryField.getName(), + "Expected 1 source field, got " + sourceCount + "."); + } + String sourceName = summaryField.getSingleSource(); + if (search.getAttribute(sourceName) == null) { + throw newProcessException(search.getName(), summaryField.getName(), + "Summary source attribute '" + sourceName + "' not found."); + } + return true; + } + + String fieldName = summaryField.getSourceField(); + SDField sourceField = search.getField(fieldName); + if (sourceField == null) { + throw newProcessException(search, summaryField, "Source field '" + fieldName + "' does not exist."); + } + if (!sourceField.doesSummarying() && + !summaryField.getTransform().equals(SummaryTransform.ATTRIBUTE) && + !summaryField.getTransform().equals(SummaryTransform.GEOPOS)) + { + // Summary transform attribute may indicate that the ilscript was rewritten to remove summary + // by another search that uses this same field in inheritance. + deployLogger.log(Level.WARNING, "Ignoring " + summaryField + ": " + sourceField + + " is not creating a summary value in its indexing statement"); + return false; + } + + if (summaryField.getTransform().isDynamic() + && summaryField.getName().equals(sourceField.getName()) + && sourceField.doesAttributing()) { + Attribute attribute=sourceField.getAttributes().get(sourceField.getName()); + if (attribute!=null) { + String destinations="document summary 'default'"; + if (summaryField.getDestinations().size()>0) { + destinations = "document summaries " + summaryField.getDestinations(); + } + deployLogger.log(Level.WARNING, "Will fetch the disk summary value of " + sourceField + " in " + destinations + + " since this summary field uses a dynamic summary value (snippet/bolding): Dynamic summaries and bolding " + + "is not supported with summary values fetched from in-memory attributes yet. If you want to see partial updates " + + "to this attribute, remove any bolding and dynamic snippeting from this field"); + // Note: The dynamic setting has already overridden the attribute map setting, + // so we do not need to actually do attribute.setSummary(false) here + // Also, we can not do this, since it makes it impossible to fetch this attribute + // in another summary + } + } + + return true; + } + + private void addToDestinations(SummaryField summaryField,Search search) { + if (summaryField.getDestinations().size()==0) { + addToDestination("default",summaryField,search); + } + else { + for (String destinationName : summaryField.getDestinations()) + addToDestination(destinationName,summaryField,search); + } + } + + private void addToDestination(String destinationName,SummaryField summaryField,Search search) { + DocumentSummary destination=search.getSummary(destinationName); + if (destination==null) { + destination=new DocumentSummary(destinationName); + search.addSummary(destination); + destination.add(summaryField); + } + else { + SummaryField existingField= + destination.getSummaryField(summaryField.getName()); + SummaryField merged=summaryField.mergeWith(existingField); + destination.add(merged); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java new file mode 100644 index 00000000000..b95f2469cd8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.document.DataType; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * This processor adds all implicit summary fields to all registered document summaries. If another field has already + * been registered with one of the implicit names, this processor will throw an {@link IllegalStateException}. + */ +public class ImplicitSummaryFields extends Processor { + + public ImplicitSummaryFields(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (DocumentSummary docsum : search.getSummaries().values()) { + addField(docsum, new SummaryField("rankfeatures", DataType.STRING, SummaryTransform.RANKFEATURES)); + addField(docsum, new SummaryField("summaryfeatures", DataType.STRING, SummaryTransform.SUMMARYFEATURES)); + } + } + + private void addField(DocumentSummary docsum, SummaryField field) { + if (docsum.getSummaryField(field.getName()) != null) { + throw new IllegalStateException("Summary class '" + docsum.getName() + "' uses reserved field name '" + + field.getName() + "'."); + } + docsum.add(field); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java new file mode 100644 index 00000000000..2c18c58d3dc --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Because of the way the parser works (allowing any token as identifier), + * it is not practical to limit the syntax of field names there, do it here. + * Important to disallow dash, has semantic in IL. + * @author vegardh + * + */ +public class IndexFieldNames extends Processor { + + private static final String FIELD_NAME_REGEXP = "[a-zA-Z]\\w*"; + + public IndexFieldNames(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField field : search.allFieldsList()) { + if (!field.getName().matches(FIELD_NAME_REGEXP) && !legalDottedPositionField(field)) { + fail(search, field, " Not a legal field name. Legal expression: " + FIELD_NAME_REGEXP); + } + } + } + + /** + * In {@link CreatePositionZCurve} we add some .position and .distance fields for pos fields. Make an exception for those for now. For 6.0, rename + * to _position and _distance and delete this method. + * @param field an {@link com.yahoo.searchdefinition.document.SDField} + * @return true if allowed + */ + private boolean legalDottedPositionField(SDField field) { + return field.getName().endsWith(".position") || field.getName().endsWith(".distance"); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java new file mode 100644 index 00000000000..fecc5e2309d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexSettingsNonFieldNames.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.Index; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.Iterator; + +/** + * Fail if: + * 1) There are index: settings without explicit index names (name same as field name) + * 2) All the index-to indexes differ from the field name. + * @author vegardh + * + */ +public class IndexSettingsNonFieldNames extends Processor { + + public IndexSettingsNonFieldNames(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField field : search.allFieldsList()) { + boolean fieldNameUsed = false; + for (Iterator i = field.getFieldNameAsIterator(); i.hasNext();) { + String iName = (String)(i.next()); + if (iName.equals(field.getName())) { + fieldNameUsed = true; + } + } + if (!fieldNameUsed) { + for (Index index : field.getIndices().values()) { + if (index.getName().equals(field.getName())) { + throw new IllegalArgumentException("Error in " + field + " in " + search + + ": When all index names differ from field name, index parameter settings must specify index name explicitly."); + } + } + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexTo2FieldSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexTo2FieldSet.java new file mode 100644 index 00000000000..3148273ddc1 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexTo2FieldSet.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * @author balder + */ +public class IndexTo2FieldSet extends Processor { + + public IndexTo2FieldSet(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField field : search.allFieldsList()) { + processField(field); + } + } + private void processField(SDField field) { + if (field.usesStructOrMap()) { + for (SDField subField : field.getStructFields()) { + processField(subField); + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java new file mode 100644 index 00000000000..c2e889f7993 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java @@ -0,0 +1,110 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; +import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.InputExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * <p>This processor modifies all indexing scripts so that they input the value of the owning field by default. It also + * ensures that all fields used as input exist.</p> + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class IndexingInputs extends Processor { + + public IndexingInputs(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField field : search.allFieldsList()) { + ScriptExpression script = field.getIndexingScript(); + if (script == null) { + continue; + } + String fieldName = field.getName(); + script = (ScriptExpression)new DefaultToCurrentField(fieldName).convert(script); + script = (ScriptExpression)new EnsureInputExpression(fieldName).convert(script); + new VerifyInputExpression(search, field).visit(script); + field.setIndexingScript(script); + } + } + + private static class DefaultToCurrentField extends ExpressionConverter { + + final String fieldName; + + DefaultToCurrentField(String fieldName) { + this.fieldName = fieldName; + } + + @Override + protected boolean shouldConvert(Expression exp) { + return exp instanceof InputExpression && ((InputExpression)exp).getFieldName() == null; + } + + @Override + protected Expression doConvert(Expression exp) { + return new InputExpression(fieldName); + } + } + + private static class EnsureInputExpression extends ExpressionConverter { + + final String fieldName; + + EnsureInputExpression(String fieldName) { + this.fieldName = fieldName; + } + + @Override + protected boolean shouldConvert(Expression exp) { + return exp instanceof StatementExpression; + } + + @Override + protected Expression doConvert(Expression exp) { + if (exp.requiredInputType() != null) { + return new StatementExpression(new InputExpression(fieldName), exp); + } else { + return exp; + } + } + } + + private class VerifyInputExpression extends ExpressionVisitor { + + private final Search search; + private final SDField field; + + public VerifyInputExpression(Search search, SDField field) { + this.search = search; + this.field = field; + } + + @Override + protected void doVisit(Expression exp) { + if (!(exp instanceof InputExpression)) { + return; + } + SDDocumentType docType = search.getDocument(); + String inputField = ((InputExpression)exp).getFieldName(); + if (docType.getField(inputField) != null) { + return; + } + fail(search, field, "Indexing script refers to field '" + inputField + "' which does not exist " + + "in document type '" + docType.getName() + "'."); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java new file mode 100644 index 00000000000..55e7dabf9ba --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java @@ -0,0 +1,141 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; +import com.yahoo.vespa.indexinglanguage.expressions.*; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.*; + +/** + * <p>This processor modifies all indexing scripts so that they output to the owning field by default. It also prevents + * any output expression from writing to any field except for the owning field. Finally, for <tt>SummaryExpression</tt>, + * this processor expands to write all appropriate summary fields.</p> + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class IndexingOutputs extends Processor { + + public IndexingOutputs(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField field : search.allFieldsList()) { + ScriptExpression script = field.getIndexingScript(); + if (script == null) { + continue; + } + Set<String> summaryFields = new TreeSet<>(); + findSummaryTo(search, field, summaryFields, summaryFields); + MyConverter converter = new MyConverter(search, field, summaryFields); + field.setIndexingScript((ScriptExpression)converter.convert(script)); + } + } + + public void findSummaryTo(Search search, SDField field, Set<String> dynamicSummary, + Set<String> staticSummary) { + Map<String, SummaryField> summaryFields = search.getSummaryFields(field); + if (summaryFields.isEmpty()) { + fillSummaryToFromField(field, dynamicSummary, staticSummary); + } else { + fillSummaryToFromSearch(search, field, summaryFields, dynamicSummary, staticSummary); + } + } + + private void fillSummaryToFromSearch(Search search, SDField field, Map<String, SummaryField> summaryFields, + Set<String> dynamicSummary, Set<String> staticSummary) { + for (SummaryField summaryField : summaryFields.values()) { + fillSummaryToFromSummaryField(search, field, summaryField, dynamicSummary, staticSummary); + } + } + + private void fillSummaryToFromSummaryField(Search search, SDField field, SummaryField summaryField, Set<String> dynamicSummary, Set<String> staticSummary) { + SummaryTransform summaryTransform = summaryField.getTransform(); + String summaryName = summaryField.getName(); + if (summaryTransform.isDynamic() && summaryField.getSourceCount() > 2) { + // Avoid writing to summary fields that have more than a single input field, as that is handled by the + // summary rewriter in the search core. + return; + } + if (summaryTransform.isDynamic()) { + DataType fieldType = field.getDataType(); + if (fieldType != DataType.URI && fieldType != DataType.STRING) { + warn(search, field, "Dynamic summaries are only supported for fields of type " + + "string, ignoring summary field '" + summaryField.getName() + + "' for sd field '" + field.getName() + "' of type " + + fieldType.getName() + "."); + return; + } + dynamicSummary.add(summaryName); + } else if (summaryTransform != SummaryTransform.ATTRIBUTE) { + staticSummary.add(summaryName); + } + } + + private static void fillSummaryToFromField(SDField field, Set<String> dynamicSummary, Set<String> staticSummary) { + for (SummaryField summaryField : field.getSummaryFields()) { + String summaryName = summaryField.getName(); + if (summaryField.getTransform().isDynamic()) { + dynamicSummary.add(summaryName); + } else { + staticSummary.add(summaryName); + } + } + } + + private class MyConverter extends ExpressionConverter { + + final Search search; + final Field field; + final Set<String> summaryFields; + + MyConverter(Search search, Field field, Set<String> summaryFields) { + this.search = search; + this.field = field; + this.summaryFields = summaryFields.isEmpty() ? Collections.singleton(field.getName()) : summaryFields; + } + + @Override + protected boolean shouldConvert(Expression exp) { + if (!(exp instanceof OutputExpression)) { + return false; + } + String fieldName = ((OutputExpression)exp).getFieldName(); + if (fieldName == null) { + return true; // inject appropriate field name + } + if (!fieldName.equals(field.getName())) { + fail(search, field, "Indexing expression '" + exp + "' attempts to write to a field other than '" + + field.getName() + "'."); + } + return false; + } + + @Override + protected Expression doConvert(Expression exp) { + List<Expression> ret = new LinkedList<>(); + if (exp instanceof AttributeExpression) { + ret.add(new AttributeExpression(field.getName())); + } else if (exp instanceof IndexExpression) { + ret.add(new IndexExpression(field.getName())); + } else if (exp instanceof SummaryExpression) { + for (String fieldName : summaryFields) { + ret.add(new SummaryExpression(fieldName)); + } + } else { + throw new UnsupportedOperationException(exp.getClass().getName()); + } + return new StatementExpression(ret); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java new file mode 100644 index 00000000000..af04deb5347 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java @@ -0,0 +1,149 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.document.*; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; +import com.yahoo.vespa.indexinglanguage.expressions.*; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.HashSet; +import java.util.Set; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class IndexingValidation extends Processor { + + public IndexingValidation(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + VerificationContext ctx = new VerificationContext(new MyAdapter(search)); + for (SDField field : search.allFieldsList()) { + ScriptExpression script = field.getIndexingScript(); + try { + script.verify(ctx); + MyConverter converter = new MyConverter(); + for (StatementExpression exp : script) { + converter.convert(exp); // TODO: stop doing this explicitly when visiting a script does not branch + } + } catch (VerificationException e) { + fail(search, field, "For expression '" + e.getExpression() + "': " + e.getMessage()); + } + } + } + + private static class MyConverter extends ExpressionConverter { + + final Set<String> outputs = new HashSet<>(); + final Set<String> prevNames = new HashSet<>(); + + @Override + protected ExpressionConverter branch() { + MyConverter ret = new MyConverter(); + ret.outputs.addAll(outputs); + ret.prevNames.addAll(prevNames); + return ret; + } + + @Override + protected boolean shouldConvert(Expression exp) { + if (exp instanceof OutputExpression) { + String fieldName = ((OutputExpression)exp).getFieldName(); + if (outputs.contains(fieldName) && !prevNames.contains(fieldName)) { + throw new VerificationException(exp, "Attempting to assign conflicting values to field '" + + fieldName + "'."); + } + outputs.add(fieldName); + prevNames.add(fieldName); + } + if (exp.createdOutputType() != null) { + prevNames.clear(); + } + return false; + } + + @Override + protected Expression doConvert(Expression exp) { + throw new UnsupportedOperationException(); + } + } + + private static class MyAdapter implements FieldTypeAdapter { + + final Search search; + + public MyAdapter(Search search) { + this.search = search; + } + + @Override + public DataType getInputType(Expression exp, String fieldName) { + SDField field = search.getDocumentField(fieldName); + if (field == null) { + throw new VerificationException(exp, "Input field '" + fieldName + "' not found."); + } + return field.getDataType(); + } + + @Override + public void tryOutputType(Expression exp, String fieldName, DataType valueType) { + String fieldDesc; + DataType fieldType; + if (exp instanceof AttributeExpression) { + Attribute attribute = search.getAttribute(fieldName); + if (attribute == null) { + throw new VerificationException(exp, "Attribute '" + fieldName + "' not found."); + } + fieldDesc = "attribute"; + fieldType = attribute.getDataType(); + } else if (exp instanceof IndexExpression) { + SDField field = search.getField(fieldName); + if (field == null) { + throw new VerificationException(exp, "Index field '" + fieldName + "' not found."); + } + fieldDesc = "index field"; + fieldType = field.getDataType(); + } else if (exp instanceof SummaryExpression) { + SummaryField field = search.getSummaryField(fieldName); + if (field == null) { + throw new VerificationException(exp, "Summary field '" + fieldName + "' not found."); + } + fieldDesc = "summary field"; + fieldType = field.getDataType(); + } else { + throw new UnsupportedOperationException(); + } + if (!fieldType.isAssignableFrom(valueType) && + !fieldType.isAssignableFrom(createCompatType(valueType))) + { + throw new VerificationException(exp, "Can not assign " + valueType.getName() + " to " + fieldDesc + + " '" + fieldName + "' which is " + fieldType.getName() + "."); + } + } + + private static DataType createCompatType(DataType origType) { + if (origType instanceof ArrayDataType) { + return DataType.getArray(createCompatType(((ArrayDataType)origType).getNestedType())); + } else if (origType instanceof MapDataType) { + MapDataType mapType = (MapDataType)origType; + return DataType.getMap(createCompatType(mapType.getKeyType()), + createCompatType(mapType.getValueType())); + } else if (origType instanceof WeightedSetDataType) { + return DataType.getWeightedSet(createCompatType(((WeightedSetDataType)origType).getNestedType())); + } else if (origType == PositionDataType.INSTANCE) { + return DataType.LONG; + } else { + return origType; + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java new file mode 100644 index 00000000000..ff8be71f8c3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.document.Field; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.InputExpression; +import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class IndexingValues extends Processor { + + public IndexingValues(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (Field field : search.getDocument().fieldSet()) { + SDField sdField = (SDField)field; + if (!sdField.isExtraField()) { + new RequireThatDocumentFieldsAreImmutable(field).convert(sdField.getIndexingScript()); + } + } + } + + private class RequireThatDocumentFieldsAreImmutable extends ExpressionConverter { + + final Field field; + Expression mutatedBy; + + RequireThatDocumentFieldsAreImmutable(Field field) { + this.field = field; + } + + @Override + public ExpressionConverter branch() { + return clone(); + } + + @Override + protected boolean shouldConvert(Expression exp) { + if (exp instanceof OutputExpression && mutatedBy != null) { + throw newProcessException(search, field, + "Indexing expression '" + mutatedBy + "' modifies the value of the " + + "document field '" + field.getName() + "'. This is no longer supported -- " + + "declare such fields outside the document."); + } + if (exp instanceof InputExpression && ((InputExpression)exp).getFieldName().equals(field.getName())) { + mutatedBy = null; + } else if (exp.createdOutputType() != null) { + mutatedBy = exp; + } + return false; + } + + @Override + protected Expression doConvert(Expression exp) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java new file mode 100644 index 00000000000..a29896ce3e8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java @@ -0,0 +1,86 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.document.NumericDataType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.Index; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; +import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; +import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.HashSet; +import java.util.Set; + +/** + * Replaces the 'index' statement of all numerical fields to 'attribute' because we no longer support numerical indexes. + * + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public class IntegerIndex2Attribute extends Processor { + + public IntegerIndex2Attribute(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField field : search.allFieldsList()) { + if (field.doesIndexing() && field.getDataType().getPrimitiveType() instanceof NumericDataType) { + // Avoid changing for example RISE fields + if (field.getIndex(field.getName()) != null && !(field.getIndex(field.getName()).getType().equals(Index.Type.VESPA))) continue; + ScriptExpression script = field.getIndexingScript(); + Set<String> attributeNames = new HashSet<>(); + new MyVisitor(attributeNames).visit(script); + field.setIndexingScript((ScriptExpression)new MyConverter(attributeNames).convert(script)); + warn(search, field, "Changed to attribute because numerical indexes (field has type "+field.getDataType().getName()+") is not currently supported." + + " Index-only settings may fail. Ignore this warning for streaming search."); + } + } + } + + private static class MyVisitor extends ExpressionVisitor { + + final Set<String> attributeNames; + + public MyVisitor(Set<String> attributeNames) { + this.attributeNames = attributeNames; + } + + @Override + protected void doVisit(Expression exp) { + if (exp instanceof AttributeExpression) { + attributeNames.add(((AttributeExpression)exp).getFieldName()); + } + } + } + + private static class MyConverter extends ExpressionConverter { + + final Set<String> attributeNames; + + public MyConverter(Set<String> attributeNames) { + this.attributeNames = attributeNames; + } + + @Override + protected boolean shouldConvert(Expression exp) { + return exp instanceof IndexExpression; + } + + @Override + protected Expression doConvert(Expression exp) { + String indexName = ((IndexExpression)exp).getFieldName(); + if (attributeNames.contains(indexName)) { + return null; + } + return new AttributeExpression(indexName); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java new file mode 100644 index 00000000000..e4e96bb51ab --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java @@ -0,0 +1,79 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.Iterator; + +/** + * Expresses literal boosts in terms of extra indices with rank boost. + * One extra index named <i>indexname</i>_exact is added for each index having + * a fields with literal-boosts of zero or more (zero to support other + * rank profiles setting a literal boost). Complete boost values in to fields + * are translated to rank boosts to the implementation indices. + * These indices has no positional + * or phrase support and contains concatenated versions of each field value + * of complete-boosted fields indexed to <i>indexname</i>. A search for indexname + * will be rewritten to also search <i>indexname</i>_exaxt + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class LiteralBoost extends Processor { + + public LiteralBoost(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + /** Adds extra search fields and indices to express literal boosts */ + @Override + public void process() { + checkRankModifierRankType(search); + addLiteralBoostsToFields(search); + reduceFieldLiteralBoosts(search); + } + + /** Checks if literal boost is given using rank: , and set the actual literal boost accordingly. */ + private void checkRankModifierRankType(Search search) { + for (SDField field : search.allFieldsList()) { + if (field.getLiteralBoost() > -1) continue; // Let explicit value take precedence + if (field.getRanking().isLiteral()) + field.setLiteralBoost(100); + } + } + + /** + * Ensures there are field boosts for all literal boosts mentioned in rank profiles. + * This is required because boost indices will only be generated by looking + * at field boosts + */ + private void addLiteralBoostsToFields(Search search) { + Iterator i = matchingRankSettingsIterator(search, RankProfile.RankSetting.Type.LITERALBOOST); + while (i.hasNext()) { + RankProfile.RankSetting setting = (RankProfile.RankSetting)i.next(); + SDField field = search.getField(setting.getFieldName()); + if (field == null) continue; + if (field.getLiteralBoost() < 0) + field.setLiteralBoost(0); + } + } + + private void reduceFieldLiteralBoosts(Search search) { + for (SDField field : search.allFieldsList()) { + if (field.getLiteralBoost()<0) continue; + reduceFieldLiteralBoost(field,search); + } + } + + private void reduceFieldLiteralBoost(SDField field,Search search) { + SDField literalField = addField(search, field, "literal", + "{ input " + field.getName() + " | tokenize | index " + field.getName() + "_literal; }", + "literal-boost"); + literalField.setWeight(field.getWeight() + field.getLiteralBoost()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java new file mode 100644 index 00000000000..856b0cf3348 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.Index; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Takes the aliases set on field by parser and sets them on correct Index or Attribute + * @author vegardh + * + */ +public class MakeAliases extends Processor { + + public MakeAliases(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + List<String> usedAliases = new ArrayList<>(); + for (SDField field : search.allFieldsList()) { + for (Map.Entry<String, String> e : field.getAliasToName().entrySet()) { + String alias = e.getKey(); + String name = e.getValue(); + String errMsg = "For search '"+search.getName()+"': alias '"+alias+"' "; + if (search.existsIndex(alias)) { + throw new IllegalArgumentException(errMsg+"is illegal since it is the name of an index."); + } + if (search.getAttribute(alias)!=null) { + throw new IllegalArgumentException(errMsg+"is illegal since it is the name of an attribute."); + } + if (usedAliases.contains(alias)) { + throw new IllegalArgumentException(errMsg+"specified more than once."); + } + usedAliases.add(alias); + + Index index = field.getIndex(name); + Attribute attribute = field.getAttributes().get(name); + if (index != null) { + index.addAlias(alias); // alias will be for index in this case, since it is the one used in a search + } else if (attribute != null && !field.doesIndexing()) { + attribute.getAliases().add(alias); + } else { + index = new Index(name); + index.addAlias(alias); + field.addIndex(index); + } + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java new file mode 100644 index 00000000000..a9f013daa98 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * <p>All summary fields which are not attributes + * must currently be present in the default summary class, + * since the default summary class also defines the docsum.dat format. + * This processor adds any missing summaries to the default summary. + * When that is decoupled from the actual summaries returned, this + * processor can be removed. Note: the StreamingSummary also takes advantage of + * the fact that default is the superset.</p> + * + * <p>All other summary logic should work unchanged without this processing step + * except that IndexStructureValidator.validateSummaryFields must be changed to + * consider all summaries, not just the default, i.e change to + * if (search.getSummaryField(expr.getFieldName()) == null)</p> + * + * <p>This must be done after other summary processors.</p> + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class MakeDefaultSummaryTheSuperSet extends Processor { + + public MakeDefaultSummaryTheSuperSet(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + DocumentSummary defaultSummary=search.getSummary("default"); + for (SummaryField summaryField : search.getUniqueNamedSummaryFields().values() ) { + if (defaultSummary.getSummaryField(summaryField.getName()) != null) continue; + if (summaryField.getTransform() == SummaryTransform.ATTRIBUTE) continue; + + defaultSummary.add(summaryField.clone()); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java new file mode 100644 index 00000000000..f21670e6f78 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.Matching.Type; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.HashMap; +import java.util.Map; + +/** + * Warn on inconsistent match settings for any index + * + * @author vegardh + */ +public class MatchConsistency extends Processor { + + public MatchConsistency(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + Map<String, Matching.Type> types = new HashMap<>(); + for (SDField field : search.allFieldsList()) { + new MyVisitor(search, field, types).visit(field.getIndexingScript()); + } + } + + void checkMatching(Search search, SDField field, Map<String, Type> types, String indexTo) { + Type prevType = types.get(indexTo); + if (prevType == null) { + types.put(indexTo, field.getMatching().getType()); + } else if (!field.getMatching().getType().equals(prevType)) { + warn(search, field, "The matching type for index '" + indexTo + "' (got " + field.getMatching().getType() + + ") is inconsistent with that given for the same index in a previous field (had " + + prevType + ")."); + } + } + + private class MyVisitor extends ExpressionVisitor { + + final Search search; + final SDField field; + final Map<String, Type> types; + + public MyVisitor(Search search, SDField field, Map<String, Type> types) { + this.search = search; + this.field = field; + this.types = types; + } + + @Override + protected void doVisit(Expression exp) { + if (exp instanceof IndexExpression) { + checkMatching(search, field, types, ((IndexExpression)exp).getFieldName()); + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java new file mode 100644 index 00000000000..efceeca0af0 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java @@ -0,0 +1,92 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Validates the match phase settings for all registered rank profiles. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class MatchPhaseSettingsValidator extends Processor { + + public MatchPhaseSettingsValidator(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (RankProfile rankProfile : rankProfileRegistry.localRankProfiles(search)) { + RankProfile.MatchPhaseSettings settings = rankProfile.getMatchPhaseSettings(); + if (settings != null) { + validateMatchPhaseSettings(rankProfile, settings); + } + } + } + + private void validateMatchPhaseSettings(RankProfile rankProfile, RankProfile.MatchPhaseSettings settings) { + String attributeName = settings.getAttribute(); + new AttributeValidator(search.getName(), rankProfile.getName(), + search.getAttribute(attributeName), attributeName).validate(); + } + + public static class AttributeValidator { + + private final String searchName; + private final String rankProfileName; + protected final Attribute attribute; + private final String attributeName; + + public AttributeValidator(String searchName, String rankProfileName, Attribute attribute, String attributeName) { + this.searchName = searchName; + this.rankProfileName = rankProfileName; + this.attribute = attribute; + this.attributeName = attributeName; + } + + public void validate() { + validateThatAttributeExists(); + validateThatAttributeIsSingleNumeric(); + validateThatAttributeIsFastSearch(); + } + + protected void validateThatAttributeExists() { + if (attribute == null) { + failValidation("does not exists"); + } + } + + protected void validateThatAttributeIsSingleNumeric() { + if (!attribute.getCollectionType().equals(Attribute.CollectionType.SINGLE) || + attribute.getType().equals(Attribute.Type.STRING) || + attribute.getType().equals(Attribute.Type.PREDICATE)) + { + failValidation("must be single value numeric, but it is '" + + attribute.getDataType().getName() + "'"); + } + } + + protected void validateThatAttributeIsFastSearch() { + if (!attribute.isFastSearch()) { + failValidation("must be fast-search, but it is not"); + } + } + + protected void failValidation(String what) { + throw new IllegalArgumentException(createMessagePrefix() + what); + } + + public String getValidationType() { return "match-phase"; } + + private String createMessagePrefix() { + return "In search definition '" + searchName + + "', rank-profile '" + rankProfileName + + "': " + getValidationType() + " attribute '" + attributeName + "' "; + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java new file mode 100644 index 00000000000..61a11fd3a4e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java @@ -0,0 +1,82 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.processing.multifieldresolver.IndexCommandResolver; +import com.yahoo.searchdefinition.processing.multifieldresolver.RankTypeResolver; +import com.yahoo.searchdefinition.processing.multifieldresolver.StemmingResolver; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * Ensures that there are no conflicting types or field settings + * in multifield indices, either by changing settings or by splitting + * conflicting fields in multiple ones with different settings. + * + * @author bratseth + */ +public class MultifieldIndexHarmonizer extends Processor { + + /** A map from index names to a List of fields going to that index */ + private Map<String,List<SDField>> indexToFields=new java.util.HashMap<>(); + + public MultifieldIndexHarmonizer(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + populateIndexToFields(search); + resolveAllConflicts(search); + } + + private void populateIndexToFields(Search search) { + for (SDField field : search.allFieldsList() ) { + if (!field.doesIndexing()) { + continue; + } + for (Iterator j = field.getFieldNameAsIterator(); j.hasNext();) { + String indexName = (String)j.next(); + addIndexField(indexName, field); + } + } + } + + private void addIndexField(String indexName,SDField field) { + List<SDField> fields=indexToFields.get(indexName); + if (fields==null) { + fields=new java.util.ArrayList<>(); + indexToFields.put(indexName,fields); + } + fields.add(field); + } + + private void resolveAllConflicts(Search search) { + for (Map.Entry<String, List<SDField>> entry : indexToFields.entrySet()) { + String indexName = entry.getKey(); + List<SDField> fields = entry.getValue(); + if (fields.size() == 1) continue; // It takes two to make a conflict + resolveConflicts(indexName, fields, search); + } + } + + /** + * Resolves all conflicts for one index + * + * @param indexName the name of the index in question + * @param fields all the fields indexed to this index + * @param search the search definition having this + */ + private void resolveConflicts(String indexName,List<SDField> fields,Search search) { + new StemmingResolver(indexName, fields, search, deployLogger).resolve(); + new IndexCommandResolver(indexName, fields, search, deployLogger).resolve(); + new RankTypeResolver(indexName, fields, search, deployLogger).resolve(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java new file mode 100644 index 00000000000..8e5122cc158 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java @@ -0,0 +1,76 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.document.*; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.vespa.indexinglanguage.expressions.*; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * The implementation of "gram" matching - splitting the incoming text and the queries into + * n-grams for matching. This will also validate the gram settings. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class NGramMatch extends Processor { + + public static final int DEFAULT_GRAM_SIZE = 2; + + public NGramMatch(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField field : search.allFieldsList()) { + if (field.getMatching().getType().equals(Matching.Type.GRAM)) + implementGramMatch(search, field); + else if (field.getMatching().getGramSize()>=0) + throw new IllegalArgumentException("gram-size can only be set when the matching mode is 'gram'"); + } + } + + private void implementGramMatch(Search search, SDField field) { + if (field.doesAttributing()) + throw new IllegalArgumentException("gram matching is not supported with attributes, use 'index' not 'attribute' in indexing"); + + int n = field.getMatching().getGramSize(); + if (n<0) + n=DEFAULT_GRAM_SIZE; // not set - use default gram size + if (n==0) + throw new IllegalArgumentException("Illegal gram size in " + field + ": Must be at least 1"); + field.getNormalizing().inferCodepoint(); + field.setStemming(Stemming.NONE); // not compatible with stemming and normalizing + field.addQueryCommand("ngram " + n); + field.setIndexingScript((ScriptExpression)new MyProvider(search, n).convert(field.getIndexingScript())); + } + + private static class MyProvider extends TypedTransformProvider { + + final int ngram; + + MyProvider(Search search, int ngram) { + super(NGramExpression.class, search); + this.ngram = ngram; + } + + @Override + protected boolean requiresTransform(Expression exp, DataType fieldType) { + return exp instanceof OutputExpression; + } + + @Override + protected Expression newTransform(DataType fieldType) { + Expression exp = new NGramExpression(null, ngram); + if (fieldType instanceof CollectionDataType) { + exp = new ForEachExpression(exp); + } + return exp; + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java new file mode 100644 index 00000000000..078e9400ec5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.indexinglanguage.ExpressionOptimizer; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Run ExpressionOptimizer on all scripts, to get rid of expressions that have no effect. + */ +public class OptimizeIlscript extends Processor { + public OptimizeIlscript(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField field : search.allFieldsList()) { + ScriptExpression script = field.getIndexingScript(); + if (script == null) { + continue; + } + field.setIndexingScript((ScriptExpression)new ExpressionOptimizer().convert(script)); + if (!field.getIndexingScript().toString().equals(script.toString())) { + warn(search, field, "Rewrote ilscript from:\n" + script.toString() + "\nto\n" + field.getIndexingScript().toString()); + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java new file mode 100644 index 00000000000..a156a98a584 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java @@ -0,0 +1,138 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.IntegerFieldValue; +import com.yahoo.document.datatypes.LongFieldValue; +import com.yahoo.searchdefinition.Index; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.BooleanIndexDefinition; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; +import com.yahoo.vespa.indexinglanguage.expressions.*; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.ArrayList; +import java.util.List; + +/** + * Validates the predicate fields. + * + * @author <a href="mailto:lesters@yahoo-inc.com">Lester Solbakken</a> + */ +public class PredicateProcessor extends Processor { + + public PredicateProcessor(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField field : search.allFieldsList()) { + if (field.getDataType() == DataType.PREDICATE) { + if (field.doesIndexing()) { + fail(search, field, "Use 'attribute' instead of 'index'. This will require a refeed if you have upgraded."); + } + if (field.doesAttributing()) { + Attribute attribute = field.getAttributes().get(field.getName()); + for (Index index : field.getIndices().values()) { + BooleanIndexDefinition booleanDefinition = index.getBooleanIndexDefiniton(); + if (booleanDefinition == null || !booleanDefinition.hasArity()) { + fail(search, field, "Missing arity value in predicate field."); + } + if (booleanDefinition.getArity() < 2) { + fail(search, field, "Invalid arity value in predicate field, must be greater than 1."); + } + double threshold = booleanDefinition.getDensePostingListThreshold(); + if (threshold <= 0 || threshold > 1) { + fail(search, field, "Invalid dense-posting-list-threshold value in predicate field. " + + "Value must be in range (0..1]."); + } + + attribute.setArity(booleanDefinition.getArity()); + attribute.setLowerBound(booleanDefinition.getLowerBound()); + attribute.setUpperBound(booleanDefinition.getUpperBound()); + + attribute.setDensePostingListThreshold(threshold); + addPredicateOptimizationIlScript(field, booleanDefinition); + } + DocumentSummary summary = search.getSummary("attributeprefetch"); + if (summary != null) { + summary.remove(attribute.getName()); + } + for (SummaryField summaryField : search.getSummaryFields(field).values()) { + summaryField.setTransform(SummaryTransform.NONE); + } + } + } else if (field.getDataType().getPrimitiveType() == DataType.PREDICATE) { + fail(search, field, "Collections of predicates are not allowed."); + } else if (field.getDataType() == DataType.RAW && field.doesIndexing()) { + fail(search, field, "Indexing of RAW fields are not supported. If you are using RAW fields for boolean search, use predicate data type instead."); + } else { + // if field is not a predicate, disallow predicate-related index parameters + for (Index index : field.getIndices().values()) { + if (index.getBooleanIndexDefiniton() != null) { + BooleanIndexDefinition def = index.getBooleanIndexDefiniton(); + if (def.hasArity()) { + fail(search, field, "Arity parameter is used only for predicate type fields."); + } else if (def.hasLowerBound() || def.hasUpperBound()) { + fail(search, field, "Parameters lower-bound and upper-bound are used only for predicate type fields."); + } else if (def.hasDensePostingListThreshold()) { + fail(search, field, "Parameter dense-posting-list-threshold is used only for predicate type fields."); + } + } + } + } + } + } + + private void addPredicateOptimizationIlScript(SDField field, BooleanIndexDefinition booleanIndexDefiniton) { + Expression script = field.getIndexingScript(); + if (script == null) { + return; + } + script = new StatementExpression(makeSetPredicateVariablesScript(booleanIndexDefiniton), script); + + ExpressionConverter converter = new PredicateOutputTransformer(search); + field.setIndexingScript(new ScriptExpression((StatementExpression)converter.convert(script))); + } + + private Expression makeSetPredicateVariablesScript(BooleanIndexDefinition options) { + List<Expression> expressions = new ArrayList<>(); + expressions.add(new SetValueExpression(new IntegerFieldValue(options.getArity()))); + expressions.add(new SetVarExpression("arity")); + if (options.hasLowerBound()) { + expressions.add(new SetValueExpression(new LongFieldValue(options.getLowerBound()))); + expressions.add(new SetVarExpression("lower_bound")); + } + if (options.hasUpperBound()) { + expressions.add(new SetValueExpression(new LongFieldValue(options.getUpperBound()))); + expressions.add(new SetVarExpression("upper_bound")); + } + return new StatementExpression(expressions); + } + + private static class PredicateOutputTransformer extends TypedTransformProvider { + + PredicateOutputTransformer(Search search) { + super(OptimizePredicateExpression.class, search); + } + + @Override + protected boolean requiresTransform(Expression exp, DataType fieldType) { + return exp instanceof OutputExpression && fieldType == DataType.PREDICATE; + } + + @Override + protected Expression newTransform(DataType fieldType) { + return new OptimizePredicateExpression(); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java new file mode 100644 index 00000000000..74f359602bd --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java @@ -0,0 +1,84 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.processing.multifieldresolver.RankProfileTypeSettingsProcessor; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Executor of processors. This defines the right order of processor execution. + * + * @author bratseth + */ +public class Processing { + + /** + * Runs all search processors on the given {@link Search} object. These will modify the search object, <b>possibly + * exchanging it with another</b>, as well as its document types. + * + * @param search The search to process. + * @param deployLogger The log to log messages and warnings for application deployment to + * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} + * @param queryProfiles The query profiles contained in the application this search is part of. + */ + public static void process(Search search, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles) { + search.process(); + new UrlFieldValidator(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new BuiltInFieldSets(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new SearchMustHaveDocument(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new ReservedDocumentNames(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new IndexFieldNames(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new IntegerIndex2Attribute(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new MakeAliases(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new SetLanguage(search, deployLogger, rankProfileRegistry, queryProfiles).process(); // Needs to come before UriHack, see ticket 6405470 + new UriHack(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new LiteralBoost(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new IndexTo2FieldSet(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new TagType(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new IndexingInputs(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new OptimizeIlscript(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new ValidateFieldWithIndexSettingsCreatesIndex(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new AttributesImplicitWord(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new CreatePositionZCurve(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new WordMatch(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new DeprecateAttributePrefetch(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new ImplicitSummaries(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new ImplicitSummaryFields(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new SummaryConsistency(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new SummaryNamesFieldCollisions(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new SummaryFieldsMustHaveValidSource(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new MakeDefaultSummaryTheSuperSet(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new Bolding(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new AttributeProperties(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new SetRankTypeEmptyOnFilters(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new IndexSettingsNonFieldNames(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new SummaryDynamicStructsArrays(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new StringSettingsOnNonStringFields(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new IndexingOutputs(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new ExactMatch(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new NGramMatch(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new TextMatch(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new MultifieldIndexHarmonizer(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new FilterFieldNames(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new MatchConsistency(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new ValidateFieldTypes(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new DisallowComplexMapAndWsetKeyTypes(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new SortingSettings(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new FieldSetValidity(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new AddExtraFieldsToDocument(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new PredicateProcessor(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new MatchPhaseSettingsValidator(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new DiversitySettingsValidator(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new TensorFieldProcessor(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new RankProfileTypeSettingsProcessor(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + + // These two should be last. + new IndexingValidation(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + new IndexingValues(search, deployLogger, rankProfileRegistry, queryProfiles).process(); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java new file mode 100644 index 00000000000..a3e0484b2fd --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java @@ -0,0 +1,131 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.*; +import com.yahoo.searchdefinition.Index; +import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.RankType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.Iterator; +import java.util.List; +import java.util.logging.Level; + +/** + * Abstract superclass of all search definition processors. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public abstract class Processor { + + protected final Search search; + protected DeployLogger deployLogger; + protected final RankProfileRegistry rankProfileRegistry; + protected final QueryProfiles queryProfiles; + + /** + * Base constructor + * @param search the search to process + * @param deployLogger Logger du use when logging deploy output. + * @param rankProfileRegistry Registry with all rank profiles, used for lookup and insertion. + * @param queryProfiles The query profiles contained in the application this search is part of. + */ + public Processor(Search search, DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + this.search = search; + this.deployLogger = deployLogger; + this.rankProfileRegistry = rankProfileRegistry; + this.queryProfiles = queryProfiles; + } + + /** + * Processes the input search definition by <b>modifying</b> the input search and its documents, and returns the + * input search definition. + */ + public abstract void process(); + + /** + * Convenience method for adding a no-strings-attached implementation field for a regular field + * + * @param search the search definition in question + * @param field the field to add an implementation field for + * @param suffix the suffix of the added implementation field (without the underscore) + * @param indexing the indexing statement of the field + * @param queryCommand the query command of the original field, or null if none + * @return the implementation field which is added to the search + */ + protected SDField addField(Search search, SDField field, String suffix, String indexing, String queryCommand) { + SDField implementationField = search.getField(field.getName() + "_" + suffix); + if (implementationField != null) { + deployLogger.log(Level.WARNING, "Implementation field " + implementationField + " added twice"); + } else { + implementationField = new SDField(search.getDocument(), field.getName() + "_" + suffix, DataType.STRING); + } + implementationField.setRankType(RankType.EMPTY); + implementationField.setStemming(Stemming.NONE); + implementationField.getNormalizing().inferCodepoint(); + implementationField.parseIndexingScript(indexing); + for (Iterator i = field.getFieldNameAsIterator(); i.hasNext();) { + String indexName = (String)i.next(); + String implementationIndexName = indexName + "_" + suffix; + Index implementationIndex = new Index(implementationIndexName); + search.addIndex(implementationIndex); + } + if (queryCommand != null) { + field.addQueryCommand(queryCommand); + } + search.addExtraField(implementationField); + search.fieldSets().addBuiltInFieldSetItem(BuiltInFieldSets.INTERNAL_FIELDSET_NAME, implementationField.getName()); + return implementationField; + } + + /** + * Returns an iterator of all the rank settings with given type in all the rank profiles in this search + * definition. + */ + protected Iterator<RankProfile.RankSetting> matchingRankSettingsIterator( + Search search, RankProfile.RankSetting.Type type) + { + List<RankProfile.RankSetting> someRankSettings = new java.util.ArrayList<>(); + + for (RankProfile profile : rankProfileRegistry.localRankProfiles(search)) { + for (Iterator j = profile.declaredRankSettingIterator(); j.hasNext(); ) { + RankProfile.RankSetting setting = (RankProfile.RankSetting)j.next(); + if (setting.getType().equals(type)) { + someRankSettings.add(setting); + } + } + } + return someRankSettings.iterator(); + } + + protected String formatError(String searchName, String fieldName, String msg) { + return "For search '" + searchName + "', field '" + fieldName + "': " + msg; + } + + protected RuntimeException newProcessException(String searchName, String fieldName, String msg) { + return new IllegalArgumentException(formatError(searchName, fieldName, msg)); + } + + protected RuntimeException newProcessException(Search search, Field field, String msg) { + return newProcessException(search.getName(), field.getName(), msg); + } + + public void fail(Search search, Field field, String msg) { + throw newProcessException(search, field, msg); + } + + protected void warn(String searchName, String fieldName, String msg) { + String fullMsg = formatError(searchName, fieldName, msg); + deployLogger.log(Level.WARNING, fullMsg); + } + + protected void warn(Search search, Field field, String msg) { + warn(search.getName(), field.getName(), msg); + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java new file mode 100644 index 00000000000..74bf2d35bac --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.HashSet; +import java.util.Set; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ReservedDocumentNames extends Processor { + + private static final Set<String> RESERVED_NAMES = new HashSet<>(); + static { + for (SDDocumentType dataType : SDDocumentType.VESPA_DOCUMENT.getTypes()) { + RESERVED_NAMES.add(dataType.getName()); + } + } + + public ReservedDocumentNames(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + String docName = search.getDocument().getName(); + if (RESERVED_NAMES.contains(docName)) { + throw new IllegalArgumentException("For search '" + search.getName() + "': Document name '" + docName + + "' is reserved."); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java new file mode 100644 index 00000000000..5b45aecc257 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * A search must have a document definition of the same name inside of it, otherwise crashes may occur as late as + * during feeding + * @author vegardh + * + */ +public class SearchMustHaveDocument extends Processor { + + public SearchMustHaveDocument(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + if (search.getDocument()==null) { + throw new IllegalArgumentException("For search '" + search.getName() + "': A search specification must have an equally named document inside of it."); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java new file mode 100644 index 00000000000..764bb1602a7 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetLanguage.java @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.document.DataType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.indexinglanguage.expressions.SetLanguageExpression; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.ArrayList; +import java.util.List; + +/** + * Check that no text field appears before a field that sets language. + * + * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a> + */ +public class SetLanguage extends Processor { + + public SetLanguage(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + List<String> textFieldsWithoutLanguage = new ArrayList<>(); + + for (SDField field : search.allFieldsList()) { + if (fieldMustComeAfterLanguageSettingField(field)) { + textFieldsWithoutLanguage.add(field.getName()); + } + if (field.containsExpression(SetLanguageExpression.class) && !textFieldsWithoutLanguage.isEmpty()) { + StringBuffer fieldString = new StringBuffer(); + for (String fieldName : textFieldsWithoutLanguage) { + fieldString.append(fieldName).append(" "); + } + warn(search, field, "Field '" + field.getName() + "' sets the language for this document, " + + "and should be defined as the first field in the searchdefinition. If you have both header and body fields, this field "+ + "should be header, if you require it to affect subsequent header fields and/or any body fields. " + + "Preceding text fields that will not have their language set: " + + fieldString.toString() + + " (This warning is omitted for any subsequent fields that also do set_language.)"); + return; + } + } + } + + private boolean fieldMustComeAfterLanguageSettingField(SDField field) { + return (!field.containsExpression(SetLanguageExpression.class) && + (field.getDataType() == DataType.STRING)); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java new file mode 100644 index 00000000000..bc676b64a71 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.document.RankType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * All rank: filter fields should have rank type empty. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class SetRankTypeEmptyOnFilters extends Processor { + + public SetRankTypeEmptyOnFilters(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField field : search.allFieldsList()) { + if (field.getRanking().isFilter()) { + field.setRankType(RankType.EMPTY); + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java new file mode 100644 index 00000000000..6e77f48b4a7 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.Sorting; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Validate conflicting settings for sorting + * @author vegardh + * + */ +public class SortingSettings extends Processor { + + public SortingSettings(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField field : search.allFieldsList()) { + for (Attribute attribute : field.getAttributes().values()) { + Sorting sorting = attribute.getSorting(); + if (sorting.getFunction()!=Sorting.Function.UCA) { + if (sorting.getStrength()!=null && sorting.getStrength()!=Sorting.Strength.PRIMARY) { + warn(search, field, "Sort strength only works for sort function 'uca'."); + } + if (sorting.getLocale()!=null && !"".equals(sorting.getLocale())) { + warn(search, field, "Sort locale only works for sort function 'uca'."); + } + } + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java new file mode 100644 index 00000000000..48c9af5556a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.NumericDataType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +public class StringSettingsOnNonStringFields extends Processor { + + public StringSettingsOnNonStringFields(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField field : search.allFieldsList()) { + if (!doCheck(field)) continue; + if (field.getMatching().isTypeUserSet()) { + warn(search, field, "Matching type "+field.getMatching().getType()+" is only allowed for string fields."); + } + if (field.getRanking().isLiteral()) { + warn(search, field, "Rank type literal only applies to string fields"); + } + } + } + + private boolean doCheck(SDField field) { + if (field.getDataType() instanceof NumericDataType) return true; + if (field.getDataType() instanceof CollectionDataType) { + if (((CollectionDataType)field.getDataType()).getNestedType() instanceof NumericDataType) { + return true; + } + } + return false; + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java new file mode 100644 index 00000000000..20eeb5f0810 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java @@ -0,0 +1,102 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.document.WeightedSetDataType; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Ensure that summary field transforms for fields having the same name + * are consistent across summary classes + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class SummaryConsistency extends Processor { + + public SummaryConsistency(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (DocumentSummary summary : search.getSummaries().values()) { + if (summary.getName().equals("default")) continue; + for (SummaryField summaryField : summary.getSummaryFields() ) { + assertConsistency(summaryField,search); + makeAttributeTransformIfAppropriate(summaryField,search); + } + } + } + + /** If the source is an attribute, make this use the attribute transform */ + private void makeAttributeTransformIfAppropriate(SummaryField summaryField,Search search) { + if (summaryField.getTransform()!=SummaryTransform.NONE) return; + Attribute attribute=search.getAttribute(summaryField.getSingleSource()); + if (attribute==null) return; + summaryField.setTransform(SummaryTransform.ATTRIBUTE); + } + + private void assertConsistency(SummaryField summaryField,Search search) { + SummaryField existingDefault=search.getSummary("default").getSummaryField(summaryField.getName()); // Compare to default + if (existingDefault!=null) { + assertConsistentTypes(existingDefault,summaryField); + makeConsistentWithDefaultOrThrow(existingDefault,summaryField); + } + else { + // If no default, compare to whichever definition of the field + SummaryField existing=search.getExplicitSummaryField(summaryField.getName()); + if (existing==null) return; + assertConsistentTypes(existing,summaryField); + makeConsistentOrThrow(existing,summaryField,search); + } + } + + private void assertConsistentTypes(SummaryField field1,SummaryField field2) { + if (field1.getDataType() instanceof WeightedSetDataType && field2.getDataType() instanceof WeightedSetDataType && + ((WeightedSetDataType)field1.getDataType()).getNestedType().equals(((WeightedSetDataType)field2.getDataType()).getNestedType())) + return; // Disregard create-if-nonexistent and create-if-zero distinction + if ( ! field1.getDataType().equals(field2.getDataType())) + throw new IllegalArgumentException(field1.toLocateString() + " is inconsistent with " + field2.toLocateString() + ": All declarations of the same summary field must have the same type"); + } + + private void makeConsistentOrThrow(SummaryField field1, SummaryField field2,Search search) { + if (field2.getTransform()==SummaryTransform.ATTRIBUTE && field1.getTransform()==SummaryTransform.NONE) { + Attribute attribute=search.getAttribute(field1.getName()); + if (attribute != null) { + field1.setTransform(SummaryTransform.ATTRIBUTE); + } + } + + if (field2.getTransform().equals(SummaryTransform.NONE)) { + field2.setTransform(field1.getTransform()); + } + else { // New field sets an explicit transform - must be the same + assertEqualTransform(field1,field2); + } + } + private void makeConsistentWithDefaultOrThrow(SummaryField defaultField,SummaryField newField) { + if (newField.getTransform().equals(SummaryTransform.NONE)) { + newField.setTransform(defaultField.getTransform()); + } + else { // New field sets an explicit transform - must be the same + assertEqualTransform(defaultField,newField); + } + } + + + private void assertEqualTransform(SummaryField field1,SummaryField field2) { + if (!field2.getTransform().equals(field1.getTransform())) { + throw new IllegalArgumentException("Conflicting summary transforms. " + field2 +" is already defined as " + + field1 + ". A field with the same name " + + "can not have different transforms in different summary classes"); + } + } + + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java new file mode 100644 index 00000000000..f97d5345cab --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.document.*; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Fail if: + * An SD field explicitly says summary:dynamic , but the field is wset, array or struct. + * If there is an explicitly defined summary class, saying dynamic in one of its summary + * fields is always legal. + * @author vegardh + * + */ +public class SummaryDynamicStructsArrays extends Processor { + + public SummaryDynamicStructsArrays(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField field : search.allFieldsList()) { + DataType type = field.getDataType(); + if (type instanceof ArrayDataType || type instanceof WeightedSetDataType + || type instanceof StructDataType) { + for (SummaryField sField : field.getSummaryFields()) { + if (sField.getTransform().equals(SummaryTransform.DYNAMICTEASER)) { + throw new IllegalArgumentException("For field '"+field.getName()+"': dynamic summary is illegal " + + "for fields of type struct, array or weighted set. Use an explicit summary class with explicit summary fields sourcing from" + + " the array/struct/weighted set."); + } + } + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java new file mode 100644 index 00000000000..0d76ada0d52 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java @@ -0,0 +1,73 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Verifies that the source fields actually refers to a valid field. + * + * @author balder + * + */ +public class SummaryFieldsMustHaveValidSource extends Processor { + SummaryFieldsMustHaveValidSource(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + @Override + public void process() { + for (DocumentSummary summary : search.getSummaries().values()) { + for (SummaryField summaryField : summary.getSummaryFields()) { + if (summaryField.getSources().isEmpty()) { + if ((summaryField.getTransform() != SummaryTransform.RANKFEATURES) && + (summaryField.getTransform() != SummaryTransform.SUMMARYFEATURES)) + { + verifySource(summaryField.getName(), summaryField, summary); + } + } else if (summaryField.getSourceCount() == 1) { + verifySource(summaryField.getSingleSource(), summaryField, summary); + } else { + for (SummaryField.Source source : summaryField.getSources()) { + if ( ! source.getName().equals(summaryField.getName()) ) { + verifySource(source.getName(), summaryField, summary); + } + } + } + } + } + + } + + private boolean isValid(String source, SummaryField summaryField, DocumentSummary summary) { + return isDocumentField(source) || + (isNotInThisSummaryClass(summary, source) && isSummaryField(source)) || + (isInThisSummaryClass(summary, source) && !source.equals(summaryField.getName())) || + ("documentid".equals(source)); + } + + private void verifySource(String source, SummaryField summaryField, DocumentSummary summary) { + if ( ! isValid(source, summaryField, summary) ) { + throw new IllegalArgumentException("For search '" + search.getName() + "', summary class '" + summary.getName() + "'," + + " summary field '" + summaryField.getName() + "': there is no valid source '" + source + "'."); + } + } + + private boolean isNotInThisSummaryClass(DocumentSummary summary, String name) { + return summary.getSummaryField(name) == null; + } + private boolean isInThisSummaryClass(DocumentSummary summary, String name) { + return summary.getSummaryField(name) != null; + } + private boolean isDocumentField(String name) { + return search.getField(name) != null; + } + + private boolean isSummaryField(String name) { + return search.getSummaryField(name) != null; + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java new file mode 100644 index 00000000000..8d99611d697 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import java.util.HashMap; +import java.util.Map; + +import com.yahoo.collections.Pair; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryField.Source; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Verifies that equally named summary fields in different summary classes don't use different fields for source. + * The summarymap config doesn't model this. + * + * @author vegardh + * + */ +public class SummaryNamesFieldCollisions extends Processor { + + public SummaryNamesFieldCollisions(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + Map<String, Pair<String, String>> fieldToClassAndSource = new HashMap<>(); + for (DocumentSummary summary : search.getSummaries().values()) { + if ("default".equals(summary.getName())) continue; + for (SummaryField summaryField : summary.getSummaryFields() ) { + if (summaryField.isImplicit()) continue; + Pair<String, String> prevClassAndSource = fieldToClassAndSource.get(summaryField.getName()); + for (Source source : summaryField.getSources()) { + if (prevClassAndSource!=null) { + String prevClass = prevClassAndSource.getFirst(); + String prevSource = prevClassAndSource.getSecond(); + if (!prevClass.equals(summary.getName())) { + if (!prevSource.equals(source.getName())) { + throw new IllegalArgumentException("For search '"+search.getName()+"', summary class '"+summary.getName()+"'," + + " summary field '"+summaryField.getName()+"':" + + " Can not use source '"+source.getName()+"' for this summary field, an equally named field in summary class '" + + prevClass + "' uses a different source: '"+prevSource+"'."); + } + } + } else { + fieldToClassAndSource.put(summaryField.getName(), new Pair<>(summary.getName(), source.getName())); + } + } + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java new file mode 100644 index 00000000000..3e211f4f632 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.document.*; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.RankType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * The implementation of the tag datatype + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class TagType extends Processor { + + public TagType(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField field : search.allFieldsList()) { + if (field.getDataType() instanceof WeightedSetDataType && ((WeightedSetDataType)field.getDataType()).isTag()) { + implementTagType(field); + } + } + } + + private void implementTagType(SDField field) { + field.setDataType(DataType.getWeightedSet(DataType.STRING,true,true)); + // Don't set matching and ranking if this field is not attribute nor index + if (!field.doesIndexing() && !field.doesAttributing()) return; + Matching m = field.getMatching(); + if (!m.isTypeUserSet()) { + m.setType(Matching.Type.WORD); + } + if (field.getRankType()==null || field.getRankType()== RankType.DEFAULT) + field.setRankType((RankType.TAGS)); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java new file mode 100644 index 00000000000..eb4efbe377f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java @@ -0,0 +1,63 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Class that processes and validates tensor fields. + * + * @author <a href="geirst@yahoo-inc.com">Geir Storli</a> + */ +public class TensorFieldProcessor extends Processor { + + public TensorFieldProcessor(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField field : search.allFieldsList()) { + if (field.getDataType() == DataType.TENSOR) { + warnUseOfTensorFieldAsAttribute(field); + validateIndexingScripsForTensorField(field); + validateAttributeSettingForTensorField(field); + } else { + validateDataTypeForField(field); + } + } + } + + private void warnUseOfTensorFieldAsAttribute(SDField field) { + if (field.doesAttributing()) { + // TODO (geirst): Remove when no longer beta + warn(search, field, "An attribute of type 'tensor' is currently beta, and re-feeding data between Vespa versions might be required."); + } + } + + private void validateIndexingScripsForTensorField(SDField field) { + if (field.doesIndexing()) { + fail(search, field, "A field of type 'tensor' cannot be specified as an 'index' field."); + } + } + + private void validateAttributeSettingForTensorField(SDField field) { + if (field.doesAttributing()) { + Attribute attribute = field.getAttributes().get(field.getName()); + if (attribute != null && attribute.isFastSearch()) { + fail(search, field, "An attribute of type 'tensor' cannot be 'fast-search'."); + } + } + } + + private void validateDataTypeForField(SDField field) { + if (field.getDataType().getPrimitiveType() == DataType.TENSOR) { + fail(search, field, "A field with collection type of tensor is not supported. Use simple type 'tensor' instead."); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java new file mode 100644 index 00000000000..21723639ece --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java @@ -0,0 +1,122 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.document.CollectionDataType; +import com.yahoo.document.DataType; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; +import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression; +import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; +import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.expressions.SummaryExpression; +import com.yahoo.vespa.indexinglanguage.expressions.TokenizeExpression; +import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.Set; +import java.util.TreeSet; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class TextMatch extends Processor { + + public TextMatch(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField field : search.allFieldsList()) { + if (field.getMatching().getType() != Matching.Type.TEXT) { + continue; + } + ScriptExpression script = field.getIndexingScript(); + if (script == null) { + continue; + } + DataType fieldType = field.getDataType(); + if (fieldType instanceof CollectionDataType) { + fieldType = ((CollectionDataType)fieldType).getNestedType(); + } + if (fieldType != DataType.STRING) { + continue; + } + Set<String> dynamicSummary = new TreeSet<>(); + Set<String> staticSummary = new TreeSet<>(); + new IndexingOutputs(search, deployLogger, rankProfileRegistry, queryProfiles).findSummaryTo(search, field, dynamicSummary, staticSummary); + MyVisitor visitor = new MyVisitor(dynamicSummary); + visitor.visit(script); + if (!visitor.requiresTokenize) { + continue; + } + ExpressionConverter converter = new MyStringTokenizer(search, findAnnotatorConfig(search, field)); + field.setIndexingScript((ScriptExpression)converter.convert(script)); + } + } + + private static AnnotatorConfig findAnnotatorConfig(Search search, SDField field) { + AnnotatorConfig ret = new AnnotatorConfig(); + Stemming activeStemming = field.getStemming(); + if (activeStemming == null) { + activeStemming = search.getStemming(); + } + ret.setStemMode(activeStemming.toStemMode()); + ret.setRemoveAccents(field.getNormalizing().doRemoveAccents()); + return ret; + } + + private static class MyVisitor extends ExpressionVisitor { + + final Set<String> dynamicSummaryFields; + boolean requiresTokenize = false; + + MyVisitor(Set<String> dynamicSummaryFields) { + this.dynamicSummaryFields = dynamicSummaryFields; + } + + @Override + protected void doVisit(Expression exp) { + if (exp instanceof IndexExpression) { + requiresTokenize = true; + } + if (exp instanceof SummaryExpression && + dynamicSummaryFields.contains(((SummaryExpression)exp).getFieldName())) + { + requiresTokenize = true; + } + } + } + + private static class MyStringTokenizer extends TypedTransformProvider { + + final AnnotatorConfig annotatorCfg; + + MyStringTokenizer(Search search, AnnotatorConfig annotatorCfg) { + super(TokenizeExpression.class, search); + this.annotatorCfg = annotatorCfg; + } + + @Override + protected boolean requiresTransform(Expression exp, DataType fieldType) { + return exp instanceof OutputExpression; + } + + @Override + protected Expression newTransform(DataType fieldType) { + Expression exp = new TokenizeExpression(null, annotatorCfg); + if (fieldType instanceof CollectionDataType) { + exp = new ForEachExpression(exp); + } + return exp; + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TypedTransformProvider.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TypedTransformProvider.java new file mode 100644 index 00000000000..0fa9cbfa05f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TypedTransformProvider.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.vespa.indexinglanguage.ValueTransformProvider; +import com.yahoo.vespa.indexinglanguage.expressions.*; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public abstract class TypedTransformProvider extends ValueTransformProvider { + + private final Search search; + private DataType fieldType; + + TypedTransformProvider(Class<? extends Expression> transformClass, Search search) { + super(transformClass); + this.search = search; + } + + @Override + protected final boolean requiresTransform(Expression exp) { + if (exp instanceof OutputExpression) { + String fieldName = ((OutputExpression)exp).getFieldName(); + if (exp instanceof AttributeExpression) { + Attribute attribute = search.getAttribute(fieldName); + if (attribute == null) { + throw new IllegalArgumentException("Attribute '" + fieldName + "' not found."); + } + fieldType = attribute.getDataType(); + } else if (exp instanceof IndexExpression) { + Field field = search.getField(fieldName); + if (field == null) { + throw new IllegalArgumentException("Index field '" + fieldName + "' not found."); + } + fieldType = field.getDataType(); + } else if (exp instanceof SummaryExpression) { + Field field = search.getSummaryField(fieldName); + if (field == null) { + throw new IllegalArgumentException("Summary field '" + fieldName + "' not found."); + } + fieldType = field.getDataType(); + } else { + throw new UnsupportedOperationException(); + } + } + return requiresTransform(exp, fieldType); + } + + @Override + protected final Expression newTransform() { + return newTransform(fieldType); + } + + protected abstract boolean requiresTransform(Expression exp, DataType fieldType); + + protected abstract Expression newTransform(DataType fieldType); +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java new file mode 100644 index 00000000000..8f9b88b0268 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.document.*; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.Arrays; +import java.util.List; + +/** + * @author balder + */ +public class UriHack extends Processor { + private static final List<String> URL_SUFFIX = + Arrays.asList("scheme", "host", "port", "path", "query", "fragment", "hostname"); + public UriHack(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField field : search.allFieldsList()) { + if (field.doesIndexing()) { + DataType fieldType = field.getDataType(); + if (fieldType instanceof CollectionDataType) { + fieldType = ((CollectionDataType)fieldType).getNestedType(); + } + if (fieldType == DataType.URI) { + processField(search, field); + } + } + } + } + + private void processField(Search search, SDField uriField) { + String uriName = uriField.getName(); + uriField.setStemming(Stemming.NONE); + DataType generatedType = DataType.STRING; + if (uriField.getDataType() instanceof ArrayDataType) { + generatedType = new ArrayDataType(DataType.STRING); + } else if (uriField.getDataType() instanceof WeightedSetDataType) { + WeightedSetDataType wdt = (WeightedSetDataType) uriField.getDataType(); + generatedType = new WeightedSetDataType(DataType.STRING, wdt.createIfNonExistent(), wdt.removeIfZero()); + } + for (String suffix : URL_SUFFIX) { + String partName = uriName + "." + suffix; + // I wonder if this is explicit in qrs or implicit in backend? + // search.addFieldSetItem(uriName, partName); + SDField partField = new SDField(partName, generatedType, true); + partField.setIndexStructureField(uriField.doesIndexing()); + partField.setRankType(uriField.getRankType()); + partField.setStemming(Stemming.NONE); + partField.getNormalizing().inferLowercase(); + if (uriField.getIndex(suffix) != null) { + partField.addIndex(uriField.getIndex(suffix)); + } + search.addExtraField(partField); + search.fieldSets().addBuiltInFieldSetItem(BuiltInFieldSets.INTERNAL_FIELDSET_NAME, partField.getName()); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java new file mode 100644 index 00000000000..b01a82b0131 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * @author bratseth + */ +public class UrlFieldValidator extends Processor { + + public UrlFieldValidator(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + for (SDField field : search.allFieldsList()) { + if ( ! field.getDataType().equals(DataType.URI)) continue; + + if (field.doesAttributing()) + throw new IllegalArgumentException("Error in " + field + " in " + search + ": " + + "uri type fields cannot be attributes"); + } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java new file mode 100644 index 00000000000..38c7338be3c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.document.DataType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.documentmodel.DocumentSummary; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.HashMap; +import java.util.Map; + +/** + * This Processor checks to make sure all fields with the same name have the same {@link DataType}. This check + * explicitly disregards whether a field is an index field, an attribute or a summary field. This is a requirement if we + * hope to move to a model where index fields, attributes and summary fields share a common field class. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class ValidateFieldTypes extends Processor { + + public ValidateFieldTypes(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + String searchName = search.getName(); + Map<String, DataType> fieldTypes = new HashMap<>(); + for (SDField field : search.allFieldsList()) { + checkFieldType(searchName, "index field", field.getName(), field.getDataType(), fieldTypes); + for (Map.Entry<String, Attribute> entry : field.getAttributes().entrySet()) { + checkFieldType(searchName, "attribute", entry.getKey(), entry.getValue().getDataType(), fieldTypes); + } + } + for (DocumentSummary summary : search.getSummaries().values()) { + for (SummaryField field : summary.getSummaryFields()) { + checkFieldType(searchName, "summary field", field.getName(), field.getDataType(), fieldTypes); + } + } + } + + private void checkFieldType(String searchName, String fieldDesc, String fieldName, DataType fieldType, + Map<String, DataType> fieldTypes) + { + DataType prevType = fieldTypes.get(fieldName); + if (prevType == null) { + fieldTypes.put(fieldName, fieldType); + } else if (!equalTypes(prevType, fieldType)) { + throw newProcessException(searchName, fieldName, "Duplicate field name with different types. Expected " + prevType.getName() + " for " + fieldDesc + + " '" + fieldName + "', got " + fieldType.getName() + "."); + } + } + + private boolean equalTypes(DataType d1, DataType d2) { + if ("tag".equals(d1.getName())) { + return "tag".equals(d2.getName()) || "WeightedSet<string>".equals(d2.getName()); + } + if ("tag".equals(d2.getName())) { + return "tag".equals(d1.getName()) || "WeightedSet<string>".equals(d1.getName()); + } + return d1.equals(d2); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java new file mode 100644 index 00000000000..a5c7d25532d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.Ranking; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * Check that fields with index settings actually creates an index or attribute + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class ValidateFieldWithIndexSettingsCreatesIndex extends Processor { + + public ValidateFieldWithIndexSettingsCreatesIndex(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + Matching defaultMatching = new Matching(); + Ranking defaultRanking = new Ranking(); + for (SDField field : search.allFieldsList()) { + if (field.doesIndexing()) { + continue; + } + if (field.doesAttributing()) { + continue; + } + if (!(field.getRanking().equals(defaultRanking))) { + fail(search, field, "Fields which are not creating an index or attribute can not contain rank settings."); + } + if (!(field.getMatching().equals(defaultMatching))) { + fail(search, field, "Fields which are not creating an index or attribute can not contain match settings."); + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java new file mode 100644 index 00000000000..24b811fc7fc --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +/** + * The implementation of word matching - with word matching the field is assumed to contain a single "word" - some + * contiguous sequence of word and number characters - but without changing the data at the indexing side (as with text + * matching) to enforce this. Word matching is thus almost like exact matching on the indexing side (no action taken), + * and like text matching on the query side. This may be suitable for attributes, where people both expect the data to + * be left as in the input document, and trivially written queries to work by default. However, this may easily lead to + * data which cannot be matched at all as the indexing and query side does not agree. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public class WordMatch extends Processor { + + public WordMatch(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + public void process() { + for (SDField field : search.allFieldsList()) { + if (!field.getMatching().getType().equals(Matching.Type.WORD)) { + continue; + } + field.setStemming(Stemming.NONE); + field.getNormalizing().inferLowercase(); + field.addQueryCommand("word"); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/IndexCommandResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/IndexCommandResolver.java new file mode 100644 index 00000000000..158515fd05e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/IndexCommandResolver.java @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing.multifieldresolver; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.Search; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.logging.Level; + +/** + * Resolver-class for harmonizing index-commands in multifield indexes + */ +public class IndexCommandResolver extends MultiFieldResolver { + + /** Commands which don't have to be harmonized between fields */ + private static List<String> ignoredCommands = new ArrayList<>(); + + /** Commands which must be harmonized between fields */ + private static List<String> harmonizedCommands = new ArrayList<>(); + + static { + String[] ignore = { "complete-boost", "literal-boost", "highlight" }; + ignoredCommands.addAll(Arrays.asList(ignore)); + String[] harmonize = { "stemming", "normalizing" }; + harmonizedCommands.addAll(Arrays.asList(harmonize)); + } + + public IndexCommandResolver(String indexName, List<SDField> fields, Search search, DeployLogger logger) { + super(indexName, fields, search, logger); + } + + /** + * Check index-commands for each field, report and attempt to fix any + * inconsistencies + */ + public void resolve() { + for (SDField field : fields) { + for (String command : field.getQueryCommands()) { + if (!ignoredCommands.contains(command)) + checkCommand(command); + } + } + } + + private void checkCommand(String command) { + for (SDField field : fields) { + if (!field.hasQueryCommand(command)) { + if (harmonizedCommands.contains(command)) { + deployLogger.log(Level.WARNING, command + " must be added to all fields going to the same index (" + indexName + ")" + + ", adding to field " + field.getName()); + field.addQueryCommand(command); + } else { + deployLogger.log(Level.WARNING, "All fields going to the same index should have the same query-commands. Field \'" + field.getName() + + "\' doesn't contain command \'" + command+"\'"); + } + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/MultiFieldResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/MultiFieldResolver.java new file mode 100644 index 00000000000..943a788fc75 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/MultiFieldResolver.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing.multifieldresolver; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.Search; +import java.util.List; + +/** + * Abstract superclass of all multifield conflict resolvers + */ +public abstract class MultiFieldResolver { + + protected String indexName; + protected List<SDField> fields; + protected Search search; + + protected DeployLogger deployLogger; + + public MultiFieldResolver(String indexName, List<SDField> fields, Search search, DeployLogger logger) { + this.indexName = indexName; + this.fields = fields; + this.search = search; + this.deployLogger = logger; + } + + /** + * Checks the list of fields for specific conflicts, and reports and/or + * attempts to correct them + */ + public abstract void resolve(); + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java new file mode 100644 index 00000000000..d98dd97ff83 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java @@ -0,0 +1,85 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing.multifieldresolver; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.FieldType; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.types.TensorFieldType; +import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.processing.Processor; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Class that processes a search instance and sets type settings on all rank profiles. + * + * Currently, type settings are limited to the type of tensor attribute fields and tensor query features. + * + * @author <a href="geirst@yahoo-inc.com">Geir Storli</a> + */ +public class RankProfileTypeSettingsProcessor extends Processor { + + private static final Pattern queryFeaturePattern = Pattern.compile("query\\((\\w+)\\)$"); + + public RankProfileTypeSettingsProcessor(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process() { + processAttributeFields(); + processQueryProfileTypes(); + + } + + private void processAttributeFields() { + for (SDField field : search.allFieldsList()) { + Attribute attribute = field.getAttributes().get(field.getName()); + if (attribute != null && attribute.tensorType().isPresent()) { + addAttributeTypeToRankProfiles(attribute.getName(), attribute.tensorType().get().toString()); + } + } + } + + private void addAttributeTypeToRankProfiles(String attributeName, String attributeType) { + for (RankProfile profile : rankProfileRegistry.allRankProfiles()) { + profile.addAttributeType(attributeName, attributeType); + } + } + + private void processQueryProfileTypes() { + for (QueryProfileType queryProfileType : queryProfiles.getRegistry().getTypeRegistry().allComponents()) { + for (Map.Entry<String, FieldDescription> fieldDescEntry : queryProfileType.fields().entrySet()) { + processFieldDescription(fieldDescEntry.getValue()); + } + } + } + + private void processFieldDescription(FieldDescription fieldDescription) { + String fieldName = fieldDescription.getName(); + FieldType fieldType = fieldDescription.getType(); + if (fieldType instanceof TensorFieldType) { + TensorFieldType tensorFieldType = (TensorFieldType)fieldType; + Matcher matcher = queryFeaturePattern.matcher(fieldName); + if (tensorFieldType.type().isPresent() && matcher.matches()) { + String queryFeature = matcher.group(1); + addQueryFeatureTypeToRankProfiles(queryFeature, tensorFieldType.type().get().toString()); + } + } + } + + private void addQueryFeatureTypeToRankProfiles(String queryFeature, String queryFeatureType) { + for (RankProfile profile : rankProfileRegistry.allRankProfiles()) { + profile.addQueryFeatureType(queryFeature, queryFeatureType); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankTypeResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankTypeResolver.java new file mode 100644 index 00000000000..787a70862a9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankTypeResolver.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing.multifieldresolver; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.searchdefinition.document.RankType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.Search; + +import java.util.List; +import java.util.logging.Level; + +/** + * Checks if fields have defined different rank types for the same + * index (typically in an index-to statement), and if they have + * output a warning and use the first ranktype. + * + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class RankTypeResolver extends MultiFieldResolver { + + public RankTypeResolver(String indexName, List<SDField> fields, Search search, DeployLogger logger) { + super(indexName, fields, search, logger); + } + + public void resolve() { + RankType rankType = null; + if (fields.size() > 0) { + boolean first = true; + for (SDField field : fields) { + if (first) { + rankType = fields.get(0).getRankType(); + first = false; + } else if (!field.getRankType().equals(rankType)) { + deployLogger.log(Level.WARNING, "In field '" + field.getName() + "' " + + field.getRankType() + " for index '" + indexName + + "' conflicts with " + rankType + + " defined for the same index in field '" + + field.getName() + "'. Using " + + rankType + "."); + field.setRankType(rankType); + } + } + } + } +} + diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/StemmingResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/StemmingResolver.java new file mode 100644 index 00000000000..f1e1899391d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/StemmingResolver.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.processing.multifieldresolver; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.language.Linguistics; +import com.yahoo.searchdefinition.Index; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.searchdefinition.processing.BuiltInFieldSets; +import com.yahoo.vespa.indexinglanguage.expressions.*; +import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; + +import java.util.List; +import java.util.logging.Level; + +/** + * Class resolving conflicts when fields with different stemming-settings are + * combined into the same index + */ +public class StemmingResolver extends MultiFieldResolver { + + public StemmingResolver(String indexName, List<SDField> fields, Search search, DeployLogger logger) { + super(indexName, fields, search, logger); + } + + @Override + public void resolve() { + checkStemmingForIndexFields(indexName, fields); + } + + private void checkStemmingForIndexFields(String indexName, List<SDField> fields) { + Stemming stemming = null; + SDField stemmingField = null; + for (SDField field : fields) { + if (stemming == null && stemmingField==null) { + stemming = field.getStemming(search); + stemmingField = field; + } else if (stemming != field.getStemming(search)) { + deployLogger.log(Level.WARNING, "Field '" + field.getName() + "' has " + field.getStemming(search) + + ", whereas field '" + stemmingField.getName() + "' has " + stemming + + ". All fields indexing to the index '" + indexName + "' must have the same stemming." + + " This should be corrected as it will make indexing fail in a few cases."); + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/package-info.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/package-info.java new file mode 100644 index 00000000000..9915e11b28d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/package-info.java @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Classes in this package (processors) implements some search + * definition features by reducing them to simpler features. + * The processors are run after parsing of the search definition, + * before creating the derived model. + * + * For simplicity, features should always be implemented here + * rather than in the derived model if possible. + * + * New processors must be added to the list in Processing. + */ +@com.yahoo.api.annotations.PackageMarker +package com.yahoo.searchdefinition.processing; diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java new file mode 100644 index 00000000000..29c0c4e8136 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java @@ -0,0 +1,156 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.configmodel.producers; + +import com.yahoo.document.config.DocumentmanagerConfig; +import static com.yahoo.document.config.DocumentmanagerConfig.*; +import com.yahoo.document.*; +import com.yahoo.document.annotation.AnnotationReferenceDataType; +import com.yahoo.document.annotation.AnnotationType; +import com.yahoo.documentmodel.DataTypeCollection; +import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.documentmodel.VespaDocumentType; +import com.yahoo.searchdefinition.document.FieldSet; +import com.yahoo.vespa.documentmodel.DocumentModel; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + +/** + * @author balder + * @since 2010-02-19 + */ +public class DocumentManager { + + public DocumentmanagerConfig.Builder produce(DocumentModel model, DocumentmanagerConfig.Builder docman) { + docman.enablecompression(false); + Set<DataType> handled = new HashSet<>(); + for(NewDocumentType documentType : model.getDocumentManager().getTypes()) { + handle(documentType, docman, handled); + handleAnnotations(documentType.getAnnotations(), docman); + if ( documentType != VespaDocumentType.INSTANCE) { + DocumentmanagerConfig.Datatype.Builder dt = new DocumentmanagerConfig.Datatype.Builder(); + docman.datatype(dt); + handleDataType(documentType, dt); + } + } + return docman; + } + + private void handle(DataTypeCollection type, DocumentmanagerConfig.Builder docman, Set<DataType> handled) { + for (DataType dataType : type.getTypes()) { + if (handled.contains(dataType)) continue; + handled.add(dataType); + if (dataType instanceof TemporaryStructuredDataType) continue; + if ((dataType.getId() < 0) || (DataType.lastPredefinedDataTypeId() < dataType.getId())) { + Datatype.Builder dtc = new Datatype.Builder(); + docman.datatype(dtc); + handleDataType(dataType, dtc); + } + } + } + + private void handleAnnotation(AnnotationType type, DocumentmanagerConfig.Annotationtype.Builder atb) { + atb. + id(type.getId()). + name(type.getName()); + if (type.getDataType() != null) { + atb.datatype(type.getDataType().getId()); + } + if ( ! type.getInheritedTypes().isEmpty()) { + for (AnnotationType inherited : type.getInheritedTypes()) { + atb.inherits(new DocumentmanagerConfig.Annotationtype.Inherits.Builder().id(inherited.getId())); + } + } + } + private void handleAnnotations(Collection<AnnotationType> types, DocumentmanagerConfig.Builder builder) { + for (AnnotationType type : types) { + DocumentmanagerConfig.Annotationtype.Builder atb = new DocumentmanagerConfig.Annotationtype.Builder(); + handleAnnotation(type, atb); + builder.annotationtype(atb); + } + } + + private void handleDataType(DataType type, Datatype.Builder dtc) { + dtc.id(type.getId()); + if (type instanceof ArrayDataType) { + CollectionDataType dt = (CollectionDataType) type; + dtc.arraytype(new Datatype.Arraytype.Builder().datatype(dt.getNestedType().getId())); + } else if (type instanceof WeightedSetDataType) { + WeightedSetDataType dt = (WeightedSetDataType) type; + dtc.weightedsettype(new Datatype.Weightedsettype.Builder(). + datatype(dt.getNestedType().getId()). + createifnonexistant(dt.createIfNonExistent()). + removeifzero(dt.removeIfZero())); + } else if (type instanceof MapDataType) { + MapDataType mtype = (MapDataType) type; + dtc.maptype(new Datatype.Maptype.Builder(). + keytype(mtype.getKeyType().getId()). + valtype(mtype.getValueType().getId())); + } else if (type instanceof DocumentType) { + DocumentType dt = (DocumentType) type; + Datatype.Documenttype.Builder doc = new Datatype.Documenttype.Builder(); + dtc.documenttype(doc); + doc. + name(dt.getName()). + headerstruct(dt.getHeaderType().getId()). + bodystruct(dt.getBodyType().getId()); + for (DocumentType inherited : dt.getInheritedTypes()) { + doc.inherits(new Datatype.Documenttype.Inherits.Builder().name(inherited.getName())); + } + } else if (type instanceof NewDocumentType) { + NewDocumentType dt = (NewDocumentType) type; + Datatype.Documenttype.Builder doc = new Datatype.Documenttype.Builder(); + dtc.documenttype(doc); + doc. + name(dt.getName()). + headerstruct(dt.getHeader().getId()). + bodystruct(dt.getBody().getId()); + for (NewDocumentType inherited : dt.getInherited()) { + doc.inherits(new Datatype.Documenttype.Inherits.Builder().name(inherited.getName())); + } + handleFieldSets(dt.getFieldSets(), doc); + } else if (type instanceof TemporaryStructuredDataType) { + //Ignored + } else if (type instanceof StructDataType) { + StructDataType dt = (StructDataType) type; + Datatype.Structtype.Builder st = new Datatype.Structtype.Builder(); + dtc.structtype(st); + st.name(dt.getName()); + if (dt.getCompressionConfig().type.getCode() != 0) { + st. + compresstype(Datatype.Structtype.Compresstype.Enum.valueOf(dt.getCompressionConfig().type.toString())). + compresslevel(dt.getCompressionConfig().compressionLevel). + compressthreshold((int)dt.getCompressionConfig().threshold). + compressminsize((int)dt.getCompressionConfig().minsize); + } + for (com.yahoo.document.Field field : dt.getFieldsThisTypeOnly()) { + Datatype.Structtype.Field.Builder fb = new Datatype.Structtype.Field.Builder(); + st.field(fb); + fb.name(field.getName()); + if (field.hasForcedId()) { + fb.id(new Datatype.Structtype.Field.Id.Builder().id(field.getId())); + } + fb.datatype(field.getDataType().getId()); + } + for (StructDataType inherited : dt.getInheritedTypes()) { + st.inherits(new Datatype.Structtype.Inherits.Builder().name(inherited.getName())); + } + } else if (type instanceof AnnotationReferenceDataType) { + AnnotationReferenceDataType annotationRef = (AnnotationReferenceDataType) type; + dtc.annotationreftype(new Datatype.Annotationreftype.Builder().annotation(annotationRef.getAnnotationType().getName())); + } else { + throw new IllegalArgumentException("Can not handle datatype '" + type.getName()); + } + } + + private void handleFieldSets(Set<FieldSet> fieldSets, Datatype.Documenttype.Builder doc) { + + for (FieldSet builtinFs : fieldSets) { + handleFieldSet(builtinFs, doc); + } + } + + private void handleFieldSet(FieldSet fs, Datatype.Documenttype.Builder doc) { + doc.fieldsets(fs.getName(), new Datatype.Documenttype.Fieldsets.Builder().fields(fs.getFieldNames())); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java new file mode 100644 index 00000000000..92b3c13d383 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java @@ -0,0 +1,158 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.configmodel.producers; + +import com.yahoo.document.*; +import com.yahoo.document.DocumenttypesConfig.Builder; +import com.yahoo.document.annotation.AnnotationReferenceDataType; +import com.yahoo.document.annotation.AnnotationType; +import com.yahoo.documentmodel.DataTypeCollection; +import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.documentmodel.VespaDocumentType; +import com.yahoo.searchdefinition.FieldSets; +import com.yahoo.searchdefinition.document.FieldSet; +import com.yahoo.vespa.documentmodel.DocumentModel; +import java.util.*; + +/** + * @author balder + */ +public class DocumentTypes { + + public DocumenttypesConfig.Builder produce(DocumentModel model, DocumenttypesConfig.Builder builder) { + Map<NewDocumentType.Name, NewDocumentType> produced = new HashMap<>(); + for(NewDocumentType documentType : model.getDocumentManager().getTypes()) { + produceInheritOrder(documentType, builder, produced); + } + return builder; + } + + private void produceInheritOrder(NewDocumentType documentType, DocumenttypesConfig.Builder builder, Map<NewDocumentType.Name, NewDocumentType> produced) { + if ( ! produced.containsKey(documentType.getFullName())) { + for (NewDocumentType inherited : documentType.getInherited()) { + produceInheritOrder(inherited, builder, produced); + } + handle(documentType, builder); + produced.put(documentType.getFullName(), documentType); + } + } + + private void handle(NewDocumentType documentType, DocumenttypesConfig.Builder builder) { + if (documentType == VespaDocumentType.INSTANCE) { + return; + } + DocumenttypesConfig.Documenttype.Builder db = new DocumenttypesConfig.Documenttype.Builder(); + db. + id(documentType.getId()). + name(documentType.getName()). + headerstruct(documentType.getHeader().getId()). + bodystruct(documentType.getBody().getId()); + Set<Integer> handled = new HashSet<>(); + for (NewDocumentType inherited : documentType.getInherited()) { + db.inherits(new DocumenttypesConfig.Documenttype.Inherits.Builder().id(inherited.getId())); + markAsHandled(handled, inherited.getAllTypes()); + } + for (DataType dt : documentType.getTypes()) { + handle(dt, db, handled); + } + for(AnnotationType annotation : documentType.getAnnotations()) { + DocumenttypesConfig.Documenttype.Annotationtype.Builder atb = new DocumenttypesConfig.Documenttype.Annotationtype.Builder(); + db.annotationtype(atb); + handle(annotation, atb); + } + handleFieldSets(documentType.getFieldSets(), db); + builder.documenttype(db); + } + + private void handleFieldSets(Set<FieldSet> fieldSets, com.yahoo.document.DocumenttypesConfig.Documenttype.Builder db) { + for (FieldSet fs : fieldSets) { + handleFieldSet(fs, db); + } + } + + private void handleFieldSet(FieldSet fs, DocumenttypesConfig.Documenttype.Builder db) { + db.fieldsets(fs.getName(), new DocumenttypesConfig.Documenttype.Fieldsets.Builder().fields(fs.getFieldNames())); + } + + private void markAsHandled(Set<Integer> handled, DataTypeCollection typeCollection) { + for (DataType type : typeCollection.getTypes()) { + handled.add(type.getId()); + } + } + + private void handle(AnnotationType annotation, DocumenttypesConfig.Documenttype.Annotationtype.Builder builder) { + builder. + id(annotation.getId()). + name(annotation.getName()); + DataType dt = annotation.getDataType(); + if (dt!=null) { + builder.datatype(dt.getId()); + } + for (AnnotationType inherited : annotation.getInheritedTypes()) { + builder.inherits(new DocumenttypesConfig.Documenttype.Annotationtype.Inherits.Builder().id(inherited.getId())); + } + } + + private void handle(DataType type, DocumenttypesConfig.Documenttype.Builder db, Set<Integer> handled) { + if ((VespaDocumentType.INSTANCE.getDataType(type.getId()) == null) && ! handled.contains(type.getId())) { + handled.add(type.getId()); + DocumenttypesConfig.Documenttype.Datatype.Builder dtb = new DocumenttypesConfig.Documenttype.Datatype.Builder(); + dtb.id(type.getId()); + if (type instanceof StructDataType) { + dtb.type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("STRUCT")); + StructDataType dt = (StructDataType) type; + DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder sb = new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder(); + dtb.sstruct(sb); + sb.name(dt.getName()); + if (dt.getCompressionConfig().type.getCode() != 0) { + sb.compression(new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Compression.Builder(). + type(DocumenttypesConfig.Documenttype.Datatype.Sstruct.Compression.Type.Enum.valueOf(dt.getCompressionConfig().type.toString())). + level(dt.getCompressionConfig().compressionLevel). + threshold((int)dt.getCompressionConfig().threshold). + minsize((int)dt.getCompressionConfig().minsize)); + } + for (com.yahoo.document.Field field : dt.getFields()) { + sb.field(new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Field.Builder(). + name(field.getName()). + id(field.getId()). + id_v6(field.getIdV6()). + datatype(field.getDataType().getId())); + handle(field.getDataType(), db, handled); + } + } else if (type instanceof ArrayDataType) { + dtb. + type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("ARRAY")). + array(new DocumenttypesConfig.Documenttype.Datatype.Array.Builder(). + element(new DocumenttypesConfig.Documenttype.Datatype.Array.Element.Builder().id(((ArrayDataType)type).getNestedType().getId()))); + handle(((ArrayDataType)type).getNestedType(), db, handled); + } else if (type instanceof WeightedSetDataType) { + dtb.type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("WSET")). + wset(new DocumenttypesConfig.Documenttype.Datatype.Wset.Builder(). + key(new DocumenttypesConfig.Documenttype.Datatype.Wset.Key.Builder(). + id(((WeightedSetDataType)type).getNestedType().getId())). + createifnonexistent(((WeightedSetDataType)type).createIfNonExistent()). + removeifzero(((WeightedSetDataType)type).removeIfZero())); + handle(((WeightedSetDataType)type).getNestedType(), db, handled); + } else if (type instanceof MapDataType) { + dtb. + type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("MAP")). + map(new DocumenttypesConfig.Documenttype.Datatype.Map.Builder(). + key(new DocumenttypesConfig.Documenttype.Datatype.Map.Key.Builder(). + id(((MapDataType)type).getKeyType().getId())). + value(new DocumenttypesConfig.Documenttype.Datatype.Map.Value.Builder(). + id(((MapDataType)type).getValueType().getId()))); + handle(((MapDataType)type).getKeyType(), db, handled); + handle(((MapDataType)type).getValueType(), db, handled); + } else if (type instanceof AnnotationReferenceDataType) { + dtb. + type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.valueOf("ANNOTATIONREF")). + annotationref(new DocumenttypesConfig.Documenttype.Datatype.Annotationref.Builder(). + annotation(new DocumenttypesConfig.Documenttype.Datatype.Annotationref.Annotation.Builder(). + id(((AnnotationReferenceDataType)type).getAnnotationType().getId()))); + } else { + return; + } + db.datatype(dtb); + } + } +} + diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java new file mode 100644 index 00000000000..6c8206d30f2 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentModel.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.documentmodel; + +import com.yahoo.documentmodel.DocumentTypeRepo; + +/** + * DocumentModel represents everything derived from a set of search definitions. + * It contains a document manager managing all defined document types. + * It contains a search manager managing all specified search definitions. + * It contains a storage manager managing all specified storage definitions. + * + * @author balder + * @since 2010-02-19 + */ +public class DocumentModel { + private DocumentTypeRepo documentMan = new DocumentTypeRepo(); + private SearchManager searchMan = new SearchManager(); + + /** + * + * @return Returns the DocumentManager + */ + public DocumentTypeRepo getDocumentManager() { return documentMan; } + + /** + * + * @return Returns the SearchManager + */ + public SearchManager getSearchManager() { return searchMan; } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java new file mode 100644 index 00000000000..42fa7b04cf6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java @@ -0,0 +1,79 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.documentmodel; + +import com.yahoo.document.Field; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; + +/** + * A document summary definition - a list of summary fields. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon S Bratseth</a> + */ +public class DocumentSummary extends FieldView { + + + /** + * Will create a DocumentSummary with the given name. + * @param name The name to use for this summary. + */ + public DocumentSummary(String name) { + super(name); + } + + /** + * The model is constrained to ensure that summary fields of the same name + * in different classes have the same summary transform, because this is + * what is supported by the backend currently. + * @param summaryField The summaryfield to add + */ + public void add(SummaryField summaryField) { + summaryField.addDestination(getName()); + super.add(summaryField); + } + + public SummaryField getSummaryField(String name) { + return (SummaryField) get(name); + } + + public Collection<SummaryField> getSummaryFields() { + ArrayList<SummaryField> fields = new ArrayList<>(getFields().size()); + for(Field f : getFields()) { + fields.add((SummaryField) f); + } + return fields; + } + + /** + * Removes implicit fields which shouldn't be included. + * This is implicitly added fields which are sources for + * other fields. We then assume they are not intended to be added + * implicitly in additon. + * This should be called when this summary is complete. + */ + public void purgeImplicits() { + List<SummaryField> falseImplicits = new ArrayList<>(); + for (SummaryField summaryField : getSummaryFields() ) { + if (summaryField.isImplicit()) continue; + for (Iterator<SummaryField.Source> j = summaryField.sourceIterator(); j.hasNext(); ) { + String sourceName = j.next().getName(); + if (sourceName.equals(summaryField.getName())) continue; + SummaryField sourceField=getSummaryField(sourceName); + if (sourceField==null) continue; + if (!sourceField.isImplicit()) continue; + falseImplicits.add(sourceField); + } + } + for (SummaryField field : falseImplicits) { + remove(field.getName()); + } + } + + public String toString() { + return "document summary '" + getName() + "'"; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/FieldView.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/FieldView.java new file mode 100644 index 00000000000..dfb44aef917 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/FieldView.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.documentmodel; + +import com.yahoo.document.Field; + +import java.io.Serializable; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * @author balder + * @since 2010-02-19 + */ +public class FieldView implements Serializable { + private String name; + private Map<String, Field> fields = new LinkedHashMap<>(); + + /** + * Creates a view with a name + * @param name Name of the view. + */ + public FieldView(String name) { + this.name = name; + } + public String getName() { return name; } + public Collection<Field> getFields() { return fields.values(); } + public Field get(String name) { return fields.get(name); } + public void remove(String name) { fields.remove(name); } + + /** + * This method will add a field to a view. All fields must come from the same document type. Not enforced here. + * @param field The field to add. + * @return Itself for chaining purposes. + */ + public FieldView add(Field field) { + if (fields.containsKey(field.getName())) { + if ( ! fields.get(field.getName()).equals(field)) { + throw new IllegalArgumentException( + "View '" + name + "' already contains a field with name '" + + field.getName() + "' and definition : " + + fields.get(field.getName()).toString() + ". Your is : " + field.toString()); + } + } else { + fields.put(field.getName(), field); + } + return this; + } + + /** + * This method will join the two views. + * @param view The view to be joined in to this. + * @return Itself for chaining. + */ + public FieldView add(FieldView view) { + for(Field field : view.getFields()) { + add(field); + } + return this; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchDef.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchDef.java new file mode 100644 index 00000000000..07b7c973841 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchDef.java @@ -0,0 +1,126 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.documentmodel; + +import com.yahoo.document.DataType; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentTypeManager; + +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Logger; + +/** + * @author balder + * @since 2010-02-19 + */ +public class SearchDef { + private final static Logger log = Logger.getLogger(SearchDef.class.getName()); + /// Name of the searchdefinition + private String name; + /// These are the real backing documenttypes + private DocumentTypeManager sources = new DocumentTypeManager(); + /// Map of all search fields + private Map<String, SearchField> fields = new HashMap<>(); + /// Map of all views that can be searched. + private Map<String, FieldView> views = new HashMap<>(); + /// Map of all aliases <alias, realname> + private Map<String, String> aliases = new HashMap<>(); + + /** + * Will create a SearchDef with the given name + * @param name The name of the searchdefinition + */ + public SearchDef(String name) { + this.name = name; + } + + /** + * This will provide you with the name of the searchdefinition. + * @return The name of the searchdefinition. + */ + public String getName() { return name; } + + public Map<String, SearchField> getFields() { return fields; } + public Map<String, FieldView> getViews() { return views; } + + /** + * Adds a document that can be mapped to this search. + * @param source A document that can be mapped to this search. + * @return Itself for chaining. + */ + public SearchDef add(DataType source) { + sources.register(source); + return this; + } + + private void noShadowing(String name) { + noFieldShadowing(name); + noViewShadowing(name); + } + + private void noFieldShadowing(String name) { + if (fields.containsKey(name)) { + throw new IllegalArgumentException("Searchdef '" + getName() + "' already contains the fields '" + fields.toString() + + "'. You are trying to add '" + name + "'. Shadowing is not supported"); + } + } + + private void noViewShadowing(String name) { + if (views.containsKey(name)) { + throw new IllegalArgumentException("Searchdef '" + getName() + "' already contains a view with name '" + + name + "'. Shadowing is not supported."); + } + } + + /** + * Adds a search field to the definition. + * @param field The field to add. + * @return Itself for chaining. + */ + public SearchDef add(SearchField field) { + try { + noFieldShadowing(field.getName()); + fields.put(field.getName(), field); + } catch (IllegalArgumentException e) { + if (views.containsKey(field.getName())) { + throw e; + } + } + return this; + } + + public SearchDef addAlias(String alias, String aliased) { + noShadowing(alias); + if (!fields.containsKey(aliased) && !views.containsKey(aliased)) { + if (aliased.contains(".")) { + // TODO Here we should nest ourself down to something that really exists. + log.warning("Aliased item '" + aliased + "' not verifiable. Allowing it to be aliased to '" + alias + " for now. Validation will come when URL/Position is structified."); + } else { + throw new IllegalArgumentException("Searchdef '" + getName() + "' has nothing named '" + aliased + "'to alias to '" + alias + "'."); + } + } + String oldAliased = aliases.get(alias); + if ((oldAliased != null)) { + if (oldAliased.equals(aliased)) { + throw new IllegalArgumentException("Searchdef '" + getName() + "' already has the alias '" + alias + + "' to '" + aliased + ". Why do you want to add it again."); + + } else { + throw new IllegalArgumentException("Searchdef '" + getName() + "' already has the alias '" + alias + + "' to '" + oldAliased + ". Cannot change it to alias '" + aliased + "'."); + } + } else { + aliases.put(alias, aliased); + } + return this; + } + + public SearchDef add(FieldView view) { + noViewShadowing(view.getName()); + if (views.containsKey(view.getName())) { + views.get(view.getName()).add(view); + } + views.put(view.getName(), view); + return this; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchField.java new file mode 100644 index 00000000000..2db81861955 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchField.java @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.documentmodel; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author balder + * @since 2010-02-19 + */ +public class SearchField extends Field { + /// Indicate if field shall be stored in memory for attribute usage. + private boolean attribute = false; + /// Indicate if the field is Vespa indexed. + private boolean indexed = false; + /// Indication to backend on how much optimization should be done. + + /** + * This is a representation of features to generate for this field. + * It can be both optimize hints, and real functional hints. + */ + public enum Feature { + WEIGHT_IN_ATTRIBUTE_POSTINGS("WeightInAttributePosting"), // Hint to put the weight in postings for attribute. + WORDPOS_IN_POSTINGS("WordPosInPosting"), // Default for generating posocc + FILTER_ONLY("FilterOnly"); // Might only generate bitvector + private String name; + Feature(String name) { this.name = name;} + public String getName() { return name; } + } + private List<Feature> featureList = new ArrayList<>(); + + public SearchField(Field field, boolean indexed, boolean attribute) { + this(field, indexed, attribute, null); + } + public SearchField(Field field, boolean indexed, boolean attribute, List<Feature> features) { + super(field.getName(), field); + this.attribute = attribute; + this.indexed = indexed; + if (features != null) { + featureList.addAll(features); + } + validate(); + } + + @SuppressWarnings({ "deprecation" }) + private void validate() { + if (attribute || !indexed) { + return; + } + DataType fieldType = getDataType(); + DataType primiType = fieldType.getPrimitiveType(); + if (DataType.STRING.equals(primiType) || DataType.URI.equals(primiType)) { + return; + } + throw new IllegalStateException("Expected type " + DataType.STRING.getName() + " for indexed field '" + + getName() + "', got " + fieldType.getName() + "."); + } + + public SearchField setIndexed() { indexed = true; validate(); return this; } + public SearchField setAttribute() { attribute = true; validate(); return this; } + public boolean isAttribute() { return attribute; } + /** + * True if field is Vespa indexed + * @return true if indexed + */ + public boolean isIndexed() { return indexed; } + public SearchField addFeature(Feature feature) { featureList.add(feature); validate(); return this; } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchManager.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchManager.java new file mode 100644 index 00000000000..29a960f7e7b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SearchManager.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.documentmodel; + +import java.util.TreeMap; + +/** + * @author balder + * @since 2010-02-19 + */ +public class SearchManager { + /// This is the list of all known search definitions + private TreeMap<String, SearchDef> defs = new TreeMap<>(); + + /** + * This will add a searchdefinition or throw an IllegalArgumentException if the name is already used + * @param def The searchdef to add + * @return itself for chaining purposes. + */ + public SearchManager add(SearchDef def) { + if (defs.containsKey(def.getName())) { + throw new IllegalArgumentException("There already exist a searchdefinition with this content:\n" + + defs.get(def.getName()).toString() + "\n No room for : " + def.toString()); + } + defs.put(def.getName(), def); + return this; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java new file mode 100644 index 00000000000..f6db82785b0 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java @@ -0,0 +1,350 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.documentmodel; + +import com.yahoo.document.DataType; +import com.yahoo.document.Field; +import com.yahoo.searchdefinition.document.TypedKey; + +import java.io.Serializable; +import java.util.*; + +import static com.yahoo.text.Lowercase.toLowerCase; + +/** + * A summary field + * + * @author bratseth + */ +public class SummaryField extends Field implements Cloneable, TypedKey { + + /** + * This class represents a source (field name) and the type of the source (only used for smart summary) + */ + public static class Source implements Serializable { + public static enum Type { + CONTEXTUAL("contextual"), + TITLE("title"), + STATIC("static"), + URL("url"); + private final String name; + Type(String name) { + this.name = name; + } + public String getName() { return name; } + } + private String name; + private Type type; + private boolean override = false; + public Source(String name) { + this.name = name; + this.type = Type.CONTEXTUAL; + } + public Source(String name, Type type) { + this.name = name; + this.type = type; + } + public String getName() { return name; } + public Type getType() { return type; } + public void setOverride(boolean override) { this.override = override; } + public boolean getOverride() { return override; } + public int hashCode() { + return name.hashCode() + type.getName().hashCode() + Boolean.valueOf(override).hashCode(); + } + public boolean equals(Object obj) { + if (!(obj instanceof Source)) { + return false; + } + Source other = (Source)obj; + return name.equals(other.name) && + type.getName().equals(other.type.getName()) && + override == other.override; + } + public String toString() { + return name; + } + } + + /** A name-value property (used for smart summary) */ + public static class Property implements Serializable { + private String name; + private String value; + public Property(String name, String value) { + this.name = name; + this.value = value; + } + public String getName() { return name; } + public String getValue() { return value; } + public @Override int hashCode() { + return name.hashCode() + 17*value.hashCode(); + } + public @Override boolean equals(Object obj) { + if (!(obj instanceof Property)) { + return false; + } + Property other = (Property)obj; + return name.equals(other.name) && value.equals(other.value); + } + } + + /** The transform to perform on the stored source */ + private SummaryTransform transform=SummaryTransform.NONE; + + /** The command used per field in vsmsummary */ + private VsmCommand vsmCommand = VsmCommand.NONE; + + /** + * The data sources for this output summary field, in prioritized order + * (use only second source if first yields no result after transformation + * and so on). If no sources are given explicitly, the field of the same + * name as this summary field is used + */ + private Set<Source> sources = new java.util.LinkedHashSet<>(); + + private Set<String> destinations=new java.util.LinkedHashSet<>(); + + /** True if this field was defined implicitly */ + private boolean implicit=false; + + /** The list of properties for this summary field */ + private List<Property> properties = new ArrayList<>(); + + /** Creates a summary field with NONE as transform */ + public SummaryField(String name, DataType type) { + this(name,type,SummaryTransform.NONE); + } + + /** Creates a summary field with NONE as transform */ + public SummaryField(Field field) { + this(field,SummaryTransform.NONE); + } + + + public SummaryField(Field field,SummaryTransform transform) { + this(field.getName(), field.getDataType(), transform); + } + + public SummaryField(String name,DataType type,SummaryTransform transform) { + super(name, type); + this.transform=transform; + } + + public void setImplicit(boolean implicit) { this.implicit=implicit; } + + @Override // override to make public + public void setDataType(DataType type) { + super.setDataType(type); + } + + public boolean isImplicit() { return implicit; } + + public void setTransform(SummaryTransform transform) { + this.transform=transform; + if (SummaryTransform.DYNAMICTEASER.equals(transform) || SummaryTransform.BOLDED.equals(transform)) { + // This is the kind of logic we want to have in processing, + // but can't because of deriveDocuments mode, which doesn't run + // processing. + setVsmCommand(VsmCommand.FLATTENJUNIPER); + } + } + + public SummaryTransform getTransform() { return transform; } + + /** Returns the first source field of this, or null if the source field is not present */ + public String getSourceField() { + String sourceName=getName(); + if (sources.size()>0) + sourceName=sources.iterator().next().getName(); + return sourceName; + } + + public void addSource(String name) { + sources.add(new Source(name)); + } + + public void addSource(Source source) { + sources.add(source); + } + + public Iterator<Source> sourceIterator() { + return sources.iterator(); + } + + public int getSourceCount() { + return sources.size(); + } + + /** Returns a modifiable set of the sources of this */ + public Set<Source> getSources() { return sources; } + + /** Returns the first source name of this, or the field name if no source has been set */ + public String getSingleSource() { + if (sources.size()==0) return getName(); + return sources.iterator().next().getName(); + } + + public void addDestination(String name) { + destinations.add(name); + } + + public final void addDestinations(Iterable<String> names) { + for (String name : names) { + addDestination(name); + } + } + + /** Returns an modifiable view of the destination set owned by this */ + public Set<String> getDestinations() { + return destinations; + } + + private String toString(Collection<?> collection) { + StringBuffer buffer=new StringBuffer(); + for (Iterator<?> i=collection.iterator(); i.hasNext(); ) { + buffer.append(i.next().toString()); + if (i.hasNext()) + buffer.append(", "); + } + return buffer.toString(); + } + + /** + * Returns a summary field which merges the settings in the given field + * into this field + * + * @param merge the field to merge with this, if null, the merged field is + * <code>this</code> field + * @throws RuntimeException if the two fields can not be merged + */ + public SummaryField mergeWith(SummaryField merge) { + if (merge==null) return this; + if (this.isImplicit()) return merge; + if (merge.isImplicit()) return this; + + if (!merge.getName().equals(getName())) + throw new IllegalArgumentException(merge + " conflicts with " + this + + ": different names"); + + if (!merge.getTransform().equals(getTransform())) + throw new IllegalArgumentException(merge + " conflicts with " + this + + ": different transforms"); + + if (!merge.getDataType().equals(getDataType())) + throw new IllegalArgumentException(merge + " conflicts with " + this + + ": different types"); + + if (!merge.isImplicit()) + setImplicit(false); + + if (isHeadOf(this.sourceIterator(),merge.sourceIterator())) { + // Ok + } + else if (isHeadOf(merge.sourceIterator(),this.sourceIterator())) { + sources=new LinkedHashSet<>(merge.sources); + } + else { + throw new IllegalArgumentException(merge + " conflicts with " + this + + ": on source list must be the start of the other"); + } + + destinations.addAll(merge.destinations); + + return this; + } + + public boolean hasSource(String name) { + for (Source s : sources) { + if (s.getName().equals(name)) { + return true; + } + } + return false; + } + + /** + * Returns true if the second list is the start of the first list + */ + private boolean isHeadOf(Iterator<?> full, Iterator<?> head) { + while (head.hasNext()) { + if (!full.hasNext()) return false; + + if (!full.next().equals(head.next())) return false; + } + return true; + } + + private String getDestinationString() + { + StringBuilder destinationString = new StringBuilder("destinations("); + for (String destination : destinations) { + destinationString.append(destination).append(" "); + } + destinationString.append(")"); + return destinationString.toString(); + } + + public String toString() { + return + "summary field '" + getName() + ' ' + getDestinationString() + + "' [type: '" + getDataType().getName() + + "' transform: '" + transform + + "', source: '" + toString(sources) + + "', to '" + toString(destinations) + "']"; + } + + /** returns a string which aids locating this field in the source search definition */ + public String toLocateString() { + return "'summary " + getName() + " type " + toLowerCase(getDataType().getName()) + "' in '" + getDestinationString() + "'"; + } + + public SummaryField clone() { + try { + SummaryField clone=(SummaryField)super.clone(); + if (this.sources!=null) + clone.sources=new LinkedHashSet<>(this.sources); + if (this.destinations!=null) + clone.destinations=new LinkedHashSet<>(destinations); + return clone; + } + catch (CloneNotSupportedException e) { + throw new RuntimeException("Programming error"); + } + } + + public VsmCommand getVsmCommand() { + return vsmCommand; + } + + public void setVsmCommand(VsmCommand vsmCommand) { + this.vsmCommand = vsmCommand; + } + + /** Adds a property to this summary field */ + public void addProperty(String name, String value) { + properties.add(new Property(name, value)); + } + + public List<Property> getProperties() { + return properties; + } + + /** + * The command used when using data from this SummaryField to generate StreamingSummary config (vsmsummary). + * Not used for ordinary Summary config. + * @author vegardh + * + */ + public enum VsmCommand { + NONE("NONE"), + FLATTENSPACE("FLATTENSPACE"), + FLATTENJUNIPER("FLATTENJUNIPER"); + + private String cmd=""; + private VsmCommand(String cmd) { + this.cmd=cmd; + } + @Override + public String toString() { + return cmd; + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java new file mode 100644 index 00000000000..05092d50951 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java @@ -0,0 +1,99 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.documentmodel; + +/** + * A value class representing a search time + * transformation on a summary field. + * + * @author bratseth + */ +public enum SummaryTransform { + + NONE("none"), + ATTRIBUTE("attribute"), + BOLDED("bolded"), + DISTANCE("distance"), + DYNAMICBOLDED("dynamicbolded"), + DYNAMICTEASER("dynamicteaser"), + POSITIONS("positions"), + RANKFEATURES("rankfeatures"), + SUMMARYFEATURES("summaryfeatures"), + TEXTEXTRACTOR("textextractor"), + GEOPOS("geopos"); + + private String name; + + private SummaryTransform(String name) { + this.name=name; + } + + public String getName() { + return name; + } + + /** Returns the bolded version of this transform if possible, throws if not */ + public SummaryTransform bold() { + switch (this) { + case NONE: + case BOLDED: + return BOLDED; + + case DYNAMICBOLDED: + case DYNAMICTEASER: + return DYNAMICBOLDED; + + default: + throw new IllegalArgumentException("Can not bold a '" + this + "' field."); + } + } + + /** Returns the unbolded version of this transform */ + public SummaryTransform unbold() { + switch (this) { + case NONE: + case BOLDED: + return NONE; + + case DYNAMICBOLDED: + return DYNAMICTEASER; + + default: + return this; + } + } + + /** Returns whether this value is bolded */ + public boolean isBolded() { + return this==BOLDED || this==DYNAMICBOLDED; + } + + /** Whether this is dynamically generated, both teasers and bolded fields are dynamic */ + public boolean isDynamic() { + return this==BOLDED || this==DYNAMICBOLDED || this==DYNAMICTEASER; + } + + /** Returns whether this is a teaser, not the complete field value */ + public boolean isTeaser() { + return this==DYNAMICBOLDED || this==DYNAMICTEASER; + } + + /** Returns whether this transform always gets its value by accessing memory only */ + public boolean isInMemory() { + switch (this) { + case ATTRIBUTE: + case DISTANCE: + case POSITIONS: + case GEOPOS: + case RANKFEATURES: + case SUMMARYFEATURES: + return true; + + default: + return false; + } + } + + public String toString() { + return name; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java new file mode 100644 index 00000000000..13d85c3a955 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -0,0 +1,516 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +import com.yahoo.config.FileReference; +import com.yahoo.config.model.api.PortInfo; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.filedistribution.PathDoesNotExistException; + +import java.util.*; + +import static com.yahoo.text.Lowercase.toLowerCase; + +/** + * Superclass for all Processes. + * + * @author gjoranv + */ +public abstract class AbstractService extends AbstractConfigProducer<AbstractConfigProducer<?>> implements Service { + + private static final long serialVersionUID = 1L; + + // The physical host this Service runs on. + private HostResource hostResource = null; + + /** + * Identifier that ensures that multiple instances of the same + * Service subclass will have unique names on the host. The first + * instance of one kind of Service will have the id 1, and the id + * will increase by 1 for each new instance. + * TODO: Do something more intelligent in Host? + */ + private int id = 0; + + /** The actual base port for this Service. */ + private int basePort; + + /** The ports allocated to this Service. */ + private List<Integer> ports = new ArrayList<>(); + + /** The optional JVM execution args for this Service. */ + // Please keep non-null, as passed to command line in service startup + private String jvmArgs = ""; + + /** The optional PRELOAD libraries for this Service. */ + // Please keep non-null, as passed to command line in service startup + private String preload = Defaults.getDefaults().vespaHome() + "lib64/vespa/malloc/libvespamalloc.so"; + + // If larger or equal to 0 it mean that explicit mmaps shall not be included in coredump. + private long mmapNoCoreLimit = -1l; + + /** The ports metainfo object */ + protected PortsMeta portsMeta = new PortsMeta(); + + /** + * Custom properties that a service may populate to communicate + * more key/value pairs to the service-list dump. + * Supported key datatypes are String, and values may be String or Integer. + */ + private HashMap<String, Object> serviceProperties = new LinkedHashMap<>(); + + /** The affinity properties of this service. */ + private Optional<Affinity> affinity = Optional.empty(); + + private boolean initialized = false; + + /** + * Preferred constructor when building from XML. Use this if you are building + * in doBuild() in an AbstractConfigProducerBuilder. + * build() will call initService() in that case, after setting hostalias and baseport. + * @param parent Parent config producer in the model tree. + * @param name Name of this service. + */ + public AbstractService(AbstractConfigProducer parent, String name) { + super(parent, name); + } + + /** + * Only used for testing. Stay away. + * @param name Name of this service. + */ + public AbstractService(String name) { + super(name); + } + + /** + * Distribute affinity on a collection of services. Services that are located on the same host + * will be assigned a specific cpu socket on that host. + * + * @param services A {@link Collection} of services of the same type, not necessarily on the same host. + */ + public static <SERVICE extends AbstractService> void distributeCpuSocketAffinity(Collection<SERVICE> services) { + Map<HostResource, List<SERVICE>> affinityMap = new HashMap<>(); + for (SERVICE service : services) { + if (!affinityMap.containsKey(service.getHostResource())) { + affinityMap.put(service.getHostResource(), new ArrayList<SERVICE>()); + } + int cpuSocket = affinityMap.get(service.getHostResource()).size(); + affinityMap.get(service.getHostResource()).add(service); + service.setAffinity(new Affinity.Builder().cpuSocket(cpuSocket).build()); + } + } + + /** + * Helper method to avoid replicating code. + * + * @param hostResource The physical host on which this service should run. + * @param userPort The wanted port given by the user. + */ + private void initService(HostResource hostResource, int userPort) { + if (initialized) { + throw new IllegalStateException("Service '" + getConfigId() + "' already initialized."); + } + if (hostResource == null) { + throw new RuntimeException("No host found for service '" + getServiceName() + "'. " + + "The hostalias is probably missing from hosts.xml."); + } + id = getIndex(hostResource); + ports = hostResource.allocateService(this, getInstanceWantedPort(userPort)); + initialized = true; + } + + /** + * Called by builder class which has not given the host or port in a constructor, hence + * initService is not yet run for this. + */ + public void initService() { + initService(this.hostResource, this.basePort); + } + + /** + * Returns the desired base port for the first instance of the + * service type. Returns '0' as default, which means that the + * service type should use the default port allocation mechanism. + * + * @return The desired base port for the first instance of the service type. + */ + public int getWantedPort() { + return 0; + } + + /** + * Returns the desired base port for this service instance, '0' if + * it should use the default port allocation mechanism. + * + * @param userWantedPort The wanted port given by the user. + * @return The desired base port for this service instance, '0' by default + */ + private int getInstanceWantedPort(int userWantedPort) { + int wantedPort = 0; + if (userWantedPort == 0) { + if (requiresWantedPort()) + wantedPort = getWantedPort(); + else if (getWantedPort() > 0) + wantedPort = getWantedPort() + ((getId() - 1) * getPortCount()); + } else { + // User defined from spec + wantedPort = userWantedPort; +/* if ((wantedPort >= Host.BASE_PORT) && + (wantedPort <= (Host.BASE_PORT + Host.MAX_PORTS))) { + throw new RuntimeException + ("Attribute 'basePort=" + wantedPort + + "' is not allowed to be inside Vespa's reserved port range " + + Host.BASE_PORT + "-" + + (Host.BASE_PORT + Host.MAX_PORTS) + "."); + } +*/ + } + return wantedPort; + } + + /** + * Override if the desired base port (returned by getWantedPort()) is the only allowed base port. + * + * @return false by default + */ + public boolean requiresWantedPort() { + return false; + } + + /** + * Override if the services does not require consecutive port numbers. I.e. if any ports + * in addition to the baseport should be allocated from Vespa's default port range. + * + * @return true by default + */ + public boolean requiresConsecutivePorts() { + return true; + } + + /** + * Gets the ports metainfo object. The service implementation + * must populate this object in the constructor. + */ + public PortsMeta getPortsMeta() { + return portsMeta; + } + + /** + * Computes and returns the i'th port for this service, based on + * this Service's baseport. + * + * @param i The offset from 'basePort' of the port to return + * @return the i'th port relative to the base port. + * @throws IllegalStateException if i is out of range. + */ + public int getRelativePort(int i) { + if (ports.size() < 1) { + throw new IllegalStateException + ("Requested port with offset " + i + " for service that " + + "has not reserved any ports: " + this); + } + if (i >= ports.size()) { + throw new IllegalStateException + ("Requested port with offset " + i + " for service that " + + "only has reserved " + ports.size() + " ports: " + this); + } + return ports.get(i); + } + + /** + * Must be overridden by services that should be started by + * config-sentinel. The returned value will be used in + * config-sentinel configuration. Returns null by default. + * + * @return null by default. + */ + public String getStartupCommand() { + return null; + } + + public Optional<String> getPreShutdownCommand() { + return Optional.empty(); + } + + /** + * Tells if this service should be autostarted by + * config-sentinel. Returned value will be used to configure the + * config-sentinel. + * + * @return true by default. + */ + public boolean getAutostartFlag() { + return true; + } + + /** + * Tells if this service should be autorestarted by + * config-sentinel. Returned value will be used to configure the + * config-sentinel. + * + * @return true by default. + */ + public boolean getAutorestartFlag() { + return true; + } + + /** + * Returns the name that identifies this service for the config-sentinel. + * + * @return the name that identifies this service for the config-sentinel. + */ + public String getServiceName() { + return getServiceType() + ((id == 1) ? "" : Integer.toString(id)); + } + + /** + * Returns the type of service. This is the class name without the + * package prefix by default. + */ + public String getServiceType() { + return toLowerCase(getShortClassName()); + } + + /** + * Strips the package prefix and returns the short classname. + * + * @return classname without package prefix. + */ + private String getShortClassName() { + Class myClass = getClass(); + Package myPackage = myClass.getPackage(); + return myClass.getName().substring(1 + myPackage.getName().length()); + } + + /** + * @return the physical host on which this service runs. + */ + public Host getHost() { + if (hostResource != null) { + return hostResource.getHost(); + } else { + return null; + } + } + + /** + * @return The hostname on which this service runs. + */ + public String getHostName() { + return hostResource.getHostName(); + } + + /** + * @return The id (index) of this service on the host where it runs + */ + public int getId() { + return id; + } + + /** + * Computes a number that identifies the service on the given + * host. The number of services of the same type (Class) is + * counted and the number is returned. + * + * @param host the host on which the service will run + * @return id number for the given service. + */ + // TODO: Do something more intelligent in the Host class..? + protected int getIndex(HostResource host) { + int i = 0; + for (Service s : host.getServices()) { + //if (s.getClass().equals(getClass()) && (s != this)) { + if (s.getServiceType().equals(getServiceType()) && (s != this)) { + i++; + } + } + return i + 1; + } + + @Override + public ServiceInfo getServiceInfo() { + Set<PortInfo> portInfos = new LinkedHashSet<>(); + for (int i = 0; i < portsMeta.getNumPorts(); i++) { + portInfos.add(new PortInfo(ports.get(i), new LinkedHashSet<>(portsMeta.getTagsAt(i)))); + } + Map<String, String> properties = new LinkedHashMap<>(); + for (Map.Entry<String, Object> prop : serviceProperties.entrySet()) { + properties.put(prop.getKey(), prop.getValue().toString()); + } + return new ServiceInfo(getServiceName(), getServiceType(), portInfos, properties, getConfigId(), getHostName()); + } + + /** + * Sets a service property value for the given key. + * + * @param key a key used for this property + * @param value a String value associated with the key + * @return this service + */ + public AbstractService setProp(String key, String value) { + serviceProperties.put(key, value); + return this; + } + + /** + * Sets a service property value for the given key. + * + * @param key a key used for this property + * @param value an Integer value associated with the key + * @return this service + */ + public AbstractService setProp(String key, Integer value) { + serviceProperties.put(key, value); + return this; + } + + /** + * Gets a service property value mapped to the given key + * as a String, or null if no such key exists. + * + * @param key a key used for lookup in the service properties + * @return the associated String value for the given key, or null + */ + public String getServicePropertyString(String key) { + return getServicePropertyString(key, null); + } + + public String getServicePropertyString(String key, String defStr) { + Object result = serviceProperties.get(key); + return (result == null) ? defStr : result.toString(); + } + + /** Optional execution args for this service */ + public String getJvmArgs() { + return jvmArgs; + } + public void setJvmArgs(String args) { + jvmArgs = (args == null) ? "" : args; + } + public void appendJvmArgs(String args) { + if ((args != null) && ! "".equals(args)) { + setJvmArgs(jvmArgs + getSeparator(jvmArgs) + args); + } + } + private static String getSeparator(String current) { + return ("".equals(current)) ? "" : " "; + } + public void prependJvmArgs(String args) { + if ((args != null) && ! "".equals(args)) { + setJvmArgs(args + getSeparator(jvmArgs) + jvmArgs); + } + } + public String getPreLoad() { return preload; } + public void setPreLoad(String preload) { + this.preload = preload; + } + public long getMMapNoCoreLimit() { return mmapNoCoreLimit; } + public void setMMapNoCoreLimit(long noCoreLimit) { + this.mmapNoCoreLimit = noCoreLimit; + } + + public String getMMapNoCoreEnvVariable() { + return (getMMapNoCoreLimit() >= 0l) + ? "VESPA_MMAP_NOCORE_LIMIT=" + getMMapNoCoreLimit() + " " + : ""; + } + + /** + * WARNING: should only be called before initService(), otherwise call at own risk! + */ + public void setBasePort(int wantedPort) { + if (initialized && getPortCount() > 0) { + // This means initServices has been called already, so you are being nasty, trying to change ports! + // Try to allocate the new ports if they are available. + if (!hostResource.isPortRangeAvailable(wantedPort, 1) || + (requiresConsecutivePorts() && ! hostResource.isPortRangeAvailable(wantedPort, getPortCount()))) + throw new IllegalStateException("For service with id " + getConfigId() + ": setBasePort is called after initService, and ports are not available."); + + hostResource.deallocatePorts(this); + ports = hostResource.allocatePorts(this, getInstanceWantedPort(wantedPort)); + } + + this.basePort = wantedPort; + } + + /** Temporary hack: reserve port with index 0 + * Must be done this way since the system test framework + * currently uses the first port as container http port. + */ + public void reservePortPrepended(int port) { + hostResource.reservePort(this, port); + ports.add(0, port); + } + + public void setHostResource(HostResource hostResource) { + this.hostResource = hostResource; + } + + public boolean isInitialized() { + return initialized; + } + + /** + * Add the given file to the application's file distributor. + * + * @param relativePath path to the file, relative to the app package. + * @return the file reference hash + */ + public FileReference sendFile(String relativePath) { + try { + return getRoot().getFileDistributor().sendFileToHost(relativePath, getHost()); + } catch (PathDoesNotExistException e) { + throw new RuntimeException("File does not exist: '" + relativePath + "'."); + } + } + + /** + * Sets up this service to be included when generating monitoring config. + * The ymon service name used will be {@link #getServiceType()} + */ + public void monitorService() { + monitorService(getServiceType()); + } + + /** + * Sets up this service to be included when generating ymon config. + * @param ymonServiceName the ymon service name to be used + */ + public void monitorService(String ymonServiceName) { + setProp("ymonService", ymonServiceName); + } + + /** + * + * The service HTTP port for health status + * @return portnumber + */ + public int getHealthPort() {return -1;} + + /** + * Overridden by subclasses. List of default dimensions to be added to this services metrics + * @return The default dimensions for this service + */ + public HashMap<String, String> getDefaultMetricDimensions(){ + return new LinkedHashMap<>(); + } + + // For testing + public int getNumPortsAllocated() { + return ports.size(); + } + + public HostResource getHostResource() { + return hostResource; + } + + public Optional<Affinity> getAffinity() { + return affinity; + } + + public void setAffinity(Affinity affinity) { + this.affinity = Optional.ofNullable(affinity); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Affinity.java b/config-model/src/main/java/com/yahoo/vespa/model/Affinity.java new file mode 100644 index 00000000000..d621c299a5d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/Affinity.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +/** + * Represents a set of affinities that can be expressed by a service. Currently only supports + * CPU socket affinity. + * + * @author lulf + * @since 5.12 + */ +public class Affinity { + private final int cpuSocket; + + private Affinity(int cpuSocket) { + this.cpuSocket = cpuSocket; + } + + public int cpuSocket() { + return cpuSocket; + } + + public static Affinity none() { + return new Builder().build(); + } + + public static class Builder { + private int cpuSocket = -1; + public Builder cpuSocket(int cpuSocket) { + this.cpuSocket = cpuSocket; + return this; + } + + public Affinity build() { + return new Affinity(cpuSocket); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Client.java b/config-model/src/main/java/com/yahoo/vespa/model/Client.java new file mode 100644 index 00000000000..15685f5f669 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/Client.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +import com.yahoo.config.model.ApplicationConfigProducerRoot; +import com.yahoo.config.model.producer.AbstractConfigProducer; + +/** + * This is a placeholder config producer that makes global configuration available through a single identifier. This + * is added directly to the {@link ApplicationConfigProducerRoot} producer, and so can be accessed by the simple "client" identifier. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class Client extends AbstractConfigProducer { + + /** + * Constructs a client config producer that is added as a child to + * the given config producer. + * + * @param parent The parent config producer. + */ + public Client(AbstractConfigProducer parent) { + super(parent, "client"); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java new file mode 100644 index 00000000000..852e4e73331 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducer.java @@ -0,0 +1,84 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.util.List; +import java.util.Map; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.ConfigInstance.Builder; +import com.yahoo.config.model.producer.UserConfigRepo; + +/** + * Interface that should be implemented by all config producing modules + * in the vespa model. + * + * @author gjoranv + */ +public interface ConfigProducer extends com.yahoo.config.ConfigInstance.Producer { + + /** + * @return the configId of this ConfigProducer. + */ + public String getConfigId(); + + /** + * @return The one and only HostSystem of the root node + */ + public HostSystem getHostSystem(); + + /** Returns the user configs of this */ + public UserConfigRepo getUserConfigs(); + + /** + * @return this ConfigProducer's children (only 1st level) + */ + public Map<String,? extends ConfigProducer> getChildren(); + + /** + * @return a List of all Services that are descendants to this ConfigProducer + */ + public List<Service> getDescendantServices(); + + /** + * Writes files that need to be written. The files will usually + * only be written when the Vespa model is generated through the + * deploy-application script. + * gv: This is primarily intended for debugging. + * @param directory directory to write files to + * @throws java.io.IOException if writing fails + */ + public void writeFiles(File directory) throws IOException; + + /** + * Dump the three of config producers to the specified stream. + * @param out The stream to print to, e.g. System.out + */ + public void dump(PrintStream out); + + /** + * Build config from this and all parent ConfigProducers, + * such that the root node's config will be added first, and this + * ConfigProducer's config last in the returned builder. + * + * @param builder The builder implemented by the concrete ConfigInstance class + * @return true if a model config producer was found, so config was applied + */ + boolean cascadeConfig(Builder builder); + + /** + * Adds user config override from this ConfigProducer to the existing builder + * + * @param builder The ConfigBuilder to add user config overrides. + * @return true if overrides were added, false if not. + */ + public boolean addUserConfig(ConfigInstance.Builder builder); + + /** + * check constraints depending on the state of the vespamodel graph. + * When overriding, you must invoke super. + */ + public void validate() throws Exception; +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducerRoot.java new file mode 100644 index 00000000000..d0bb89beca7 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProducerRoot.java @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.admin.Admin; +import com.yahoo.vespa.model.filedistribution.FileDistributor; + +import java.util.Set; + + +/** + * Intended to be used as an external interface to the vespa model root. + * + * @author tonytv + */ +public interface ConfigProducerRoot extends ConfigProducer { + + /** + * Adds the given producer (at any depth level) as descendant to this root nodes. + * + * @param id string id of descendant + * @param descendant the producer to add to this root node + */ + void addDescendant(String id, AbstractConfigProducer descendant); + + /** + * @return an unmodifiable copy of the set of configIds in this root. + */ + Set<String> getConfigIds(); + + ConfigInstance.Builder getConfig(ConfigInstance.Builder builder, String configId); + + /** + * Resolves config of the given type and config id. + * @param clazz The type of config + * @param configId The config id + * @return A config instance of the given type + */ + public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> clazz, String configId); + + /** + * Get the global deploy state of this model. + */ + DeployState getDeployState(); + + FileDistributor getFileDistributor(); + + Admin getAdmin(); + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigProxy.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProxy.java new file mode 100644 index 00000000000..22fdca6af50 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigProxy.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +/** + * There is one config proxy running on each Vespa host, and one instance of + * this class is therefore created by each instance of class {@link + * com.yahoo.vespa.model.Host}. + * + * NOTE: The Config proxy is not started by the config system, and + * does not receive any config. It's included here so we know what host + * it runs on, and to give an error message if another service tries + * to reserve the port it is using. + * + * @author Vidar Larsen + * @author Harald Musum + */ +public class ConfigProxy extends AbstractService { + + /** + * Creates a new ConfigProxy instance. + * + * @param host hostname + */ + public ConfigProxy(Host host) { + super(host, "configproxy"); + portsMeta.on(0).tag("rpc").tag("client").tag("status").tag("rpc").tag("admin"); + setProp("clustertype", "hosts"); + setProp("clustername", "admin"); + } + + /** + * Returns the desired base port for this service. + */ + public int getWantedPort() { return 19090; } + + /** + * The desired base port is the only allowed base port. + */ + public boolean requiresWantedPort() { return true; } + + /** + * ConfigProxy needs one rpc client port. + * + * @return The number of ports reserved by the config proxy + */ + public int getPortCount() { return 1; } + + /** + * The config proxy is not started by the config system! + */ + public boolean getAutostartFlag() { return false; } + + /** + * The config proxy is not started by the config system! + */ + public boolean getAutorestartFlag() { return false; } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java b/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java new file mode 100644 index 00000000000..adcb93bc539 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/ConfigSentinel.java @@ -0,0 +1,107 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +import com.yahoo.cloud.config.SentinelConfig; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Zone; + +/** + * There is one config-sentinel running on each Vespa host, and one + * instance of this class is therefore created by each instance of + * class {@link Host}. + * + * @author gjoranv + */ +public class ConfigSentinel extends AbstractService implements SentinelConfig.Producer { + + private final ApplicationId applicationId; + private final Zone zone; + + /** + * Constructs a new ConfigSentinel for the given host. + * + * @param host Physical host on which to run. + */ + public ConfigSentinel(Host host, ApplicationId applicationId, Zone zone) { + super(host, "sentinel"); + this.applicationId = applicationId; + this.zone = zone; + portsMeta.on(0).tag("rpc").tag("notyet"); + portsMeta.on(1).tag("telnet").tag("interactive").tag("http").tag("state"); + setProp("clustertype", "hosts"); + setProp("clustername", "admin"); + } + + /** + * Returns the desired base port for this service. + */ + public int getWantedPort() { return 19097; } + + /** + * The desired base port is the only allowed base port. + */ + public boolean requiresWantedPort() { return true; } + + /** + * @return The number of ports reserved by the Sentinel. + */ + public int getPortCount() { return 2; } + + @Override + public int getHealthPort() {return getRelativePort(1); } + + /** + * Overrides parent method as this is named config-sentinel and not configsentinel all over Vespa + * @return service type for config-sentinel + */ + public String getServiceType(){ + return "config-sentinel"; + } + + @Override + public void getConfig(SentinelConfig.Builder builder) { + builder.application(getApplicationConfig()); + for (Service s : getHostResource().getServices()) { + if (s.getStartupCommand() != null) { + builder.service(getServiceConfig(s)); + } + } + } + + private SentinelConfig.Application.Builder getApplicationConfig() { + SentinelConfig.Application.Builder builder = new SentinelConfig.Application.Builder(); + builder.tenant(applicationId.tenant().value()); + builder.name(applicationId.application().value()); + builder.environment(zone.environment().value()); + builder.region(zone.region().value()); + builder.instance(applicationId.instance().value()); + return builder; + } + + private SentinelConfig.Service.Builder getServiceConfig(Service s) { + SentinelConfig.Service.Builder serviceBuilder = new SentinelConfig.Service.Builder(); + serviceBuilder.command(s.getStartupCommand()); + serviceBuilder.name(s.getServiceName()); + serviceBuilder.autostart(s.getAutostartFlag()); + serviceBuilder.autorestart(s.getAutorestartFlag()); + serviceBuilder.id(s.getConfigId()); + serviceBuilder.affinity(getServiceAffinity(s)); + setPreShutdownCommand(serviceBuilder, s); + return serviceBuilder; + } + + private void setPreShutdownCommand(SentinelConfig.Service.Builder serviceBuilder, Service service) { + if (service.getPreShutdownCommand().isPresent()) { + serviceBuilder.preShutdownCommand(service.getPreShutdownCommand().get()); + } + } + + + private SentinelConfig.Service.Affinity.Builder getServiceAffinity(Service s) { + SentinelConfig.Service.Affinity.Builder builder = new SentinelConfig.Service.Affinity.Builder(); + if (s.getAffinity().isPresent()) { + builder.cpuSocket(s.getAffinity().get().cpuSocket()); + } + return builder; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Host.java b/config-model/src/main/java/com/yahoo/vespa/model/Host.java new file mode 100644 index 00000000000..99109a881a1 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/Host.java @@ -0,0 +1,121 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +import java.io.File; +import java.io.IOException; +import java.net.UnknownHostException; +import java.util.*; +import java.util.logging.Level; +import com.yahoo.cloud.config.SentinelConfig; +import com.yahoo.config.model.producer.AbstractConfigProducer; + +/** + * A physical host, running a set of services. + * The identity of a host is its hostname. Hosts are comparable on their host name. + * + * @author gjoranv + */ +public final class Host extends AbstractConfigProducer<AbstractConfigProducer<?>> implements SentinelConfig.Producer, Comparable<Host> { + + private ConfigSentinel configSentinel = null; + private final String hostname; + private final boolean multitenant; + + /** + * Constructs a new Host instance. + * + * @param parent parent AbstractConfigProducer in the config model. + * @param hostname hostname for this host. + */ + public Host(AbstractConfigProducer parent, String hostname) { + this(parent, hostname, false); + } + + private Host(AbstractConfigProducer parent, String hostname, boolean multitenant) { + super(parent, hostname); + Objects.requireNonNull(hostname, "The host name of a host cannot be null"); + this.multitenant = multitenant; + this.hostname = hostname; + if (parent instanceof HostSystem) { + checkName((HostSystem) parent, hostname); + } + } + + private void checkName(HostSystem parent, String hostname) { + // Give a warning if the host does not exist + if (! parent.getIp(hostname).equals("0.0.0.0")) { + // Host exists - warn if given hostname is not a fully qualified one. + String canonical=hostname; + try { + canonical = parent.getCanonicalHostname(hostname); + } catch (UnknownHostException e) { + deployLogger().log(Level.WARNING, "Unable to find canonical hostname of host: " + hostname); + } + if ((null != canonical) && (! hostname.equals(canonical))) { + deployLogger().log(Level.WARNING, "Host named '" + hostname + "' will not receive any config " + + "since it does not match its canonical hostname: " + canonical); + } + } + } + + public static Host createMultitenantHost(AbstractConfigProducer parent, String hostname) { + return new Host(parent, hostname, true); + } + + // For testing + Host(AbstractConfigProducer parent) { + super(parent, "testhost"); + hostname = "testhost"; + configSentinel = null; + multitenant = false; + } + + public String getHostName() { + return hostname; + } + + public boolean isMultitenant() { + return multitenant; + } + + /** + * Returns the string representation of this Host object. + * @return The string representation of this Host object. + */ + public String toString() { + return "host '" + getHostName() + "'"; + } + + @Override + public void writeFiles(File directory) throws IOException { + } + + @Override + public void getConfig(SentinelConfig.Builder builder) { + // TODO (MAJOR_RELEASE): This shouldn't really be here, but we need to make sure users can upgrade if we change sentinel to use hosts/<hostname>/sentinel instead of hosts/<hostname> + // as config id. We should probably wait for a major release + if (configSentinel != null) { + configSentinel.getConfig(builder); + } + } + + public void setConfigSentinel(ConfigSentinel configSentinel) { + this.configSentinel = configSentinel; + } + + @Override + public int hashCode() { return hostname.hashCode(); } + + @Override + public boolean equals(Object other) { + if (other == this) return true; + if ( ! (other instanceof Host)) return false; + return ((Host)other).hostname.equals(hostname); + } + + @Override + public int compareTo(Host other) { + return this.hostname.compareTo(other.hostname); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java new file mode 100644 index 00000000000..782487ee12c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java @@ -0,0 +1,251 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.model.api.ServiceInfo; + +import java.util.*; +import java.util.logging.Level; +import java.util.stream.Collector; +import java.util.stream.Collectors; + +/** + * A host representation. The identity of this is the identity of its Host. + * TODO: Merge with {@link Host} + * Host resources are ordered by their host order. + * + * @author lulf + * @since 5.12 + */ +public class HostResource implements Comparable<HostResource> { + + public final static int BASE_PORT = 19100; + public final static int MAX_PORTS = 799; + private final Host host; + + // Map from "sentinel name" to service + private final Map<String,Service> services = new LinkedHashMap<>(); + private final Map<Integer, Service> portDB = new LinkedHashMap<>(); + + private int allocatedPorts = 0; + + /** + * Create a new {@link HostResource} bound to a specific {@link com.yahoo.vespa.model.Host}. + * + * @param host {@link com.yahoo.vespa.model.Host} object to bind to. + */ + public HostResource(Host host) { + this.host = host; + } + + /** + * Return the currently bounded {@link com.yahoo.vespa.model.Host}. + * @return the {@link com.yahoo.vespa.model.Host} if bound, null if not. + */ + public Host getHost() { return host; } + + /** + * Returns the baseport of the first available port range of length numPorts, + * or 0 if there is no range of that length available. + * + * @param numPorts The length of the desired port range. + * @return The baseport of the first available range, or 0 if no range is available. + */ + public int nextAvailableBaseport(int numPorts) { + int range = 0; + int port = BASE_PORT; + for (; port < BASE_PORT + MAX_PORTS && (range < numPorts); port++) { + if (portDB.containsKey(port)) { + range = 0; + continue; + } + range++; + } + return range == numPorts ? + port - range : + 0; + } + + boolean isPortRangeAvailable(int start, int numPorts) { + int range = 0; + int port = start; + for (; port < BASE_PORT + MAX_PORTS && (range < numPorts); port++) { + if (portDB.containsKey(port)) { + return false; + } + range++; + } + return range == numPorts; + } + + /** + * Adds service and allocates resources for it. + * + * @param service The Service to allocate resources for + * @param wantedPort the wanted port for this service + * @return The allocated ports for the Service. + */ + List<Integer> allocateService(AbstractService service, int wantedPort) { + List<Integer> ports = allocatePorts(service, wantedPort); + assert (getService(service.getServiceName()) == null) : + ("There is already a service with name '" + service.getServiceName() + "' registered on " + this + + ". Most likely a programming error - all service classes must have unique names, even in different packages!"); + + services.put(service.getServiceName(), service); + return ports; + } + + // TODO: make private when LocalApplication and all model services has stopped reallocating ports _after_ allocateServices has been called + List<Integer> allocatePorts(AbstractService service, int wantedPort) { + List<Integer> ports = new ArrayList<>(); + if (service.getPortCount() < 1) + return ports; + + int serviceBasePort = BASE_PORT + allocatedPorts; + if (wantedPort > 0) { + if (service.getPortCount() < 1) { + throw new RuntimeException(service + " wants baseport " + wantedPort + + ", but it has not reserved any ports, so it cannot name a desired baseport."); + } + if (service.requiresWantedPort() || canUseWantedPort(service, wantedPort, serviceBasePort)) + serviceBasePort = wantedPort; + } + + reservePort(service, serviceBasePort); + ports.add(serviceBasePort); + + int remainingPortsStart = service.requiresConsecutivePorts() ? + serviceBasePort + 1: + BASE_PORT + allocatedPorts; + for (int i = 0; i < service.getPortCount() - 1; i++) { + int port = remainingPortsStart + i; + reservePort(service, port); + ports.add(port); + } + return ports; + } + + // TODO: this is a hack to allow calling AbstractService.setBasePort _after_ the services has been initialized, + // i.e. modifying the baseport. Done by e.g. LocalApplication. Try to remove usage of this method! + void deallocatePorts(AbstractService service) { + for (Iterator<Map.Entry<Integer,Service>> i=portDB.entrySet().iterator(); i.hasNext();) { + Map.Entry<Integer, Service> e = i.next(); + Service s = e.getValue(); + if (s.equals(service)) + i.remove(); + } + } + + private boolean canUseWantedPort(AbstractService service, int wantedPort, int serviceBasePort) { + for (int i = 0; i < service.getPortCount(); i++) { + int port = wantedPort + i; + if (portDB.containsKey(port)) { + AbstractService s = (AbstractService)portDB.get(port); + s.getRoot().getDeployState().getDeployLogger().log(Level.WARNING, service.getServiceName() +" cannot reserve port " + port + " on " + + this + ": Already reserved for " + s.getServiceName() + + ". Using default port range from " + serviceBasePort); + return false; + } + if (!service.requiresConsecutivePorts()) break; + } + return true; + } + + /** + * Reserves the desired port for the given service, or throws as exception if the port + * is not available. + * + * @param service the service that wishes to reserve the port. + * @param port the port to be reserved. + */ + void reservePort(AbstractService service, int port) { + if (portDB.containsKey(port)) { + portAlreadyReserved(service, port); + } else { + if (inVespasPortRange(port)) { + allocatedPorts++; + if (allocatedPorts > MAX_PORTS) { + noMoreAvailablePorts(); + } + } + portDB.put(port, service); + } + } + + private boolean inVespasPortRange(int port) { + return port >= BASE_PORT && + port < BASE_PORT + MAX_PORTS; + } + private void portAlreadyReserved(AbstractService service, int port) { + AbstractService otherService = (AbstractService)portDB.get(port); + int nextAvailablePort = nextAvailableBaseport(service.getPortCount()); + if (nextAvailablePort == 0) { + noMoreAvailablePorts(); + } + String msg = (service.getClass().equals(otherService.getClass()) && service.requiresWantedPort()) + ? "You must set port explicitly for all instances of this service type, except the first one. " + : ""; + throw new RuntimeException(service.getServiceName() + " cannot reserve port " + port + + " on " + this + ": Already reserved for " + otherService.getServiceName() + + ". " + msg + "Next available port is: " + nextAvailablePort); + } + + + private void noMoreAvailablePorts() { + throw new RuntimeException + ("Too many ports are reserved in Vespa's port range (" + + BASE_PORT + ".." + (BASE_PORT+MAX_PORTS) + ") on " + this + + ". Move one or more services to another host, or outside this port range."); + } + + + /** + * Returns the service with the given "sentinel name" on this Host, + * or null if the name does not match any service. + * + * @param sentinelName the sentinel name of the service we want to return + * @return The service with the given sentinel name + */ + public Service getService(String sentinelName) { + return services.get(sentinelName); + } + + /** + * Returns a List of all services running on this Host. + * @return a List of all services running on this Host. + */ + public List<Service> getServices() { + return new ArrayList<>(services.values()); + } + + public HostInfo getHostInfo() { + return new HostInfo(getHostName(), services.values().stream() + .map(service -> service.getServiceInfo()) + .collect(Collectors.toSet())); + } + + @Override + public String toString() { + return "host '" + host.getHostName() + "'"; + } + + public String getHostName() { + return host.getHostName(); + } + + @Override + public int hashCode() { return host.hashCode(); } + + @Override + public boolean equals(Object other) { + if (other == this) return true; + if ( ! (other instanceof HostResource)) return false; + return ((HostResource)other).getHost().equals(this.getHost()); + } + + @Override + public int compareTo(HostResource other) { + return this.host.compareTo(other.host); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java new file mode 100644 index 00000000000..42feee08f43 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -0,0 +1,203 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.config.provision.*; +import com.yahoo.net.HostName; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.*; +import java.util.logging.Level; +import java.util.stream.Collectors; + +/** + * The parent node for all Host instances, and thus accessible + * to enable services to get their Host. + * + * @author gjoranv + */ +public class HostSystem extends AbstractConfigProducer<Host> { + + private Map<String,String> ipAddresses = new LinkedHashMap<>(); + private Map<String,String> hostnames = new LinkedHashMap<>(); + + private final Map<String, HostResource> hostname2host = new LinkedHashMap<>(); + private final HostProvisioner provisioner; + private final Map<HostResource, Set<ClusterMembership>> mapping = new LinkedHashMap<>(); + + public HostSystem(AbstractConfigProducer parent, String name, HostProvisioner provisioner) { + super(parent, name); + this.provisioner = provisioner; + } + + /** + * Returns the host with the given hostname. + * + * @param name the hostname of the host. + * @return the host with the given hostname. + */ + public HostResource getHostByHostname(String name) { + // TODO: please eliminate the following ugly hack + if ("localhost.fortestingpurposesonly".equals(name)) { + String localhost = "localhost"; + if ( ! getChildren().containsKey(localhost)) { + new Host(this, localhost); + } + return new HostResource(getChildren().get(localhost)); + } + return hostname2host.get(name); + } + + /** + * Returns the canonical name of a given host. This will cache names for faster lookup. + * + * @param hostname the hostname to retrieve the canonical hostname for. + * @return The canonical hostname, or null if unable to resolve. + * @throws UnknownHostException if the hostname cannot be resolved + */ + public String getCanonicalHostname(String hostname) throws UnknownHostException { + if ( ! hostnames.containsKey(hostname)) { + hostnames.put(hostname, lookupCanonicalHostname(hostname)); + } + return hostnames.get(hostname); + } + + /** + * Static helper method that looks up the canonical name of a given host. + * + * @param hostname the hostname to retrieve the canonical hostname for. + * @return The canonical hostname, or null if unable to resolve. + * @throws UnknownHostException if the hostname cannot be resolved + */ + public static String lookupCanonicalHostname(String hostname) throws UnknownHostException { + return java.net.InetAddress.getByName(hostname).getCanonicalHostName(); + } + + /** + * Returns the if address of a host. + * + * @param hostname the hostname to retrieve the ip address for. + * @return The string representation of the ip-address. + */ + public String getIp(String hostname) { + if (ipAddresses.containsKey(hostname)) return ipAddresses.get(hostname); + + String ipAddress; + if (hostname.startsWith(MockRoot.MOCKHOST)) { + ipAddress = "0.0.0.0"; + } else { + try { + InetAddress address = InetAddress.getByName(hostname); + ipAddress = address.getHostAddress(); + } catch (java.net.UnknownHostException e) { + log.warning("Unable to find valid IP address of host: " + hostname); + ipAddress = "0.0.0.0"; + } + } + ipAddresses.put(hostname, ipAddress); + return ipAddress; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (HostResource host : mapping.keySet()) { + sb.append(host).append(","); + } + if (sb.length() > 0) sb.deleteCharAt(sb.length() - 1); + return sb.toString(); + } + + public HostResource getHost(String hostAlias) { + HostSpec hostSpec = provisioner.allocateHost(hostAlias); + for (Map.Entry<HostResource, Set<ClusterMembership>> entrySet : mapping.entrySet()) { + HostResource resource = entrySet.getKey(); + if (resource.getHostName().equals(hostSpec.hostname())) { + entrySet.getValue().add(hostSpec.membership().orElse(null)); + return resource; + } + } + return addNewHost(hostSpec); + } + + private HostResource addNewHost(HostSpec hostSpec) { + Host host = new Host(this, hostSpec.hostname()); + HostResource hostResource = new HostResource(host); + hostname2host.put(host.getHostName(), hostResource); + Set<ClusterMembership> hostMemberships = new LinkedHashSet<>(); + if (hostSpec.membership().isPresent()) + hostMemberships.add(hostSpec.membership().get()); + mapping.put(hostResource, hostMemberships); + return hostResource; + } + + public List<HostResource> getHosts() { + return mapping.keySet().stream() + .filter(host -> !host.getHost().isMultitenant()) + .collect(Collectors.toList()); + } + + public Map<HostResource, ClusterMembership> allocateHosts(ClusterSpec cluster, Capacity capacity, int groups, DeployLogger logger) { + List<HostSpec> allocatedHosts = provisioner.prepare(cluster, capacity, groups, new ProvisionDeployLogger(logger)); + // TODO: Let hostresource own a membership rather than using a map? + Map<HostResource, ClusterMembership> retAllocatedHosts = new LinkedHashMap<>(); + for (HostSpec host : allocatedHosts) { + // This is needed for single node host provisioner to work in unit tests for hosted vespa applications. + Optional<HostResource> existingHost = getExistingHost(host); + if (existingHost.isPresent()) { + retAllocatedHosts.put(existingHost.get(), host.membership().orElse(null)); + } else { + retAllocatedHosts.put(addNewHost(host), host.membership().orElse(null)); + } + } + return retAllocatedHosts; + } + + private Optional<HostResource> getExistingHost(HostSpec key) { + List<HostResource> hosts = mapping.keySet().stream() + .filter(resource -> resource.getHostName().equals(key.hostname())) + .collect(Collectors.toList()); + if (hosts.isEmpty()) { + return Optional.empty(); + } else { + return Optional.of(hosts.get(0)); + } + } + + public void addBoundHost(HostResource host) { + mapping.put(host, new LinkedHashSet<>()); + hostname2host.put(host.getHostName(), host); + } + + Map<HostSpec, Set<ClusterMembership>> getHostToServiceSpecMapping() { + Map<HostSpec, Set<ClusterMembership>> specMapping = new LinkedHashMap<>(); + for (Map.Entry<HostResource, Set<ClusterMembership>> entrySet : mapping.entrySet()) { + if (!entrySet.getKey().getHost().isMultitenant()) { + Optional<ClusterMembership> membership = entrySet.getValue().stream().filter(m -> m != null).findFirst(); + specMapping.put(new HostSpec(entrySet.getKey().getHostName(), membership), new LinkedHashSet<>(entrySet.getValue())); + } + } + return specMapping; + } + + /** A provision logger which forwards to a deploy logger */ + private static class ProvisionDeployLogger implements ProvisionLogger { + + private final DeployLogger deployLogger; + + public ProvisionDeployLogger(DeployLogger deployLogger) { + this.deployLogger = deployLogger; + } + + @Override + public void log(Level level, String message) { + deployLogger.log(level, message); + } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/InstanceResolver.java b/config-model/src/main/java/com/yahoo/vespa/model/InstanceResolver.java new file mode 100644 index 00000000000..9c761e425a7 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/InstanceResolver.java @@ -0,0 +1,196 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +import com.yahoo.config.*; +import com.yahoo.config.codegen.CNode; +import com.yahoo.config.codegen.ConfigGenerator; +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.config.codegen.LeafCNode; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.buildergen.ConfigDefinition; +import com.yahoo.yolean.Exceptions; +import com.yahoo.vespa.config.*; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.List; +import java.util.Map; + +/** + * <p> + * This class is capable of resolving config from a config model for a given request. It will handle + * incompatibilities of the def version in the request and the version of the config classes the model + * is using. + * </p> + * <p> + * This class is agnostic of transport protocol and server implementation. + * </p> + * <p> + * Thread safe. + * </p> + * + * @author vegardh + * @since 5.1.5 + */ +// TODO This functionality should be on VespaModel itself, but we don't have a way right now to apply a config override to a ConfigInstance.Builder +class InstanceResolver { + + private static final java.util.logging.Logger log = java.util.logging.Logger.getLogger(InstanceResolver.class.getName()); + + /** + * Resolves this config key into a correctly typed ConfigInstance using the given config builder. + * FIXME: Make private once config overrides are deprecated.? + * + * @param key a ConfigKey + * @param builder a ConfigBuilder to create the instance from. + * @param targetDef the def to use + * @return the config instance or null of no producer for this found in model + */ + static ConfigInstance resolveToInstance(ConfigKey<?> key, ConfigBuilder builder, InnerCNode targetDef) { + ConfigDefinitionKey defKey = new ConfigDefinitionKey(key); + try { + if (targetDef != null) applyDef(builder, targetDef); + ConfigInstance instance = getInstance(defKey, builder.getClass().getClassLoader()); + Class<? extends ConfigInstance> clazz = instance.getClass(); + return clazz.getConstructor(new Class<?>[]{builder.getClass()}).newInstance(builder); + } catch (Exception e) { + throw new ConfigurationRuntimeException(e); + } + } + + /** + * Resolves this config key into a correctly typed ConfigBuilder using the given config model. + * FIXME: Make private once config overrides are deprecated.? + * + * @return the config builder or null if no producer for this found in model + */ + static ConfigBuilder resolveToBuilder(ConfigKey<?> key, VespaModel model, ConfigDefinition targetDef) { + if (model == null) return null; + ConfigDefinitionKey defKey = new ConfigDefinitionKey(key); + ConfigInstance.Builder builder = model.createBuilder(defKey, targetDef); + model.getConfig(builder, key.getConfigId()); + return builder; + } + + /** + * If some fields on the builder are null now, set them from the def. Do recursively. + * <p> + * If the targetDef has some schema incompatibilities, they are not handled here + * (except logging in some cases), but in ConfigInstance.serialize(). + * + * @param builder a {@link com.yahoo.config.ConfigBuilder} + * @param targetDef a config definition + * @throws Exception if applying values form config definitions fails + */ + static void applyDef(ConfigBuilder builder, InnerCNode targetDef) throws Exception { + for (Map.Entry<String, CNode> e: targetDef.children().entrySet()) { + CNode node = e.getValue(); + if (node instanceof LeafCNode) { + setLeafValueIfUnset(targetDef, builder, (LeafCNode)node); + } else if (node instanceof InnerCNode) { + // Is there a private field on the builder that matches this inner node in the def? + if (hasField(builder.getClass(), node.getName())) { + Field innerField = builder.getClass().getDeclaredField(node.getName()); + innerField.setAccessible(true); + Object innerFieldVal = innerField.get(builder); + if (innerFieldVal instanceof List) { + // inner array? Check that list elems are ConfigBuilder + List<?> innerList = (List<?>) innerFieldVal; + for (Object b : innerList) { + if (b instanceof ConfigBuilder) { + applyDef((ConfigBuilder) b, (InnerCNode) node); + } + } + } else if (innerFieldVal instanceof ConfigBuilder) { + // Struct perhaps + applyDef((ConfigBuilder) innerFieldVal, (InnerCNode) node); + } else { + // Likely a config value mismatch. That is handled in ConfigInstance.serialize() (error message, omit from response.) + } + } + } + } + } + + private static boolean hasField(Class<?> aClass, String name) { + for (Field field : aClass.getDeclaredFields()) { + if (name.equals(field.getName())) { + return true; + } + } + return false; + } + + private static void setLeafValueIfUnset(InnerCNode targetDef, Object builder, LeafCNode node) throws Exception { + if (hasField(builder.getClass(), node.getName())) { + Field field = builder.getClass().getDeclaredField(node.getName()); + field.setAccessible(true); + Object val = field.get(builder); + if (val==null) { + // Not set on builder, if the leaf node has a default value, try the private setter that takes String + try { + if (node.getDefaultValue()!=null) { + Method setter = builder.getClass().getDeclaredMethod(node.getName(), String.class); + setter.setAccessible(true); + setter.invoke(builder, node.getDefaultValue().getValue()); + } + } catch (Exception e) { + log.severe("For config '"+targetDef.getFullName()+"': Unable to apply the default value for field '"+node.getName()+ + "' to config Builder (where it wasn't set): "+ + Exceptions.toMessageString(e)); + } + } + } + } + + /** + * Create a ConfigBuilder given a definition key and a payload + * @param key The key to use to create the correct builder. + * @param payload The payload to populate the builder with. + * @return A ConfigBuilder initialized with payload. + */ + static ConfigBuilder createBuilderFromPayload(ConfigDefinitionKey key, VespaModel model, ConfigPayload payload, ConfigDefinition targetDef) { + ConfigBuilder builderInstance = model.createBuilder(key, targetDef); + if (builderInstance == null || builderInstance instanceof GenericConfig.GenericConfigBuilder) { + if (log.isLoggable(LogLevel.SPAM)) { + log.log(LogLevel.SPAM, "Creating generic builder for key=" + key); + } + return new GenericConfig.GenericConfigBuilder(key, new ConfigPayloadBuilder(payload)); + } + ConfigTransformer transformer = new ConfigTransformer(builderInstance.getClass().getDeclaringClass()); + return transformer.toConfigBuilder(payload); + } + + /** + * Returns a {@link ConfigInstance} of right type for given key using reflection + * + * @param cKey a ConfigKey + * @return a {@link ConfigInstance} or null if not available in classpath + */ + static ConfigInstance getInstance(ConfigDefinitionKey cKey, ClassLoader instanceLoader) { + String className = ConfigGenerator.createClassName(cKey.getName()); + Class<?> clazz; + String fullClassName = packageName(cKey) + "." + className; + try { + clazz = instanceLoader != null ? instanceLoader.loadClass(fullClassName) : Class.forName(fullClassName); + } catch (ClassNotFoundException e) { + return null; + } + Object i; + try { + i = clazz.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new ConfigurationRuntimeException(e); + } + if (!(i instanceof ConfigInstance)) { + throw new ConfigurationRuntimeException(fullClassName + " is not a ConfigInstance, can not produce config for the name '" + cKey.getName() + "'."); + } + return (ConfigInstance) i; + } + + static String packageName(ConfigDefinitionKey cKey) { + String prefix = "com.yahoo."; + return prefix + (cKey.getNamespace().isEmpty() ? CNode.DEFAULT_NAMESPACE : cKey.getNamespace()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Logd.java b/config-model/src/main/java/com/yahoo/vespa/model/Logd.java new file mode 100644 index 00000000000..41127f6423b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/Logd.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +/** + * There is one logd running on each Vespa host, and one instance of + * this class is therefore created by each instance of class {@link + * Host}. + * + * @author gjoranv + */ +public class Logd extends AbstractService { + + /** + * Creates a new Logd instance. + */ + public Logd(Host host) { + super(host, "logd"); + setProp("clustertype", "hosts"); + setProp("clustername", "admin"); + } + + /** + * Logd does not need any ports. + * + * @return The number of ports reserved by the logd + */ + public int getPortCount() { return 0; } + + /** + * @return The command used to start logd + */ + public String getStartupCommand() { return "exec sbin/logd"; } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/PlainFormatter.java b/config-model/src/main/java/com/yahoo/vespa/model/PlainFormatter.java new file mode 100644 index 00000000000..d424f4fa31b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/PlainFormatter.java @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +import java.util.logging.Formatter; +import java.util.logging.LogRecord; + +/** + * A log formatter that returns a plain log message only with level, not + * including timestamp and method (as java.util.logging.SimpleFormatter). + * See bug #1789867. + * + * @author gjoranv + */ +public class PlainFormatter extends Formatter { + + public PlainFormatter() { + super(); + } + + public String format(LogRecord record) { + StringBuffer sb = new StringBuffer(); + + sb.append(record.getLevel().getName()).append(": "); + sb.append(formatMessage(record)).append("\n"); + + return sb.toString(); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/PortsMeta.java b/config-model/src/main/java/com/yahoo/vespa/model/PortsMeta.java new file mode 100644 index 00000000000..ea2151f9976 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/PortsMeta.java @@ -0,0 +1,168 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; + +/** + * Track metainformation about the ports of a service. + * + * @author Vidar Larsen + */ +public class PortsMeta implements Serializable { + /** A list of all ports. The list elements are lists of strings. */ + private List<LinkedList<String>> ports; + + /** Remember the rpc admin port offset. */ + private Integer rpcAdminOffset = null; + /** Remember the rpc status port offset. */ + private Integer rpcStatusOffset = null; + /** Remember the https admin port offset. */ + private Integer httpAdminOffset = null; + /** Remember the http status port offset. */ + private Integer httpStatusOffset = null; + + /** Keep track of the offset to register for, when chaining */ + private Integer currentOffset; + + /** Create a new PortsMeta object. */ + public PortsMeta() { + ports = new ArrayList<>(); + } + + /** + * Set up the port to tag, for chained usage. + * @param offset The relative port to tag. + * @return this portsmeta, to allow .tag calls. + */ + public PortsMeta on(int offset) { + this.currentOffset = offset; + return this; + } + + /** + * Tag a previously setup port (using 'on') with the specified tag. + * @param meta The tag to apply to the current port. + * @return this portsmeta, to allow further .tag calls. + */ + public PortsMeta tag(String meta) { + if (currentOffset == null) { + throw new NullPointerException("No current port offset to tag, use 'on#1'"); + } + return register(currentOffset, meta); + } + + /** + * Register a given metainfo string to the port at offset. + * @param offset 0-based index to identify the port + * @param meta a String to be added to the given ports meta set. + * @return this for convenient chaining. + */ + private PortsMeta register(int offset, String meta) { + // Allocate new LinkedLists on each element up-to-and-including offset + for (int i = ports.size(); i <= offset; i++) { + ports.add(i, new LinkedList<String>()); + } + ports.get(offset).addFirst(meta); + + return this; + } + + /** + * Check if the port at a specific offset contains a particular meta attribute. + * @param offset The relative port offset + * @param meta The meta info we want to check for + * @return boolean true if the specific port has registered the meta + */ + public boolean contains(int offset, String meta) { + return offset < ports.size() && ports.get(offset).contains(meta); + } + + /** + * Get the number of ports with registered meta. + * @return the number of ports that have been registered. + */ + public int getNumPorts() { + return ports.size(); + } + /** + * Get an iterator of the Strings registered at the specific point. + * @param offset The relative offset to inquire about tags. + * @return List of tags. + */ + public List<String> getTagsAt(int offset) { + try { + return ports.get(offset); + } catch (IndexOutOfBoundsException e) { + throw new RuntimeException("Trying to get ports meta with offset " + offset + + ", which is outside the range 0 to " + ports.size(), e); + } + } + + /** + * Get the offset to the rpc port used for admin. + * @return Integer the offset, or null if none set. + */ + public Integer getRpcAdminOffset() { + if (rpcAdminOffset == null) { + for (int p = 0; p < getNumPorts(); p++) { + if (contains(p, "rpc") && contains(p, "admin") && !contains(p, "nc")) { + rpcAdminOffset = p; + break; + } + } + } + return rpcAdminOffset; + } + + /** + * Get the offset to the rpc port used for status. + * @return Integer the offset, or null if none set. + */ + public Integer getRpcStatusOffset() { + if (rpcStatusOffset == null) { + for (int p = 0; p < getNumPorts(); p++) { + if (contains(p, "rpc") && contains(p, "status") && !contains(p, "nc")) { + rpcStatusOffset = p; + break; + } + } + } + return rpcStatusOffset; + } + + /** + * Get the offset to the http port used for admin. + * @return Integer the offset, or null if none set. + */ + public Integer getHttpAdminOffset() { + if (httpAdminOffset == null) { + for (int p = 0; p < getNumPorts(); p++) { + if (contains(p, "http") && contains(p, "admin")) { + httpAdminOffset = p; + break; + } + } + } + return httpAdminOffset; + } + + /** + * Get the offset to the http port used for status. + * @return Integer the offset, or null if none set. + */ + public Integer getHttpStatusOffset() { + if (httpStatusOffset == null) { + for (int p = 0; p < getNumPorts(); p++) { + if (contains(p, "http") && contains(p, "status")) { + httpStatusOffset = p; + break; + } + } + } + return httpStatusOffset; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/RecentLogFilter.java b/config-model/src/main/java/com/yahoo/vespa/model/RecentLogFilter.java new file mode 100644 index 00000000000..84f33b637fb --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/RecentLogFilter.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +import java.util.LinkedList; +import java.util.logging.Filter; +import java.util.logging.LogRecord; + +/** + * A filter for log messages that prevents the most recently published messages + * to be published again. See bug #461290. + * + * @author gjoranv + */ +public class RecentLogFilter implements Filter { + + LinkedList<String> recent = new LinkedList<>(); + static final int maxMessages = 6; + + public boolean isLoggable(LogRecord record) { + String msg = record.getMessage(); + if (msg != null && recent.contains(msg)) { + return false; // duplicate + } else { + recent.addLast(msg); + if (recent.size() > maxMessages) { + recent.removeFirst(); + } + return true; // new message + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Service.java b/config-model/src/main/java/com/yahoo/vespa/model/Service.java new file mode 100644 index 00000000000..ccb6397dd46 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/Service.java @@ -0,0 +1,142 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +import com.yahoo.config.model.api.ServiceInfo; + +import java.util.HashMap; +import java.util.Optional; + + +/** + * Representation of a process which runs a service + * + * @author gjoranv + */ +public interface Service extends ConfigProducer { + + /** + * Services that should be started by config-sentinel must return + * non-null. The returned value will be used in config-sentinel + * configuration. + * TODO: Should change this to Optional of String + */ + public String getStartupCommand(); + + /** + * Services that wish that a command should be run before shutdown + * should return the command here. The command will be executed + * by the config sentinel before sending SIGTERM to the service. + * The command is executed without a timeout. + */ + public Optional<String> getPreShutdownCommand(); + + /** + * Tells if this service should be autostarted by + * config-sentinel. Returned value will be used to configure the + * config-sentinel. + */ + public boolean getAutostartFlag(); + + /** + * Tells if this service should be autorestarted by + * config-sentinel. Returned value will be used to configure the + * config-sentinel. + */ + public boolean getAutorestartFlag(); + + /** + * Returns the type of service. E.g. the class-name without the + * package prefix. + */ + public String getServiceType(); + + /** + * Returns the name that identifies this service for the config-sentinel. + */ + public String getServiceName(); + + /** + * Returns the desired base port for this service, or '0' if this + * service should use the default port allocation mechanism. + * + * @return The desired base port for this service. + */ + public int getWantedPort(); + + /** + * Returns true if the desired base port (returned by + * getWantedPort()) for this service is the only allowed base + * port. + * + * @return true if this Service requires the wanted base port. + */ + public boolean requiresWantedPort(); + + /** + * Returns the number of ports needed by this service. + */ + public int getPortCount(); + + /** + * Returns a PortsMeta object, giving access to more information + * about the different ports of this service. + */ + public PortsMeta getPortsMeta(); + + /** + * @return the physical host on which this service runs. + */ + public Host getHost(); + + /** + * Get meta information about service. + * @return an instance of {@link com.yahoo.config.model.api.ServiceInfo} + */ + public ServiceInfo getServiceInfo(); + + /** + * @return The hostname on which this service runs. + */ + public String getHostName(); + + /** Optional JVM execution args for this service */ + public String getJvmArgs(); + + /** + * Computes and returns the i'th port for this service, based on + * this Service's baseport. + * + * @param i The offset from 'basePort' of the port to return + * @return the i'th port relative to the base port. + * @throws IllegalStateException if i is out of range. + */ + public int getRelativePort(int i); + + /** + * Gets a service property value mapped to the given key + * as a String, or the value in <code>defStr</code> if no such key exists. + * + * @param key a key used for lookup in the service properties + * @param defStr default String value returned if no value for key found + * @return the associated String value for the given key, or + */ + public String getServicePropertyString(String key, String defStr); + + /** + * @return the service health port, to report status to yamas + */ + public int getHealthPort(); + + /** + * + * @return HashMap of default dimensions for metrics. + */ + public HashMap<String,String> getDefaultMetricDimensions(); + + /** + * Return the Affinity of this service if it has. + * + * @return The {@link com.yahoo.vespa.model.Affinity} for this service. + */ + public Optional<Affinity> getAffinity(); +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ServiceProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/ServiceProvider.java new file mode 100644 index 00000000000..29508eda1eb --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/ServiceProvider.java @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +/** + * A provider of services (an entity which has its own top-level tag in the services.xml file). + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public interface ServiceProvider { + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/SimpleConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/SimpleConfigProducer.java new file mode 100644 index 00000000000..3e5f816f583 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/SimpleConfigProducer.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +import com.yahoo.config.model.producer.AbstractConfigProducer; + +/** + * Some configuration level with no special handling of its own. + * + * @author arnej27959 + */ +public final class SimpleConfigProducer<T extends AbstractConfigProducer<?>> extends AbstractConfigProducer<T> { + + private static final long serialVersionUID = 1L; + + /** + * Creates a new instance + * + * @param parent parent ConfigProducer. + * @param configId name of this instance + */ + public SimpleConfigProducer(AbstractConfigProducer parent, String configId) { + super(parent, configId); + } + + //Ease access restriction + @Override + public void addChild(T child) { + super.addChild(child); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaConfigModelRegistry.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaConfigModelRegistry.java new file mode 100644 index 00000000000..6b9aa3fe8c5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaConfigModelRegistry.java @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +import com.yahoo.config.model.ConfigModelRegistry; +import com.yahoo.config.model.admin.AdminModel; +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.config.model.builder.xml.ConfigModelId; +import com.yahoo.vespa.model.builder.xml.dom.*; +import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; +import com.yahoo.vespa.model.container.xml.ContainerModelBuilder.Networking; +import com.yahoo.vespa.model.generic.GenericServicesBuilder; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Resolves the Vespa config model types + * + * @author bratseth + */ +public class VespaConfigModelRegistry extends ConfigModelRegistry { + + private final List<ConfigModelBuilder> builderList = new ArrayList<>(); + + /** Creates a bundled model class registry which forwards unresolved requests to the argument instance */ + public VespaConfigModelRegistry(ConfigModelRegistry chained) { + super(chained); + builderList.add(new AdminModel.BuilderV2()); + builderList.add(new AdminModel.BuilderV4()); + builderList.add(new DomRoutingBuilder()); + builderList.add(new DomClientsBuilder()); + builderList.add(new DomContentBuilder()); + builderList.add(new ContainerModelBuilder(false, Networking.enable)); + builderList.add(new GenericServicesBuilder()); + } + + @Override + public Collection<ConfigModelBuilder> resolve(ConfigModelId id) { + Set<ConfigModelBuilder> builders = new HashSet<>(chained().resolve(id)); + for (ConfigModelBuilder builder : builderList) { + if (builder.handlesElements().contains(id)) + builders.add(builder); + } + return builders; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java new file mode 100644 index 00000000000..9a23be1f5c5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -0,0 +1,557 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +import com.yahoo.config.ConfigBuilder; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.ConfigInstance.Builder; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.codegen.ConfigGenerator; +import com.yahoo.config.codegen.InnerCNode; +import com.yahoo.config.model.ApplicationConfigProducerRoot; +import com.yahoo.config.model.ConfigModelRegistry; +import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.FileDistribution; +import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.model.producer.AbstractConfigProducerRoot; +import com.yahoo.config.model.producer.UserConfigRepo; +import com.yahoo.config.provision.ProvisionInfo; +import com.yahoo.config.subscription.ConfigInstanceUtil; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigPayload; +import com.yahoo.vespa.config.ConfigPayloadBuilder; +import com.yahoo.vespa.config.GenericConfig; +import com.yahoo.vespa.config.buildergen.ConfigDefinition; +import com.yahoo.vespa.model.admin.Admin; +import com.yahoo.vespa.model.application.validation.ValidationId; +import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.vespa.model.builder.VespaModelBuilder; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.clients.Clients; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.ContainerModel; +import com.yahoo.vespa.model.content.Content; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; +import com.yahoo.vespa.model.filedistribution.FileDistributor; +import com.yahoo.vespa.model.generic.service.ServiceCluster; +import com.yahoo.vespa.model.routing.Routing; +import com.yahoo.vespa.model.search.AbstractSearchCluster; +import com.yahoo.vespa.model.utils.internal.ReflectionUtil; +import com.yahoo.yolean.Exceptions; +import org.xml.sax.SAXException; + +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.lang.reflect.Constructor; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.logging.Logger; + +import static com.yahoo.text.StringUtilities.quote; + +/** + * <p> + * The root ConfigProducer node for all Vespa systems (there is currently only one). + * The main class for building the Vespa model. + * </p> + * The vespa model starts in an unfrozen state, where children can be added freely, + * but no structure dependent information can be used. + * When frozen, structure dependent information(such as config id and controller) are + * made available, but no additional config producers can be added. + * + * @author gjoranv + */ +public final class VespaModel extends AbstractConfigProducerRoot implements Serializable, Model { + + private static final long serialVersionUID = 1L; + + public static final Logger log = Logger.getLogger(VespaModel.class.getPackage().toString()); + private ConfigModelRepo configModelRepo = new ConfigModelRepo(); + private final Optional<ProvisionInfo> info; + + /** + * The config id for the root config producer + */ + public static final String ROOT_CONFIGID = ""; + + private ApplicationConfigProducerRoot root = null; + + /** + * Generic service instances - service clusters which have no specific model + */ + private List<ServiceCluster> serviceClusters = new ArrayList<>(); + + private DeployState deployState; + + /** The validation overrides of this. This is never null. */ + private final ValidationOverrides validationOverrides; + + /** Creates a Vespa Model from internal model types only */ + public VespaModel(ApplicationPackage app) throws IOException, SAXException { + this(app, new NullConfigModelRegistry()); + } + + /** Creates a Vespa Model from internal model types only */ + public VespaModel(DeployState deployState) throws IOException, SAXException { + this(new NullConfigModelRegistry(), deployState); + } + + /** + * Constructs vespa model using config given in app + * + * @param app the application to create a model from + * @param configModelRegistry a registry of config model "main" classes which may be used + * to instantiate config models + */ + public VespaModel(ApplicationPackage app, ConfigModelRegistry configModelRegistry) throws IOException, SAXException { + this(configModelRegistry, new DeployState.Builder().applicationPackage(app).build()); + } + + /** + * Constructs vespa model using config given in app + * + * @param configModelRegistry a registry of config model "main" classes which may be used + * to instantiate config models + * @param deployState the global deploy state to use for this model. + */ + public VespaModel(ConfigModelRegistry configModelRegistry, DeployState deployState) throws IOException, SAXException { + super("vespamodel"); + this.deployState = deployState; + this.validationOverrides = deployState.validationOverrides(); + configModelRegistry = new VespaConfigModelRegistry(configModelRegistry); + VespaModelBuilder builder = new VespaDomBuilder(); + root = builder.getRoot(VespaModel.ROOT_CONFIGID, deployState, this); + configModelRepo.readConfigModels(deployState, builder, root, configModelRegistry); + addServiceClusters(deployState.getApplicationPackage(), builder); + setupRouting(); + log.log(LogLevel.DEBUG, "hostsystem=" + getHostSystem()); + this.info = Optional.of(createProvisionInfo()); + getAdmin().addPerHostServices(getHostSystem().getHosts(), deployState.getProperties()); + freezeModelTopology(); + root.prepare(configModelRepo); + configModelRepo.prepareConfigModels(); + validateWrapExceptions(); + this.deployState = null; + } + + private ProvisionInfo createProvisionInfo() { + HostSystem hostSystem = root.getHostSystem(); + return ProvisionInfo.withHosts(hostSystem.getHostToServiceSpecMapping().keySet()); + } + + private void validateWrapExceptions() { + try { + validate(); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException("Error while validating model:", e); + } + } + + /** Adds generic application specific clusters of services */ + private void addServiceClusters(ApplicationPackage app, VespaModelBuilder builder) { + for (ServiceCluster sc : builder.getClusters(app, this)) + serviceClusters.add(sc); + } + + private void setupRouting() { + root.setupRouting(configModelRepo); + } + + /** Returns the one and only HostSystem of this VespaModel */ + public HostSystem getHostSystem() { + return root.getHostSystem(); + } + + /** Return a collection of all hostnames used in this application */ + @Override + public Set<HostInfo> getHosts() { + Set<HostInfo> hosts = new LinkedHashSet<>(); + for (HostResource host : root.getHostSystem().getHosts()) { + if (!host.getHost().isMultitenant()) { + hosts.add(host.getHostInfo()); + } + } + return hosts; + } + + public FileDistributor getFileDistributor() { + return root.getFileDistributionConfigProducer().getFileDistributor(); + } + + /** Returns this models Vespa instance */ + public ApplicationConfigProducerRoot getVespa() { return root; } + + @Override + public boolean allowModelVersionMismatch() { + return validationOverrides.allows(ValidationId.configModelVersionMismatch) || + validationOverrides.allows(ValidationId.skipOldConfigModels); // implies this + } + + @Override + public boolean skipOldConfigModels() { + return validationOverrides.allows(ValidationId.skipOldConfigModels); + } + + /** + * Resolves config of the given type and config id, by first instantiating the correct {@link com.yahoo.config.ConfigInstance.Builder}, + * calling {@link #getConfig(com.yahoo.config.ConfigInstance.Builder, String)}. The default values used will be those of the config + * types in the model. + * + * @param clazz The type of config + * @param configId The config id + * @return A config instance of the given type + */ + public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> clazz, String configId) { + try { + ConfigInstance.Builder builder = newBuilder(clazz); + getConfig(builder, configId); + return newConfigInstance(clazz, builder); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + + /** + * Populates an instance of configClass with config produced by configProducer. + */ + public static <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> configClass, ConfigProducer configProducer) { + try { + Builder builder = newBuilder(configClass); + populateConfigBuilder(builder, configProducer); + return newConfigInstance(configClass, builder); + } catch (Exception e) { + throw new RuntimeException("Failed getting config for class " + configClass.getName(), e); + } + } + + private static <CONFIGTYPE extends ConfigInstance> CONFIGTYPE newConfigInstance(Class<CONFIGTYPE> configClass, Builder builder) + throws NoSuchMethodException, InstantiationException, IllegalAccessException, java.lang.reflect.InvocationTargetException { + + Constructor<CONFIGTYPE> constructor = configClass.getConstructor(builder.getClass()); + return constructor.newInstance(builder); + } + + private static Builder newBuilder(Class<? extends ConfigInstance> configClass) + throws ClassNotFoundException, InstantiationException, IllegalAccessException { + + Class builderClazz = configClass.getClassLoader().loadClass(configClass.getName() + "$Builder"); + return (Builder)builderClazz.newInstance(); + } + + /** + * Throw if the config id does not exist in the model. + * + * @param configId a config id + */ + protected void checkId(String configId) { + if ( ! id2producer.containsKey(configId)) { + log.log(LogLevel.DEBUG, "Invalid config id: " + configId); + } + } + + /** + * Resolves config for a given config id and populates the given builder with the config. + * + * @param builder a configinstance builder + * @param configId the config id for the config client + * @return the builder if a producer was found, and it did apply config, null otherwise + */ + @SuppressWarnings("unchecked") + @Override + public ConfigInstance.Builder getConfig(ConfigInstance.Builder builder, String configId) { + checkId(configId); + Optional<ConfigProducer> configProducer = getConfigProducer(configId); + if ( ! configProducer.isPresent()) return null; + + populateConfigBuilder(builder, configProducer.get()); + return builder; + } + + private static void populateConfigBuilder(Builder builder, ConfigProducer configProducer) { + boolean found = configProducer.cascadeConfig(builder); + boolean foundOverride = configProducer.addUserConfig(builder); + if (logDebug()) { + log.log(LogLevel.DEBUG, "Trying to get config for " + builder.getClass().getDeclaringClass().getName() + + " for config id " + quote(configProducer.getConfigId()) + + ", found=" + found + ", foundOverride=" + foundOverride); + } + } + + /** TODO: Remove once configserver has switched to using {@link VespaModel#getConfig(ConfigKey, ConfigDefinition, ConfigPayload)} instead. **/ + public ConfigPayload getConfig(ConfigKey configKey, InnerCNode targetDef, ConfigPayload userOverride) throws IOException { + return getConfig(configKey, targetDef == null ? null : new ConfigDefinition(targetDef), userOverride); + } + + /** + * Resolve config for a given key and a def. Apply an override if given. + * + * @param configKey The key to resolve. + * @param targetDef The def file to use for the schema. + * @param userOverride A user override that should be applied to the config + * @return The payload as a list of strings + */ + @Override + public ConfigPayload getConfig(ConfigKey configKey, com.yahoo.vespa.config.buildergen.ConfigDefinition targetDef, ConfigPayload userOverride) throws IOException { + ConfigBuilder builder = InstanceResolver.resolveToBuilder(configKey, this, targetDef); + if (builder != null) { + log.log(LogLevel.DEBUG, () -> "Found builder for " + configKey); + // Support deprecated configs/ user override + applyConfigsOverride(configKey, builder, userOverride, targetDef); + ConfigPayload payload; + InnerCNode innerCNode = targetDef != null ? targetDef.getCNode() : null; + if (builder instanceof GenericConfig.GenericConfigBuilder) { + payload = getConfigFromGenericBuilder(builder); + } else { + payload = getConfigFromBuilder(configKey, builder, innerCNode); + } + return (innerCNode != null) ? payload.applyDefaultsFromDef(innerCNode) : payload; + } + return null; + } + + private ConfigPayload getConfigFromBuilder(ConfigKey configKey, ConfigBuilder builder, InnerCNode targetDef) { + try { + ConfigInstance instance = InstanceResolver.resolveToInstance(configKey, builder, targetDef); + log.log(LogLevel.DEBUG, () -> "getConfigFromBuilder for " + configKey + ",instance=" + instance); + return ConfigPayload.fromInstance(instance); + } catch (ConfigurationRuntimeException e) { + // This can happen in cases where services ask for config that no longer exist before they have been able + // to reconfigure themselves. This happens for instance whenever jdisc reconfigures itself until + // ticket 6599572 is fixed. When that happens, consider propagating a full error rather than empty payload + // back to the client. + log.log(LogLevel.INFO, "Error resolving instance for key '" + configKey + "', returning empty config: " + Exceptions.toMessageString(e)); + return ConfigPayload.fromBuilder(new ConfigPayloadBuilder()); + } + } + + private ConfigPayload getConfigFromGenericBuilder(ConfigBuilder builder) throws IOException { + return ((GenericConfig.GenericConfigBuilder) builder).getPayload(); + } + + private void applyConfigsOverride(ConfigKey configKey, ConfigBuilder builder, ConfigPayload userOverride, ConfigDefinition targetDef) { + if (userOverride != null) { + ConfigBuilder override = InstanceResolver.createBuilderFromPayload(new ConfigDefinitionKey(configKey), this, userOverride, targetDef); + ConfigInstanceUtil.setValues(builder, override); + } + } + + @Override + public Set<ConfigKey<?>> allConfigsProduced() { + Set<ConfigKey<?>> keySet = new LinkedHashSet<>(); + for (ConfigProducer producer : id2producer().values()) { + keySet.addAll(configsProduced(producer)); + } + return keySet; + } + + public ConfigInstance.Builder createBuilder(ConfigDefinitionKey key, ConfigDefinition targetDef) { + String className = ConfigGenerator.createClassName(key.getName()); + Class<?> clazz; + + final String fullClassName = InstanceResolver.packageName(key) + "." + className; + final String builderName = fullClassName + "$Builder"; + final String producerName = fullClassName + "$Producer"; + ClassLoader classLoader = getConfigClassLoader(producerName); + if (classLoader == null) { + classLoader = getClass().getClassLoader(); + if (logDebug()) { + log.log(LogLevel.DEBUG, "No producer found to get classloader from for " + fullClassName + ". Using default"); + } + } + try { + clazz = classLoader.loadClass(builderName); + } catch (ClassNotFoundException e) { + if (logDebug()) { + log.log(LogLevel.DEBUG, "Tried to load " + builderName + ", not found, trying with generic builder"); + } + // TODO: Enable config compiler when configserver is using new API. + // ConfigCompiler compiler = new LazyConfigCompiler(Files.createTempDir()); + // return compiler.compile(targetDef.generateClass()).newInstance(); + return new GenericConfig.GenericConfigBuilder(key, new ConfigPayloadBuilder()); + } + Object i; + try { + i = clazz.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new ConfigurationRuntimeException(e); + } + if (!(i instanceof ConfigInstance.Builder)) { + throw new ConfigurationRuntimeException(fullClassName + " is not a ConfigInstance.Builder, can not produce config for the name '" + key.getName() + "'."); + } + return (ConfigInstance.Builder) i; + } + + private static boolean logDebug() { + return log.isLoggable(LogLevel.DEBUG); + } + + /** + * The set of all config ids present + * @return set of config ids + */ + public Set<String> allConfigIds() { + return id2producer.keySet(); + } + + @Override + public void distributeFiles(FileDistribution fileDistribution) { + getFileDistributor().sendDeployedFiles(fileDistribution); + } + + @Override + public void reloadDeployFileDistributor(FileDistribution fileDistribution) { + getFileDistributor().reloadDeployFileDistributor(fileDistribution); + } + + @Override + public Optional<ProvisionInfo> getProvisionInfo() { + return info; + } + + private static Set<ConfigKey<?>> configsProduced(ConfigProducer cp) { + Set<ConfigKey<?>> ret = ReflectionUtil.configsProducedByInterface(cp.getClass(), cp.getConfigId()); + UserConfigRepo userConfigs = cp.getUserConfigs(); + for (ConfigDefinitionKey userKey : userConfigs.configsProduced()) { + ret.add(new ConfigKey<>(userKey.getName(), cp.getConfigId(), userKey.getNamespace())); + } + return ret; + } + + @Override + public DeployState getDeployState() { + if (deployState == null) { + throw new IllegalStateException("Cannot call getDeployState() once model has been built"); + } + return deployState; + } + + /** + * @return an unmodifiable copy of the set of configIds in this VespaModel. + */ + public Set<String> getConfigIds() { + return Collections.unmodifiableSet(id2producer.keySet()); + } + + /** + * Returns the admin component of the vespamodel. + * + * @return Admin + */ + public Admin getAdmin() { + return root.getAdmin(); + } + + /** + * Adds the descendant (at any depth level), so it can be looked up + * on configId in the Map. + * + * @param configId the id to register with, not necessarily equal to descendant.getConfigId(). + * @param descendant The configProducer descendant to add + */ + public void addDescendant(String configId, AbstractConfigProducer descendant) { + if (id2producer.containsKey(configId)) { + throw new RuntimeException + ("Config ID '" + configId + "' cannot be reserved by an instance of class '" + + descendant.getClass().getName() + + "' since it is already used by an instance of class '" + + id2producer.get(configId).getClass().getName() + + "'. (This is commonly caused by service/node index " + + "collisions in the config.)"); + } + id2producer.put(configId, descendant); + } + + /** + * Writes MODEL.cfg files for all config producers. + * + * @param baseDirectory dir to write files to + */ + public void writeFiles(File baseDirectory) throws IOException { + super.writeFiles(baseDirectory); + for (ConfigProducer cp : id2producer.values()) { + try { + File destination = new File(baseDirectory, cp.getConfigId().replace("/", File.separator)); + cp.writeFiles(destination); + } catch (IOException e) { + throw new IOException(cp.getConfigId() + ": " + e.getMessage()); + } + } + } + + public Clients getClients() { + return configModelRepo.getClients(); + } + + /** Returns all search clusters, both in Search and Content */ + public List<AbstractSearchCluster> getSearchClusters() { + return Content.getSearchClusters(configModelRepo()); + } + + /** Returns a map of content clusters by ID */ + public Map<String, ContentCluster> getContentClusters() { + Map<String, ContentCluster> clusters = new LinkedHashMap<>(); + for (Content model : configModelRepo.getModels(Content.class)) { + clusters.put(model.getId(), model.getCluster()); + } + return Collections.unmodifiableMap(clusters); + } + + /** Returns a map of container clusters by ID */ + public Map<String, ContainerCluster> getContainerClusters() { + Map<String, ContainerCluster> clusters = new LinkedHashMap<>(); + for (ContainerModel model : configModelRepo.getModels(ContainerModel.class)) { + clusters.put(model.getId(), model.getCluster()); + } + return Collections.unmodifiableMap(clusters); + } + + /** Returns the routing config model. This might be null. */ + public Routing getRouting() { + return configModelRepo.getRouting(); + } + + public FileDistributionConfigProducer getFileDistributionConfigProducer() { + return root.getFileDistributionConfigProducer(); + } + + /** The clusters of application specific generic services */ + public List<ServiceCluster> serviceClusters() { + return serviceClusters; + } + + /** Returns an unmodifiable view of the mapping of config id to {@link ConfigProducer} */ + public Map<String, ConfigProducer> id2producer() { + return Collections.unmodifiableMap(id2producer); + } + + /** + * @return this root's model repository + */ + public ConfigModelRepo configModelRepo() { + return configModelRepo; + } + + @Override + public DeployLogger deployLogger() { + return getDeployState().getDeployLogger(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java new file mode 100644 index 00000000000..0dccc2c9750 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModelFactory.java @@ -0,0 +1,196 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model; + +import com.google.inject.Inject; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.ConfigModelRegistry; +import com.yahoo.config.model.MapConfigModelRegistry; +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.api.ConfigModelPlugin; +import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.ModelCreateResult; +import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.model.application.provider.ApplicationPackageXmlFilesValidator; +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.config.model.deploy.DeployProperties; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.provision.HostsXmlProvisioner; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Version; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.config.VespaVersion; +import com.yahoo.vespa.model.application.validation.Validation; + +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.io.Reader; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Logger; + +/** + * Factory for creating {@link VespaModel} instances. + * + * @author lulf + */ +public class VespaModelFactory implements ModelFactory { + + private static final Logger log = Logger.getLogger(VespaModelFactory.class.getName()); + private final ConfigModelRegistry configModelRegistry; + private final Zone zone; + + @Inject + public VespaModelFactory(ComponentRegistry<ConfigModelPlugin> pluginRegistry, Zone zone) { + List<ConfigModelBuilder> modelBuilders = new ArrayList<>(); + for (ConfigModelPlugin plugin : pluginRegistry.allComponents()) { + if (plugin instanceof ConfigModelBuilder) { + modelBuilders.add((ConfigModelBuilder) plugin); + } + } + this.configModelRegistry = new MapConfigModelRegistry(modelBuilders); + this.zone = zone; + } + + public VespaModelFactory(ConfigModelRegistry configModelRegistry) { + if (configModelRegistry == null) { + this.configModelRegistry = new NullConfigModelRegistry(); + log.info("Will not load config models from plugins, as no registry is available"); + } else { + this.configModelRegistry = configModelRegistry; + } + this.zone = Zone.defaultZone(); + } + + @Override + public Version getVersion() { + return Version.fromIntValues(VespaVersion.major, VespaVersion.minor, VespaVersion.micro); + } + + @Override + public Model createModel(ModelContext modelContext) { + return buildModel(createDeployState(modelContext)); + } + + private VespaModel buildModel(DeployState deployState) { + try { + return new VespaModel(configModelRegistry, deployState); + } catch (IOException | SAXException e) { + throw new RuntimeException(e); + } + } + + private DeployState createDeployState(ModelContext modelContext) { + DeployState.Builder builder = new DeployState.Builder() + .applicationPackage(modelContext.applicationPackage()) + .deployLogger(modelContext.deployLogger()) + .configDefinitionRepo(modelContext.configDefinitionRepo()) + .fileRegistry(modelContext.getFileRegistry()) + .permanentApplicationPackage(modelContext.permanentApplicationPackage()) + .properties(createDeployProperties(modelContext.properties())) + .modelHostProvisioner(createHostProvisioner(modelContext)) + .rotations(modelContext.properties().rotations()) + .zone(zone); + modelContext.previousModel().ifPresent(builder::previousModel); + return builder.build(); + } + + private DeployProperties createDeployProperties(ModelContext.Properties properties) { + return new DeployProperties.Builder() + .applicationId(properties.applicationId()) + .configServerSpecs(properties.configServerSpecs()) + .multitenant(properties.multitenant()) + .hostedVespa(properties.hostedVespa()) + .vespaVersion(getVersion()) + .zone(properties.zone()) + .build(); + } + + + private static HostProvisioner createHostProvisioner(ModelContext modelContext) { + if (isHostedVespaRoutingApplication(modelContext)) { + //TODO: This belongs in HostedVespaProvisioner. + //Added here for now since com.yahoo.config.model.api.HostProvisioner is not created per application, + //and allocation is independent of ApplicationPackage. + return new HostsXmlProvisioner(hostsXml(modelContext)); + } else { + return modelContext.hostProvisioner().orElse( + DeployState.getDefaultModelHostProvisioner(modelContext.applicationPackage())); + } + } + + private static Reader hostsXml(ModelContext modelContext) { + Reader hosts = modelContext.applicationPackage().getHosts(); + if (hosts == null) { + //TODO: throw InvalidApplicationException directly. Not possible now, as it resides in the configserver module. + //SessionPreparer maps IllegalArgumentException -> InvalidApplicationException + throw new IllegalArgumentException("Hosted vespa routing application must use " + ApplicationPackage.HOSTS + + " to allocate hosts."); + } + return hosts; + } + + private static boolean isHostedVespaRoutingApplication(ModelContext modelContext) { + ApplicationId id = modelContext.properties().applicationId(); + return modelContext.properties().hostedVespa() && id.isHostedVespaRoutingApplication(); + } + + @Override + public ModelCreateResult createAndValidateModel(ModelContext modelContext, boolean ignoreValidationErrors) { + if (modelContext.appDir().isPresent()) { + ApplicationPackageXmlFilesValidator validator = + ApplicationPackageXmlFilesValidator.createDefaultXMLValidator(modelContext.appDir().get(), + modelContext.deployLogger(), + modelContext.vespaVersion()); + try { + validator.checkApplication(); + ApplicationPackageXmlFilesValidator.checkIncludedDirs(modelContext.applicationPackage()); + } catch (IllegalArgumentException e) { + rethrowUnlessIgnoreErrors(e, ignoreValidationErrors); + } catch (Exception e) { + throw new RuntimeException(e); + } + + } else { + validateXML(modelContext.applicationPackage(), modelContext.deployLogger(), ignoreValidationErrors); + } + DeployState deployState = createDeployState(modelContext); + VespaModel model = buildModel(deployState); + List<ConfigChangeAction> changeActions = validateModel(model, deployState, ignoreValidationErrors); + return new ModelCreateResult(model, changeActions); + } + + private void validateXML(ApplicationPackage applicationPackage, DeployLogger deployLogger, boolean ignoreValidationErrors) { + try { + applicationPackage.validateXML(deployLogger); + } catch (IllegalArgumentException e) { + rethrowUnlessIgnoreErrors(e, ignoreValidationErrors); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private List<ConfigChangeAction> validateModel(VespaModel model, DeployState deployState, boolean ignoreValidationErrors) { + try { + deployState.getApplicationPackage().validateXML(deployState.getDeployLogger()); + return Validation.validate(model, ignoreValidationErrors, deployState); + } catch (IllegalArgumentException e) { + rethrowUnlessIgnoreErrors(e, ignoreValidationErrors); + } catch (Exception e) { + throw new RuntimeException(e); + } + return new ArrayList<>(); + } + + private static void rethrowUnlessIgnoreErrors(IllegalArgumentException e, boolean ignoreValidationErrors) { + if (!ignoreValidationErrors) { + throw e; + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/AbstractMonitoringSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/AbstractMonitoringSystem.java new file mode 100644 index 00000000000..155dcca03d9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/AbstractMonitoringSystem.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin; + +import java.util.Objects; + +/** + * + * Represents an abstract monitoring service + * + * @author musum + * @since 5.1.20 + * +*/ +class AbstractMonitoringSystem implements MonitoringSystem { + + private final Integer interval; + private final String clustername; + + public AbstractMonitoringSystem(String clustername, Integer interval) { + Objects.requireNonNull(clustername); + Objects.requireNonNull(interval); + this.clustername = clustername; + this.interval = interval; + } + + @Override + public Integer getInterval() { + return interval; + } + + @Override + public Integer getIntervalSeconds() { + return interval * 60; + } + + @Override + public String getClustername() { + return clustername; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java new file mode 100644 index 00000000000..38a1e59433f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -0,0 +1,248 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin; + +import com.yahoo.cloud.config.log.LogdConfig; +import com.yahoo.cloud.config.ZookeepersConfig; +import com.yahoo.config.model.api.ConfigServerSpec; +import com.yahoo.config.model.deploy.DeployProperties; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.cloud.config.SlobroksConfig; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.model.*; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; +import com.yahoo.vespa.model.filedistribution.FileDistributor; +import com.yahoo.vespa.model.filedistribution.FileDistributorService; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import com.yahoo.vespa.model.HostResource; + +/** + * This is the admin pseudo-plugin of the Vespa model, responsible for + * creating all admin services. + * + * @author gjoranv + */ +public class Admin extends AbstractConfigProducer implements Serializable { + + private static final long serialVersionUID = 1L; + + private final Yamas yamas; + private final Map<String,MetricsConsumer> metricsConsumers; + private final List<Configserver> configservers = new ArrayList<>(); + + private final List<Slobrok> slobroks = new ArrayList<>(); + private Configserver defaultConfigserver; + private Logserver logserver; + + /** + * The single cluster controller cluster shared by all content clusters by default when not multitenant. + * If multitenant, this is null. + */ + private ContainerCluster clusterControllers; + + private ZooKeepersConfigProvider zooKeepersConfigProvider; + private FileDistributionConfigProducer fileDistribution; + private final boolean multitenant; + + public Admin(AbstractConfigProducer parent, Yamas yamas, Map<String, MetricsConsumer> metricsConsumers, boolean multitenant) { + super(parent, "admin"); + this.yamas = yamas; + this.metricsConsumers = metricsConsumers; + this.multitenant = multitenant; + } + + public Configserver getConfigserver() { + return defaultConfigserver; + } + + /** Returns the configured yamas end point. Is null if yamas is not configured */ + public Yamas getYamas() { + return yamas; + } + + /** Returns the configured userMetricConsumers. Null if not configured */ + public Map<String, MetricsConsumer> getUserMetricsConsumers(){ + return metricsConsumers; + } + + /** Returns a list of all config servers */ + public List<Configserver> getConfigservers() { + return configservers; + } + + public List<ConfigServerSpec> getConfigServerSpecs() { + List<ConfigServerSpec> serverSpecs = new ArrayList<>(); + for (Configserver server : getConfigservers()) { + serverSpecs.add(server.getConfigServerSpec()); + } + return serverSpecs; + } + + public void removeSlobroks() { slobroks.clear(); } + + /** Returns an immutable list of the slobroks in this */ + public List<Slobrok> getSlobroks() { return Collections.unmodifiableList(slobroks); } + + public void setLogserver(Logserver logserver) { this.logserver = logserver; } + + public Logserver getLogserver() { return logserver; } + + public void addConfigservers(List<Configserver> configservers) { + this.configservers.addAll(configservers); + if (this.configservers.size() > 0) { + this.defaultConfigserver = configservers.get(0); + } + this.zooKeepersConfigProvider = new ZooKeepersConfigProvider(configservers); + } + + public void addSlobroks(List<Slobrok> slobroks) { + this.slobroks.addAll(slobroks); + } + + public ContainerCluster getClusterControllers() { return clusterControllers; } + + public void setClusterControllers(ContainerCluster clusterControllers) { + if (multitenant) throw new RuntimeException("Should not use admin cluster controller in a multitenant environment"); + this.clusterControllers = clusterControllers; + } + + public ZooKeepersConfigProvider getZooKeepersConfigProvider() { + return zooKeepersConfigProvider; + } + + public void getConfig(LogdConfig.Builder builder) { + builder. + logserver(new LogdConfig.Logserver.Builder(). + host(logserver.getHostName()). + port(logserver.getRelativePort(1))); + } + + public void getConfig(SlobroksConfig.Builder builder) { + for (Slobrok slob : slobroks) { + builder. + slobrok(new SlobroksConfig.Slobrok.Builder(). + connectionspec(slob.getConnectionSpec())); + } + } + + public void getConfig(ZookeepersConfig.Builder builder) { + zooKeepersConfigProvider.getConfig(builder); + } + + public void setFileDistribution(FileDistributionConfigProducer fileDistribution) { + this.fileDistribution = fileDistribution; + } + + public FileDistributionConfigProducer getFileDistributionConfigProducer() { + return fileDistribution; + } + + public List<HostResource> getClusterControllerHosts() { + List<HostResource> hosts = new ArrayList<>(); + if (multitenant) { + if (logserver != null) + hosts.add(logserver.getHostResource()); + } else { + for (Configserver configserver : getConfigservers()) { + hosts.add(configserver.getHostResource()); + } + } + return hosts; + } + + /** + * Adds services to all hosts in the system. + */ + public void addPerHostServices(List<HostResource> hosts, DeployProperties properties) { + if (slobroks.isEmpty()) // TODO: Move to caller + slobroks.addAll(createDefaultSlobrokSetup()); + for (HostResource host : hosts) { + if (!host.getHost().isMultitenant()) { + addCommonServices(host, properties); + } + } + } + private void addCommonServices(HostResource host, DeployProperties properties) { + addConfigSentinel(host, properties.applicationId(), properties.zone()); + addLogd(host); + addConfigProxy(host); + addFileDistribution(host); + } + + private void addConfigSentinel(HostResource host, ApplicationId applicationId, Zone zone) { + ConfigSentinel configSentinel = new ConfigSentinel(host.getHost(), applicationId, zone); + addAndInitializeService(host, configSentinel); + host.getHost().setConfigSentinel(configSentinel); + } + + private void addLogd(HostResource host) { + addAndInitializeService(host, new Logd(host.getHost())); + } + + private void addConfigProxy(HostResource host) { + addAndInitializeService(host, new ConfigProxy(host.getHost())); + } + + public void addAndInitializeService(HostResource host, AbstractService service) { + service.setHostResource(host); + service.initService(); + } + + private void addFileDistribution(HostResource host) { + FileDistributor fileDistributor = fileDistribution.getFileDistributor(); + HostResource deployHost = getHostSystem().getHostByHostname(fileDistributor.fileSourceHost()); + if (deployHostIsMissing(deployHost)) { + throw new RuntimeException("Could not find host in the application's host system: '" + + fileDistributor.fileSourceHost() + "'. Hostsystem=" + getHostSystem()); + } + + FileDistributorService fds = new FileDistributorService(fileDistribution, host.getHost().getHostName(), + fileDistribution.getFileDistributor(), fileDistribution.getOptions(), host == deployHost); + fds.setHostResource(host); + fds.initService(); + fileDistribution.addFileDistributionService(host.getHost(), fds); + } + + private boolean deployHostIsMissing(HostResource deployHost) { + return !multitenant && deployHost == null; + } + + // If not configured by user: Use default setup: max 3 slobroks, 1 on the default configserver host + private List<Slobrok> createDefaultSlobrokSetup() { + List<HostResource> hosts = getHostSystem().getHosts(); + List<Slobrok> slobs = new ArrayList<>(); + if (logserver != null) { + Slobrok slobrok = new Slobrok(this, 0); + addAndInitializeService(logserver.getHostResource(), slobrok); + slobs.add(slobrok); + } + + int n = 0; + while ((n < hosts.size()) && (slobs.size() < 3)) { + HostResource host = hosts.get(n); + if ((logserver== null || host != logserver.getHostResource()) && ! host.getHost().isMultitenant()) { + Slobrok newSlobrok = new Slobrok(this, slobs.size()); + addAndInitializeService(host, newSlobrok); + slobs.add(newSlobrok); + } + n++; + } + int j = 0; + for (Slobrok s : slobs) { + s.setProp("index", j); + j++; + } + return slobs; + } + + public boolean multitenant() { + return multitenant; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java new file mode 100644 index 00000000000..d12ab40835d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java @@ -0,0 +1,145 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; + +import java.util.logging.Logger; + +import com.yahoo.config.model.api.ConfigServerSpec; +import com.yahoo.log.LogLevel; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.AbstractService; + +/** + * Represents a Configserver. There may be one or more Configservers in a + * Vespa system. + * + * NOTE: The Configserver is not started by the config system, and + * does not receive any config. It's included here so we know what host + * it runs on, and to give an error message if another service tries + * to reserve the ports it is using. + * + * @author gjoranv + */ +public class Configserver extends AbstractService { + private static final long serialVersionUID = 1L; + public static final int defaultPort = 19070; + private static final Logger log = Logger.getLogger(Configserver.class.getName()); + + public Configserver(AbstractConfigProducer parent, String name) { + super(parent, name); + portsMeta.on(0).tag("rpc").tag("config"); + portsMeta.on(1).tag("http").tag("config").tag("state"); + setProp("clustertype", "admin"); + setProp("clustername", "admin"); + monitorService(); + } + + /** + * Returns the desired base port for this service. + */ + public int getWantedPort() { + try { + // TODO: Provide configserver port as argument when creating this service instead + Process process = new ProcessBuilder(Defaults.getDefaults().vespaHome() + "libexec/vespa/vespa-config.pl", "-configserverport").start(); + InputStream in = process.getInputStream(); + BufferedReader reader = new BufferedReader(new InputStreamReader(in)); + return Integer.parseInt(reader.readLine().trim()); + } catch (Exception exception) { + log.log(LogLevel.DEBUG, "Error reading port from script, using " + defaultPort); + return defaultPort; + } + } + + /** + * The desired base port is the only allowed base port. + * @return 'true' always + */ + public boolean requiresWantedPort() { + return getId() < 2; + } + + /** + * @return the number of ports needed by the configserver. + */ + public int getPortCount() { return 2; } + + /** + * The configserver is not started by the config system! + */ + public boolean getAutostartFlag() { return false; } + + /** + * The configserver is not started by the config system! + */ + public boolean getAutorestartFlag() { return false; } + + private int getConfigServerRpcPort() { + return getRelativePort(0); + } + + private int getConfigServerHttpPort() { + return getRelativePort(1); + } + + public ConfigServerSpec getConfigServerSpec() { + return new Spec(getHostName(), getConfigServerRpcPort(), getConfigServerHttpPort(), ZooKeepersConfigProvider.zkPort); + } + + @Override + public int getHealthPort() { + return getRelativePort(1); + } + + // TODO: Remove this implementation when we are on Hosted Vespa. + public static class Spec implements ConfigServerSpec { + private final String hostName; + private final int configServerPort; + private final int httpPort; + private final int zooKeeperPort; + public String getHostName() { + return hostName; + } + + public int getConfigServerPort() { + return configServerPort; + } + + public int getHttpPort() { + return httpPort; + } + + public int getZooKeeperPort() { + return zooKeeperPort; + } + + @Override + public boolean equals(Object o) { + if (o instanceof ConfigServerSpec) { + ConfigServerSpec other = (ConfigServerSpec)o; + + return hostName.equals(other.getHostName()) && + configServerPort == other.getConfigServerPort() && + httpPort == other.getHttpPort() && + zooKeeperPort == other.getZooKeeperPort(); + } else { + return false; + } + } + + @Override + public int hashCode() { + return hostName.hashCode(); + } + + public Spec(String hostName, int configServerPort, int httpPort, int zooKeeperPort) { + this.hostName = hostName; + this.configServerPort = configServerPort; + this.httpPort = httpPort; + this.zooKeeperPort = zooKeeperPort; + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/DefaultMetricConsumers.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/DefaultMetricConsumers.java new file mode 100644 index 00000000000..6082ca9f72d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/DefaultMetricConsumers.java @@ -0,0 +1,294 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin; + + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * A class to set up the default metrics for all services to be forwarded to Yamas + * + * @author <a href="mailto:trygve@yahoo-inc.com">Trygve Bolsø Berdal</a> + */ +public class DefaultMetricConsumers { + + /** + * Populates a map of with consumer as key and metrics for that consumer as value. The metrics + * are to be forwarded to consumers (ymon and yamas are the options at the moment). + * + * @return A map of default metric consumers and default metrics for that consumer. + */ + public Map<String, MetricsConsumer> getDefaultMetricsConsumers() { + Map<String, MetricsConsumer> metricsConsumers = new LinkedHashMap<>(); + metricsConsumers.put("yamas", getDefaultYamasConsumer()); + metricsConsumers.put("ymon", getDefaultYmonConsumer()); + return metricsConsumers; + } + + private MetricsConsumer getDefaultYmonConsumer(){ + Map<String, Metric> metricMap = new LinkedHashMap<>(); + for (Metric metric : commonMetrics()) { + metricMap.put(metric.getName(), metric); + } + + return new MetricsConsumer("ymon", metricMap); + } + + private MetricsConsumer getDefaultYamasConsumer(){ + // include common metrics + List<Metric> metrics = commonMetrics(); + + //Search node + // jobs + metrics.add(new Metric("content.proton.documentdb.job.total.average")); + metrics.add(new Metric("content.proton.documentdb.job.attribute_flush.average")); + metrics.add(new Metric("content.proton.documentdb.job.memory_index_flush.average")); + metrics.add(new Metric("content.proton.documentdb.job.disk_index_fusion.average")); + metrics.add(new Metric("content.proton.documentdb.job.document_store_flush.average")); + metrics.add(new Metric("content.proton.documentdb.job.document_store_compact.average")); + metrics.add(new Metric("content.proton.documentdb.job.bucket_move.average")); + metrics.add(new Metric("content.proton.documentdb.job.lid_space_compact.average")); + metrics.add(new Metric("content.proton.documentdb.job.removed_documents_prune.average")); + + // lid space + metrics.add(new Metric("content.proton.documentdb.ready.lid_space.lid_bloat_factor.average")); + metrics.add(new Metric("content.proton.documentdb.notready.lid_space.lid_bloat_factor.average")); + metrics.add(new Metric("content.proton.documentdb.removed.lid_space.lid_bloat_factor.average")); + metrics.add(new Metric("content.proton.documentdb.ready.lid_space.lid_fragmentation_factor.average")); + metrics.add(new Metric("content.proton.documentdb.notready.lid_space.lid_fragmentation_factor.average")); + metrics.add(new Metric("content.proton.documentdb.removed.lid_space.lid_fragmentation_factor.average")); + + // resource usage + metrics.add(new Metric("content.proton.resource_usage.disk.average")); + metrics.add(new Metric("content.proton.resource_usage.memory.average")); + metrics.add(new Metric("content.proton.resource_usage.feeding_blocked.last")); + metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.enum_store.average")); + metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.multi_value.average")); + metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.feeding_blocked.last")); + + // transaction log + metrics.add(new Metric("content.proton.transactionlog.entries.average")); + + // document store + metrics.add(new Metric("content.proton.documentdb.ready.document_store.disk_usage.average")); + metrics.add(new Metric("content.proton.documentdb.ready.document_store.disk_bloat.average")); + metrics.add(new Metric("content.proton.documentdb.ready.document_store.max_bucket_spread.average")); + metrics.add(new Metric("content.proton.documentdb.notready.document_store.disk_usage.average")); + metrics.add(new Metric("content.proton.documentdb.notready.document_store.disk_bloat.average")); + metrics.add(new Metric("content.proton.documentdb.notready.document_store.max_bucket_spread.average")); + metrics.add(new Metric("content.proton.documentdb.removed.document_store.disk_usage.average")); + metrics.add(new Metric("content.proton.documentdb.removed.document_store.disk_bloat.average")); + metrics.add(new Metric("content.proton.documentdb.removed.document_store.max_bucket_spread.average")); + + + //Storage + metrics.add(new Metric("vds.memfilepersistence.cache.files.average")); + metrics.add(new Metric("vds.memfilepersistence.cache.body.average")); + metrics.add(new Metric("vds.memfilepersistence.cache.header.average")); + metrics.add(new Metric("vds.memfilepersistence.cache.meta.average")); + metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.average")); + metrics.add(new Metric("vds.visitor.allthreads.completed.sum.average")); + metrics.add(new Metric("vds.visitor.allthreads.created.sum.rate","visit")); + + metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.average")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.average")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.average")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.average")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.splitbuckets.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.joinbuckets.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.setbucketstates.count.rate")); + + metrics.add(new Metric("vds.filestor.spi.put.success.average")); + metrics.add(new Metric("vds.filestor.spi.remove.success.average")); + metrics.add(new Metric("vds.filestor.spi.update.success.average")); + metrics.add(new Metric("vds.filestor.spi.get.success.average")); + metrics.add(new Metric("vds.filestor.spi.iterate.success.average")); + metrics.add(new Metric("vds.filestor.spi.put.success.rate")); + metrics.add(new Metric("vds.filestor.spi.remove.success.rate")); + metrics.add(new Metric("vds.filestor.spi.update.success.rate")); + metrics.add(new Metric("vds.filestor.spi.get.success.rate")); + metrics.add(new Metric("vds.filestor.spi.iterate.success.rate")); + + + //Distributor + metrics.add(new Metric("vds.visitor.sum.latency.average")); + metrics.add(new Metric("vds.visitor.sum.failed.rate")); + metrics.add(new Metric("vds.idealstate.buckets_rechecking.average")); + metrics.add(new Metric("vds.idealstate.idealstate_diff.average")); + metrics.add(new Metric("vds.idealstate.buckets_toofewcopies.average")); + metrics.add(new Metric("vds.idealstate.buckets_toomanycopies.average")); + metrics.add(new Metric("vds.idealstate.buckets.average")); + metrics.add(new Metric("vds.idealstate.buckets_notrusted.average")); + + + metrics.add(new Metric("vds.distributor.puts.sum.latency.average")); + metrics.add(new Metric("vds.distributor.puts.sum.ok.rate")); + metrics.add(new Metric("vds.distributor.puts.sum.failures.total.rate")); + metrics.add(new Metric("vds.distributor.removes.sum.latency.average")); + metrics.add(new Metric("vds.distributor.removes.sum.ok.rate")); + metrics.add(new Metric("vds.distributor.removes.sum.failures.total.rate")); + metrics.add(new Metric("vds.distributor.updates.sum.latency.average")); + metrics.add(new Metric("vds.distributor.updates.sum.ok.rate")); + metrics.add(new Metric("vds.distributor.updates.sum.failures.total.rate")); + metrics.add(new Metric("vds.distributor.removelocations.sum.latency.average")); + metrics.add(new Metric("vds.distributor.removelocations.sum.ok.rate")); + metrics.add(new Metric("vds.distributor.removelocations.sum.failures.total.rate")); + metrics.add(new Metric("vds.distributor.gets.sum.latency.average")); + metrics.add(new Metric("vds.distributor.gets.sum.ok.rate")); + metrics.add(new Metric("vds.distributor.gets.sum.failures.total.rate")); + metrics.add(new Metric("vds.distributor.docsstored.average")); + metrics.add(new Metric("vds.distributor.bytesstored.average")); + metrics.add(new Metric("vds.visitor.sum.latency.average")); + metrics.add(new Metric("vds.visitor.sum.failed.rate")); + + // Cluster Controller + + metrics.add(new Metric("cluster-controller.down.count.last")); + metrics.add(new Metric("cluster-controller.initializing.count.last")); + metrics.add(new Metric("cluster-controller.maintenance.count.last")); + metrics.add(new Metric("cluster-controller.retired.count.last")); + metrics.add(new Metric("cluster-controller.stopping.count.last")); + metrics.add(new Metric("cluster-controller.up.count.last")); + metrics.add(new Metric("cluster-controller.cluster-state-change.count", "content.cluster-controller.cluster-state-change.count")); + + metrics.add(new Metric("cluster-controller.is-master.last")); + // TODO(hakon): Update this name once persistent "count" metrics has been implemented. + // DO NOT RELY ON THIS METRIC YET. + metrics.add(new Metric("cluster-controller.node-event.count")); + + //Errors from qrserver + metrics.add(new Metric("error.timeout.rate","error.timeout")); + metrics.add(new Metric("error.backends_oos.rate","error.backends_oos")); + metrics.add(new Metric("error.plugin_failure.rate","error.plugin_failure")); + metrics.add(new Metric("error.backend_communication_error.rate","error.backend_communication_error")); + metrics.add(new Metric("error.empty_document_summaries.rate","error.empty_document_summaries")); + metrics.add(new Metric("error.invalid_query_parameter.rate","error.invalid_query_parameter")); + metrics.add(new Metric("error.internal_server_error.rate", "error.internal_server_error")); + metrics.add(new Metric("error.misconfigured_server.rate","error.misconfigured_server")); + metrics.add(new Metric("error.invalid_query_transformation.rate","error.invalid_query_transformation")); + metrics.add(new Metric("error.result_with_errors.rate","error.result_with_errors")); + metrics.add(new Metric("error.unspecified.rate","error.unspecified")); + metrics.add(new Metric("error.unhandled_exception.rate","error.unhandled_exception")); + metrics.add(new Metric("http.status.1xx.rate")); + metrics.add(new Metric("http.status.2xx.rate")); + metrics.add(new Metric("http.status.3xx.rate")); + metrics.add(new Metric("http.status.4xx.rate")); + metrics.add(new Metric("http.status.5xx.rate")); + + + // container + metrics.add(new Metric("serverRejectedRequests.rate")); + metrics.add(new Metric("serverRejectedRequests.count")); + + metrics.add(new Metric("serverThreadPoolSize.average")); + metrics.add(new Metric("serverThreadPoolSize.min")); + metrics.add(new Metric("serverThreadPoolSize.max")); + metrics.add(new Metric("serverThreadPoolSize.rate")); + metrics.add(new Metric("serverThreadPoolSize.count")); + metrics.add(new Metric("serverThreadPoolSize.last")); + + metrics.add(new Metric("serverActiveThreads.average")); + metrics.add(new Metric("serverActiveThreads.min")); + metrics.add(new Metric("serverActiveThreads.max")); + metrics.add(new Metric("serverActiveThreads.rate")); + metrics.add(new Metric("serverActiveThreads.count")); + metrics.add(new Metric("serverActiveThreads.last")); + + metrics.add(new Metric("httpapi_latency.average")); + metrics.add(new Metric("httpapi_pending.average")); + metrics.add(new Metric("httpapi_num_operations.rate")); + metrics.add(new Metric("httpapi_num_updates.rate")); + metrics.add(new Metric("httpapi_num_removes.rate")); + metrics.add(new Metric("httpapi_num_puts.rate")); + metrics.add(new Metric("httpapi_succeeded.rate")); + metrics.add(new Metric("httpapi_failed.rate")); + + + // Config server + metrics.add(new Metric("configserver.requests.count", "configserver.requests")); + metrics.add(new Metric("configserver.failedRequests.count", "configserver.failedRequests")); + metrics.add(new Metric("configserver.latency.average", "configserver.latency")); + metrics.add(new Metric("configserver.cacheConfigElems.last", "configserver.cacheConfigElems")); + metrics.add(new Metric("configserver.cacheChecksumElems.last", "configserver.cacheChecksumElems")); + metrics.add(new Metric("configserver.hosts.last", "configserver.hosts")); + metrics.add(new Metric("configserver.delayedResponses.count", "configserver.delayedResponses")); + metrics.add(new Metric("configserver.sessionChangeErrors.count", "configserver.sessionChangeErrors")); + + + Map<String, Metric> metricMap = new LinkedHashMap<>(); + for (Metric metric : metrics) { + metricMap.put(metric.getName(), metric); + } + + return new MetricsConsumer("yamas", metricMap); + } + + // Common metrics for ymon and yamas. For ymon metric names needs to be less then 19 characters long + private List<Metric> commonMetrics(){ + List<Metric> metrics = new ArrayList<>(); + + //Searchnode + metrics.add(new Metric("proton.numstoreddocs.last", "documents_total")); + metrics.add(new Metric("proton.numindexeddocs.last", "documents_ready")); + metrics.add(new Metric("proton.numactivedocs.last", "documents_active")); + metrics.add(new Metric("proton.numremoveddocs.last", "documents_removed")); + + metrics.add(new Metric("proton.docsinmemory.last", "documents_inmemory")); + metrics.add(new Metric("proton.diskusage.last", "diskusage")); + metrics.add(new Metric("proton.memoryusage.max", "content.proton.memoryusage.max")); + metrics.add(new Metric("proton.transport.query.count.rate", "query_requests")); + metrics.add(new Metric("proton.transport.docsum.docs.rate", "document_requests")); + metrics.add(new Metric("proton.transport.docsum.latency.average", "content.proton.transport.docsum.latency.average")); + metrics.add(new Metric("proton.transport.query.latency.average", "query_latency")); + + //Docproc - per chain + metrics.add(new Metric("documents_processed.rate", "documents_processed")); + + //Qrserver + metrics.add(new Metric("peak_qps.average", "peak_qps")); + metrics.add(new Metric("search_connections.average", "search_connections")); + metrics.add(new Metric("active_queries.average", "active_queries")); + metrics.add(new Metric("queries.rate", "queries")); + metrics.add(new Metric("query_latency.average", "mean_query_latency")); + metrics.add(new Metric("query_latency.max", "max_query_latency")); + metrics.add(new Metric("query_latency.95percentile", "95p_query_latency")); + metrics.add(new Metric("query_latency.99percentile", "99p_query_latency")); + metrics.add(new Metric("failed_queries.rate", "failed_queries")); + metrics.add(new Metric("hits_per_query.average", "hits_per_query")); + metrics.add(new Metric("empty_results.rate", "empty_results")); + metrics.add(new Metric("requestsOverQuota.rate")); + metrics.add(new Metric("requestsOverQuota.count")); + + //Storage + metrics.add(new Metric("vds.datastored.alldisks.docs.average","docs")); + metrics.add(new Metric("vds.datastored.alldisks.bytes.average","bytes")); + metrics.add(new Metric("vds.visitor.allthreads.averagevisitorlifetime.sum.average","visitorlifetime")); + metrics.add(new Metric("vds.visitor.allthreads.averagequeuewait.sum.average","visitorqueuewait")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.count.rate","put")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.count.rate","remove")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.count.rate","get")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.count.rate","update")); + metrics.add(new Metric("vds.filestor.alldisks.queuesize.average","diskqueuesize")); + metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.average","diskqueuewait")); + + + //Distributor + metrics.add(new Metric("vds.idealstate.delete_bucket.done_ok.rate","deleteok")); + metrics.add(new Metric("vds.idealstate.delete_bucket.done_failed.rate","deletefailed")); + metrics.add(new Metric("vds.idealstate.delete_bucket.pending.average","deletepending")); + metrics.add(new Metric("vds.idealstate.merge_bucket.done_ok.rate","mergeok")); + metrics.add(new Metric("vds.idealstate.merge_bucket.done_failed.rate","mergefailed")); + metrics.add(new Metric("vds.idealstate.merge_bucket.pending.average","mergepending")); + metrics.add(new Metric("vds.idealstate.split_bucket.done_ok.rate","splitok")); + metrics.add(new Metric("vds.idealstate.split_bucket.done_failed.rate","splitfailed")); + metrics.add(new Metric("vds.idealstate.split_bucket.pending.average","splitpending")); + metrics.add(new Metric("vds.idealstate.join_bucket.done_ok.rate","joinok")); + metrics.add(new Metric("vds.idealstate.join_bucket.done_failed.rate","joinfailed")); + metrics.add(new Metric("vds.idealstate.join_bucket.pending.average","joinpending")); + + return metrics; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java new file mode 100644 index 00000000000..a5115d6c358 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/FileDistributionOptions.java @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin; + +import com.yahoo.binaryprefix.BinaryPrefix; +import com.yahoo.binaryprefix.BinaryScaledAmount; +import com.yahoo.cloud.config.filedistribution.FiledistributorConfig; + +/** + * Options for controlling the behavior of the file distribution services. + * @author tonytv + */ +public class FileDistributionOptions implements FiledistributorConfig.Producer { + public static FileDistributionOptions defaultOptions() { + return new FileDistributionOptions(); + } + + private FileDistributionOptions() {} + + private BinaryScaledAmount uploadbitrate = new BinaryScaledAmount(); + private BinaryScaledAmount downloadbitrate = new BinaryScaledAmount(); + + //Called through reflection + public void downloadbitrate(BinaryScaledAmount amount) { + ensureNonNegative(amount); + downloadbitrate = amount; + } + + //Called through reflection + public void uploadbitrate(BinaryScaledAmount amount) { + ensureNonNegative(amount); + uploadbitrate = amount; + } + + private void ensureNonNegative(BinaryScaledAmount amount) { + if (amount.amount < 0) + throw new IllegalArgumentException("Expected non-negative number, got " + amount.amount); + } + + private int byteRate(BinaryScaledAmount bitRate) { + BinaryScaledAmount byteRate = bitRate.divide(8); + return (int)byteRate.as(BinaryPrefix.unit); + } + + @Override + public void getConfig(FiledistributorConfig.Builder builder) { + builder.maxuploadspeed((double)byteRate(uploadbitrate)); + builder.maxdownloadspeed((double)byteRate(downloadbitrate)); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java new file mode 100644 index 00000000000..12341297ebb --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Logserver.java @@ -0,0 +1,75 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.AbstractService; + +/** + * Represents the Logserver. There is exactly one logserver in a Vespa + * system. + * + * @author gjoranv + */ +public class Logserver extends AbstractService { + private static final long serialVersionUID = 1L; + private static final String logArchiveDir = "$ROOT/logs/vespa/logarchive"; + + public Logserver(AbstractConfigProducer parent) { + super(parent, "logserver"); + portsMeta.on(0).tag("unused"); + portsMeta.on(1).tag("logtp"); + portsMeta.on(2).tag("logtp").tag("telnet").tag("last-errors-holder"); + portsMeta.on(3).tag("logtp").tag("telnet").tag("replicator"); + setProp("clustertype", "admin"); + setProp("clustername", "admin"); + monitorService(); + } + + /** + * @return the startup command for the logserver + */ + public String getStartupCommand() { + return "exec $ROOT/bin/logserver-start " + getMyJVMArgs() + " " + getJvmArgs(); + } + + /** + * @return the jvm args to be used by the logserver. + */ + private String getMyJVMArgs() { + StringBuilder sb = new StringBuilder(); + sb.append("-Dlogserver.http.port=").append(getRelativePort(0)); + sb.append(" "); + sb.append("-Dlogserver.listenport=").append(getRelativePort(1)); + sb.append(" "); + sb.append("-Dlogserver.last-errors-holder.port=").append(getRelativePort(2)); + sb.append(" "); + sb.append("-Dlogserver.replicator.port=").append(getRelativePort(3)); + sb.append(" "); + sb.append("-Dlogserver.logarchive.dir=" + logArchiveDir); + return sb.toString(); + } + + /** + * Returns the desired base port for this service. + */ + public int getWantedPort() { + return 19080; + } + + /** + * The desired base port is the only allowed base port. + * + * @return 'true' always + */ + public boolean requiresWantedPort() { + return true; + } + + /** + * @return the number of ports needed by the logserver. + */ + public int getPortCount() { + return 4; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Metric.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Metric.java new file mode 100644 index 00000000000..e1056f954dc --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Metric.java @@ -0,0 +1,99 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin; + +import java.util.HashMap; +import java.util.Map; + +/** + * Helper class to model a metric. + * + * @author trygve + */ +public class Metric { + + private final String name; + private String outputName; + private final Map<String, String> dimensions = new HashMap<>(); + private final String description; + + /** + * @param name The metric name + * @param outputName The name of the metric in yamas + * @param description The description of this metric + */ + public Metric(String name, String outputName, String description) { + this.name = name; + this.outputName = outputName; + this.description = description; + } + + /** + * Creates a metric with empty dimensions and consumers containing the default consumer + * + * @param name the metric name + * @param outputName name tp be used in yamas + */ + public Metric(String name, String outputName) { + this(name, outputName, ""); + } + + /** + * Creates a metric with same outputname as metricname and empty dimensions and consumers containing the default consumer and + * + * @param name The name of the metric, same name used for outputname + */ + public Metric(String name) { + this(name, name); + } + + public String getDescription() { + return this.description; + } + + + public String getOutputName() { + return outputName; + } + + public void setOutputName(String outputName) { + this.outputName = outputName; + } + + public String getName() { + return name; + } + + public Map<String, String> getDimensions() { + return dimensions; + } + + @Override + public String toString() { + return "Metric{" + + "name='" + name + '\'' + + ", outputName='" + outputName + '\'' + + ", dimensions=" + dimensions + + '}'; + } + + + /** + * Two metrics are considered equal if they have the same name. + */ + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Metric metric = (Metric) o; + + return name.equals(metric.name); + + } + + @Override + public int hashCode() { + return name.hashCode(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/MetricsConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/MetricsConsumer.java new file mode 100644 index 00000000000..0bed49f38ac --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/MetricsConsumer.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin; + +import java.util.Map; + +/** + * Represents an arbitrary metric consumer + * + * @author trygve + */ +public class MetricsConsumer { + private final String consumer; + private final Map<String, Metric> metrics; + + /** + * @param consumer The consumer + * @param metrics The metrics for the the consumer + */ + public MetricsConsumer(String consumer, Map<String, Metric> metrics) { + this.consumer = consumer; + this.metrics = metrics; + } + + public String getConsumer() { + return consumer; + } + + /** + * @return Map of metric with metric name as key + */ + public Map<String, Metric> getMetrics() { + return metrics; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/ModelConfigProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/ModelConfigProvider.java new file mode 100644 index 00000000000..1e9b485a939 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/ModelConfigProvider.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin; + +import com.yahoo.config.model.ApplicationConfigProducerRoot; +import com.yahoo.config.model.producer.AbstractConfigProducer; + +/** + * A config provider for the model config. The ModelConfig is a common config and produced by {@link ApplicationConfigProducerRoot} , this config + * producer exists to make the admin/model config id exist for legacy reasons. + * @author <a href="mailto:musum@yahoo-inc.com">musum</a> + * @author gjoranv + * @since 5.0.8 + */ +public class ModelConfigProvider extends AbstractConfigProducer { + + public ModelConfigProvider(AbstractConfigProducer<?> parent) { + super(parent, "model"); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/MonitoringSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/MonitoringSystem.java new file mode 100644 index 00000000000..de95c9a8d9a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/MonitoringSystem.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin; + +/** + * Interface for different monitoring services + * + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + */ +public interface MonitoringSystem { + /** + * @return Snapshot interval in minutes + */ + public Integer getInterval(); + + /** + * @return Snapshot interval in seconds. + */ + public Integer getIntervalSeconds(); + + /** + * @return the monitoring cluster name + */ + public String getClustername(); +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java new file mode 100644 index 00000000000..a2b7c7a5532 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Slobrok.java @@ -0,0 +1,77 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.AbstractService; + +/** + * Represents a Slobrok service. + * + * @author gjoranv + */ +public class Slobrok extends AbstractService { + private static final long serialVersionUID = 1L; + + /** + * @param parent The parent ConfigProducer. + * @param index unique index for all slobroks + */ + public Slobrok(AbstractConfigProducer parent, int index) { + super(parent, "slobrok." + index); + portsMeta.on(0).tag("rpc").tag("admin").tag("status"); + portsMeta.on(1).tag("http").tag("state"); + setProp("index", index); + setProp("clustertype", "slobrok"); + setProp("clustername", "admin"); + monitorService(); + } + + @Override + public boolean requiresConsecutivePorts() { + return false; + } + + @Override + public int getWantedPort() { + if (getId() == 1) { + return 19099; + } else { + return 0; + } + } + + public String getStartupCommand() { + return "exec $ROOT/bin/slobrok -p " + getPort() + + " -s " + getStatePort() + + " -c " + getConfigId(); + } + + /** + * @return The number of ports needed by the slobrok. + */ + public int getPortCount() { + return 2; + } + + /** + * @return The port on which this slobrok should respond, as a String. + */ + public String getPort() { + return String.valueOf(getRelativePort(0)); + } + + /** + * @return The port on which the state server should respond + */ + public String getStatePort() { + return String.valueOf(getRelativePort(1)); + } + + /** + * @return The connection spec to this Slobrok + */ + public String getConnectionSpec() { + return "tcp/" + getHostName() + ":" + getPort(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Yamas.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Yamas.java new file mode 100644 index 00000000000..43775fe0dec --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Yamas.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin; + +import java.io.Serializable; +import java.util.Objects; + +/** + * Properties for yamas monitoring service + * + * @author musum + * @since 5.1.20 + */ +public class Yamas extends AbstractMonitoringSystem implements Serializable { + public Yamas(String clustername, Integer interval) { + super(clustername, interval); + } +} + + diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/ZooKeepersConfigProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/ZooKeepersConfigProvider.java new file mode 100644 index 00000000000..6c8b486c8a6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/ZooKeepersConfigProvider.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin; + +import com.yahoo.collections.CollectionUtil; +import com.yahoo.cloud.config.ZookeepersConfig; +import com.yahoo.config.model.api.ConfigServerSpec; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author tonytv + */ +public class ZooKeepersConfigProvider implements ZookeepersConfig.Producer { + + public static final int zkPort = Integer.getInteger("zk_port", 2181); + + private final List<Configserver> configServers; + + public ZooKeepersConfigProvider(List<Configserver> configServers) { + if (configServers == null) { + configServers = new ArrayList<>(); + } + this.configServers = configServers; + } + + // format for each element: hostname:port + public List<String> getZooKeepers() { + List<String> servers = new ArrayList<>(); + for (Configserver server : configServers) { + ConfigServerSpec serverSpec = server.getConfigServerSpec(); + servers.add(serverSpec.getHostName() + ":" + serverSpec.getZooKeeperPort()); + + } + return servers; + } + + // format: hostname1:port2,hostname2:port2,... + public String getZooKeepersConnectionSpec() { + return CollectionUtil.mkString(getZooKeepers(), ","); + } + + @Override + public void getConfig(ZookeepersConfig.Builder builder) { + builder.zookeeperserverlist(getZooKeepersConnectionSpec()); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerCluster.java new file mode 100644 index 00000000000..b82f58fbb40 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerCluster.java @@ -0,0 +1,77 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin.clustercontroller; + +import com.google.common.base.Joiner; +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.cloud.config.ZookeepersConfig; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.Service; +import com.yahoo.vespa.model.admin.Configserver; +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerCluster; + +import java.util.ArrayList; +import java.util.Collection; + +/** + * Used if clustercontroller is run standalone (not as part of the config server ZooKeeper cluster) + * to provide common configs to container components. + * + * @author lulf + * @since 5.6 + */ +public class ClusterControllerCluster extends AbstractConfigProducer<ContainerCluster> implements ZookeeperServerConfig.Producer, ZookeepersConfig.Producer { + + private static final int ZK_CLIENT_PORT = 2181; + private ContainerCluster containerCluster = null; + + public ClusterControllerCluster(AbstractConfigProducer parent, String subId) { + super(parent, subId); + } + + @Override + public void getConfig(ZookeeperServerConfig.Builder builder) { + builder.clientPort(ZK_CLIENT_PORT); + for (Container c : containerCluster.getContainers()) { + ClusterControllerContainer container = (ClusterControllerContainer) c; + ZookeeperServerConfig.Server.Builder serverBuilder = new ZookeeperServerConfig.Server.Builder(); + serverBuilder.hostname(container.getHostName()); + serverBuilder.id(container.getIndex()); + builder.server(serverBuilder); + } + } + + @Override + public void getConfig(ZookeepersConfig.Builder builder) { + final Collection<String> controllerHosts = new ArrayList<>(); + for (Container container : containerCluster.getContainers()) { + controllerHosts.add(container.getHostName() + ":" + ZK_CLIENT_PORT); + } + builder.zookeeperserverlist(Joiner.on(",").join(controllerHosts)); + } + + @Override + protected void addChild(ContainerCluster cluster) { + super.addChild(cluster); + this.containerCluster = cluster; + } + + @Override + public void validate() { + assert(containerCluster != null); + for (Container c1 : containerCluster.getContainers()) { + assert(c1 instanceof ClusterControllerContainer); + for (Service service : c1.getHostResource().getServices()) { + if (service instanceof Configserver) { + throw new IllegalArgumentException("Error validating cluster controller cluster: cluster controller '" + c1.getConfigId() + "' is set to run on the same host as a configserver"); + } + } + for (Container c2 : containerCluster.getContainers()) { + if (c1 != c2 && c1.getHostName().equals(c2.getHostName())) { + throw new IllegalArgumentException("Error validating cluster controller cluster: cluster controllers '" + c1.getConfigId() + "' and '" + c2.getConfigId() + "' share the same host"); + } + } + } + } +} + diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerComponent.java new file mode 100644 index 00000000000..8ec1cecf47a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerComponent.java @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin.clustercontroller; + +import com.yahoo.component.ComponentSpecification; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.component.SimpleComponent; + +/** + * Sets up a simple component to keep the state of the cluster controller, even when configuration changes. + */ +public class ClusterControllerComponent extends SimpleComponent +{ + public ClusterControllerComponent() { + super(new ComponentModel(new BundleInstantiationSpecification( + new ComponentSpecification("clustercontroller"), + new ComponentSpecification("com.yahoo.vespa.clustercontroller.apps.clustercontroller.ClusterController"), + new ComponentSpecification("clustercontroller-apps")))); + } +}
\ No newline at end of file diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerConfigurer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerConfigurer.java new file mode 100644 index 00000000000..e3e79a57261 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerConfigurer.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin.clustercontroller; + +import com.yahoo.component.ComponentSpecification; +import com.yahoo.vespa.config.content.FleetcontrollerConfig; +import com.yahoo.vespa.config.content.StorDistributionConfig; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.component.SimpleComponent; +import com.yahoo.vespa.model.content.cluster.ContentCluster; + +/** + * Model serving class. Wraps fleet controller config serving. + */ +public class ClusterControllerConfigurer extends SimpleComponent implements StorDistributionConfig.Producer, + FleetcontrollerConfig.Producer +{ + private final ContentCluster cluster; + private final int index; + private final int nodeCount; + + public ClusterControllerConfigurer(ContentCluster cluster, int index, int nodeCount) { + super(new ComponentModel(new BundleInstantiationSpecification( + new ComponentSpecification("clustercontroller" + "-" + cluster.getName() + "-configurer"), + new ComponentSpecification("com.yahoo.vespa.clustercontroller.apps.clustercontroller.ClusterControllerClusterConfigurer"), + new ComponentSpecification("clustercontroller-apps")))); + this.cluster = cluster; + this.index = index; + this.nodeCount = nodeCount; + } + + @Override + public void getConfig(StorDistributionConfig.Builder builder) { + cluster.getConfig(builder); + } + + @Override + public void getConfig(FleetcontrollerConfig.Builder builder) { + cluster.getConfig(builder); + cluster.getClusterControllerConfig().getConfig(builder); + builder.index(index); + builder.fleet_controller_count(nodeCount); + builder.http_port(0); + builder.rpc_port(0); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java new file mode 100644 index 00000000000..1290b0b22d6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -0,0 +1,110 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.admin.clustercontroller; + +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.container.BundlesConfig; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.log.LogLevel; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.config.content.FleetcontrollerConfig; +import static com.yahoo.vespa.defaults.Defaults.getDefaults; +import com.yahoo.vespa.model.application.validation.RestartConfigs; +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.component.AccessLogComponent; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.Handler; + +import java.util.Set; +import java.util.TreeSet; + +/** + * Extends the container producer to allow us to override ports. + */ +@RestartConfigs({FleetcontrollerConfig.class, ZookeeperServerConfig.class}) +public class ClusterControllerContainer extends Container implements BundlesConfig.Producer, ZookeeperServerConfig.Producer { + private static final ComponentSpecification CLUSTERCONTROLLER_BUNDLE = new ComponentSpecification("clustercontroller-apps"); + private static final ComponentSpecification ZKFACADE_BUNDLE = new ComponentSpecification("zkfacade"); + private final int index; + + private final Set<String> bundles = new TreeSet<>(); + + public ClusterControllerContainer(AbstractConfigProducer parent, int index, boolean runStandaloneZooKeeper) { + super(parent, "" + index); + this.index = index; + addHandler( + new Handler(new ComponentModel(new BundleInstantiationSpecification( + new ComponentSpecification("clustercontroller-status"), + new ComponentSpecification("com.yahoo.vespa.clustercontroller.apps.clustercontroller.StatusHandler"), + CLUSTERCONTROLLER_BUNDLE))), "clustercontroller-status/*" + ); + addHandler( + new Handler(new ComponentModel(new BundleInstantiationSpecification( + new ComponentSpecification("clustercontroller-state-restapi-v2"), + new ComponentSpecification("com.yahoo.vespa.clustercontroller.apps.clustercontroller.StateRestApiV2Handler"), + CLUSTERCONTROLLER_BUNDLE))), "cluster/v2/*" + ); + if (runStandaloneZooKeeper) { + addComponent(new Component<>(new ComponentModel(new BundleInstantiationSpecification( + new ComponentSpecification("clustercontroller-zkrunner"), + new ComponentSpecification("com.yahoo.vespa.zookeeper.ZooKeeperServer"), ZKFACADE_BUNDLE)))); + addComponent(new Component<>(new ComponentModel(new BundleInstantiationSpecification( + new ComponentSpecification("clustercontroller-zkprovider"), + new ComponentSpecification("com.yahoo.vespa.clustercontroller.apps.clustercontroller.StandaloneZooKeeperProvider"), CLUSTERCONTROLLER_BUNDLE)))); + } else { + addComponent(new Component<>(new ComponentModel(new BundleInstantiationSpecification( + new ComponentSpecification("clustercontroller-zkprovider"), + new ComponentSpecification("com.yahoo.vespa.clustercontroller.apps.clustercontroller.DummyZooKeeperProvider"), CLUSTERCONTROLLER_BUNDLE)))); + } + addBundle("file:" + getDefaults().underVespaHome("lib/jars/clustercontroller-apps-jar-with-dependencies.jar")); + addBundle("file:" + getDefaults().underVespaHome("lib/jars/clustercontroller-apputil-jar-with-dependencies.jar")); + addBundle("file:" + getDefaults().underVespaHome("lib/jars/clustercontroller-core-jar-with-dependencies.jar")); + addBundle("file:" + getDefaults().underVespaHome("lib/jars/clustercontroller-utils-jar-with-dependencies.jar")); + addBundle("file:" + getDefaults().underVespaHome("lib/jars/zkfacade-jar-with-dependencies.jar")); + + log.log(LogLevel.DEBUG, "Adding access log for cluster controller ..."); + addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.queryAccessLog, "controller")); + } + + @Override + public int getWantedPort() { + return 19050; + } + + @Override + public boolean requiresWantedPort() { + return index == 0; + } + + @Override + public String getServiceType() { + return "container-clustercontroller"; + } + + private void addHandler(Handler h, String binding) { + h.addServerBindings("http://*/" + binding, + "https://*/" + binding); + super.addHandler(h); + } + + public void addBundle(String bundlePath) { + bundles.add(bundlePath); + } + + @Override + public void getConfig(BundlesConfig.Builder builder) { + for (String bundle : bundles) { + builder.bundle(bundle); + } + } + + @Override + public void getConfig(ZookeeperServerConfig.Builder builder) { + builder.myid(index); + } + + int getIndex() { + return index; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/package-info.java new file mode 100644 index 00000000000..bd3556cc605 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.model.admin.clustercontroller; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/package-info.java new file mode 100644 index 00000000000..e3c7221f7b7 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/package-info.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * Provides the classes for the admin components of the Vespa config + * model. + * + * The {@link com.yahoo.vespa.model.admin.Admin Admin} class is + * the natural starting point. It takes the admin part of the + * user-defined application specification as input argument to its + * constructor. The services given in the specification are + * instantiated, in addition to some mandatory services (e.g. {@link + * com.yahoo.vespa.model.admin.Slobrok Slobrok}) that will be created + * even if the user does not specify them. + */ +@ExportPackage +package com.yahoo.vespa.model.admin; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComponentValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComponentValidator.java new file mode 100644 index 00000000000..00553f681fe --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComponentValidator.java @@ -0,0 +1,88 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.path.Path; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.application.api.DeployLogger; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.jar.Attributes; +import java.util.jar.JarFile; +import java.util.jar.Manifest; +import java.util.logging.Level; +import java.util.zip.ZipException; + +/** + * A validator for bundles. Uses BND library for some of the validation (not active yet) + * + * @author musum + * @since 2010-11-11 + */ +public class ComponentValidator extends Validator { + private JarFile jarFile; + + public ComponentValidator() { + } + + public ComponentValidator(JarFile jarFile) { + this.jarFile = jarFile; + } + + @Override + public void validate(VespaModel model, DeployState deployState) { + ApplicationPackage app = deployState.getApplicationPackage(); + for (ComponentInfo info : app.getComponentsInfo(deployState.getProperties().vespaVersion())) { + try { + this.jarFile = new JarFile(app.getFileReference(Path.fromString(info.getPathRelativeToAppDir()))); + } catch (ZipException e) { + throw new IllegalArgumentException("Error opening jar file '" + info.getPathRelativeToAppDir() + + "'. Please check that this is a valid jar file"); + } catch (IOException e) { + e.printStackTrace(); + } + try { + validateAll(deployState.getDeployLogger()); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + public void validateAll(DeployLogger deployLogger) throws IOException { + validateOSGIHeaders(deployLogger); + } + + public void validateOSGIHeaders(DeployLogger deployLogger) throws IOException { + Manifest mf = jarFile.getManifest(); + if (mf == null) { + throw new IllegalArgumentException("Non-existing or invalid manifest in " + jarFile.getName()); + } + + // Check for required OSGI headers + Attributes attributes = mf.getMainAttributes(); + HashSet<String> mfAttributes = new HashSet<>(); + for (Object attributeSet : attributes.entrySet()) { + Map.Entry<Object, Object> e = (Map.Entry<Object, Object>) attributeSet; + mfAttributes.add(e.getKey().toString()); + } + List<String> requiredOSGIHeaders = Arrays.asList( + "Bundle-ManifestVersion", "Bundle-Name", "Bundle-SymbolicName", "Bundle-Version"); + for (String header : requiredOSGIHeaders) { + if (!mfAttributes.contains(header)) { + throw new IllegalArgumentException("Required OSGI header '" + header + + "' was not found in manifest in '" + jarFile.getName() + "'"); + } + } + + if (attributes.getValue("Bundle-Version").endsWith(".SNAPSHOT")) { + deployLogger.log(Level.WARNING, "Deploying snapshot bundle " + jarFile.getName() + + ".\nTo use this bundle, you must include the qualifier 'SNAPSHOT' in the version specification in services.xml."); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidator.java new file mode 100644 index 00000000000..9b62d66bc07 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/DeploymentFileValidator.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.ContainerModel; + +import java.io.Reader; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Validates that deployment file (deployment.xml) has valid values (for now + * only global-service-id is validated) + * + * @author musum + */ +public class DeploymentFileValidator extends Validator { + + @Override + public void validate(VespaModel model, DeployState deployState) { + Optional<Reader> deployment = deployState.getApplicationPackage().getDeployment(); + + if (deployment.isPresent()) { + Reader deploymentReader = deployment.get(); + DeploymentSpec deploymentSpec = DeploymentSpec.fromXml(deploymentReader); + final Optional<String> globalServiceId = deploymentSpec.globalServiceId(); + if (globalServiceId.isPresent()) { + Set<ContainerCluster> containerClusters = model.getRoot().configModelRepo().getModels(ContainerModel.class).stream(). + map(ContainerModel::getCluster).filter(cc -> cc.getName().equals(globalServiceId.get())).collect(Collectors.toSet()); + if (containerClusters.size() != 1) { + throw new IllegalArgumentException("global-service-id '" + globalServiceId.get() + "' specified in deployment.xml does not match any container cluster id"); + } + } + } + } +}
\ No newline at end of file diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java new file mode 100644 index 00000000000..d9aa600a840 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.Index; +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.search.AbstractSearchCluster; +import com.yahoo.vespa.model.search.DocumentDatabase; +import com.yahoo.vespa.model.search.IndexedSearchCluster; + +import java.util.Map; + +/** + * match:prefix for indexed fields not supported + * @author vegardh + * + */ +public class NoPrefixForIndexes extends Validator { + + @Override + public void validate(VespaModel model, DeployState deployState) { + for (AbstractSearchCluster cluster : model.getSearchClusters()) { + if (cluster instanceof IndexedSearchCluster) { + IndexedSearchCluster sc = (IndexedSearchCluster) cluster; + for (DocumentDatabase docDb : sc.getDocumentDbs()) { + DerivedConfiguration sdConfig = docDb.getDerivedConfiguration(); + Search search = sdConfig.getSearch(); + for (SDField field : search.allFieldsList()) { + if (field.doesIndexing()) { + //if (!field.getIndexTo().isEmpty() && !field.getIndexTo().contains(field.getName())) continue; + if (field.getMatching().getAlgorithm().equals(Matching.Algorithm.PREFIX)) { + failField(search, field); + } + for (Map.Entry<String, Index> e : field.getIndices().entrySet()) { + if (e.getValue().isPrefix()) { + failField(search, field); + } + } + } + } + } + } + } + } + + private void failField(Search search, SDField field) { + throw new IllegalArgumentException("For search '" + search.getName() + "', field '" + field.getName() + + "': match/index:prefix is not supported for indexes."); + } +}
\ No newline at end of file diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java new file mode 100644 index 00000000000..89a9d245de7 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -0,0 +1,159 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.io.IOUtils; +import com.yahoo.log.InvalidLogFormatException; +import com.yahoo.log.LogMessage; +import com.yahoo.yolean.Exceptions; +import com.yahoo.system.ProcessExecuter; +import com.yahoo.text.StringUtilities; +import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.collections.Pair; +import com.yahoo.config.ConfigInstance; +import com.yahoo.vespa.config.search.IndexschemaConfig; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.search.AbstractSearchCluster; +import com.yahoo.vespa.model.search.DocumentDatabase; +import com.yahoo.vespa.model.search.IndexedSearchCluster; +import com.yahoo.vespa.model.search.SearchCluster; + +import java.io.File; +import java.io.IOException; +import java.util.logging.Level; + +/** + * Validate rank setup for all search clusters (rank-profiles, index-schema, attributes configs), validating done + * by running through the binary 'verify_ranksetup' + * + * @author vegardh + * + */ +public class RankSetupValidator extends Validator { + private final boolean force; + + public RankSetupValidator(boolean force) { + this.force = force; + } + + @Override + public void validate(VespaModel model, DeployState deployState) { + File cfgDir = makeTempConfigDir(deployState.getDeployLogger()); + if (cfgDir == null) return; + + for (AbstractSearchCluster cluster : model.getSearchClusters()) { + // Skipping rank expression checking for streaming clusters, not implemented yet + if (cluster.isRealtime()) { + IndexedSearchCluster sc = (IndexedSearchCluster) cluster; + String clusterDir = cfgDir.getAbsolutePath() + "/" + sc.getClusterName() + "/"; + for (DocumentDatabase docDb : sc.getDocumentDbs()) { + String searchDir = clusterDir + docDb.getDerivedConfiguration().getSearch().getName() + "/"; + writeConfigs(searchDir, docDb); + if (!validate("dir:" + searchDir, sc, docDb.getDerivedConfiguration().getSearch().getName(), deployState.getDeployLogger(), cfgDir)) { + return; + } + } + } + } + deleteTempDir(cfgDir); + } + + private boolean validate(String configId, SearchCluster sc, String sdName, DeployLogger logger, File tempDir) { + try { + boolean ret = execValidate(configId, sc, sdName, logger); + if (!ret) { + // Give up, don't say same error msg repeatedly + deleteTempDir(tempDir); + } + return ret; + } catch (IllegalArgumentException e) { + deleteTempDir(tempDir); + throw e; + } + } + + private void deleteTempDir(File dir) { + if (!IOUtils.recursiveDeleteDir(dir)) { + throw new RuntimeException("Failed deleting " + dir); + } + } + + private void writeConfigs(String dir, AbstractConfigProducer producer) { + try { + RankProfilesConfig.Builder rpb = new RankProfilesConfig.Builder(); + RankProfilesConfig.Producer rpProd = (RankProfilesConfig.Producer) producer; + rpProd.getConfig(rpb); + writeConfig(dir, "rank-profiles.cfg", new RankProfilesConfig(rpb)); + + IndexschemaConfig.Builder isB = new IndexschemaConfig.Builder(); + IndexschemaConfig.Producer isProd = (IndexschemaConfig.Producer) producer; + isProd.getConfig(isB); + writeConfig(dir, "indexschema.cfg", new IndexschemaConfig(isB)); + + AttributesConfig.Builder acb = new AttributesConfig.Builder(); + AttributesConfig.Producer acProd = (AttributesConfig.Producer) producer; + acProd.getConfig(acb); + writeConfig(dir, "attributes.cfg", new AttributesConfig(acb)); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } + + private static void writeConfig(String dir, String configName, ConfigInstance config) throws IOException { + IOUtils.writeFile(dir + configName, StringUtilities.implodeMultiline(ConfigInstance.serialize(config)), false); + } + + private boolean execValidate(String configId, SearchCluster sc, String sdName, DeployLogger logger) { + String job = "verify_ranksetup-bin " + configId; + ProcessExecuter executer = new ProcessExecuter(); + try { + Pair<Integer, String> ret = executer.exec(job); + if (ret.getFirst() != 0) { + validateFail(ret.getSecond(), sc, sdName, logger); + } + } catch (IOException e) { + validateWarn(executer, e, logger); + return false; + } + return true; + } + + @SuppressWarnings({"UnusedDeclaration"}) + private void validateWarn(ProcessExecuter executer, Exception e, DeployLogger logger) { + String msg = "Unable to execute 'verify_ranksetup', validation of rank expressions will only take place when you start Vespa: " + + Exceptions.toMessageString(e); + logger.log(Level.WARNING, msg); + } + + private void validateFail(String output, SearchCluster sc, String sdName, DeployLogger logger) { + String errMsg = "For search cluster '" + sc.getClusterName() + "', search definition '" + sdName + "': error in rank setup. Details:\n"; + for (String line : output.split("\n")) { + // Remove debug lines from start script + if (line.startsWith("debug\t")) continue; + try { + LogMessage logmsg = LogMessage.parseNativeFormat(line); + errMsg = errMsg + logmsg.getLevel() + ": " + logmsg.getPayload() + "\n"; + } catch (InvalidLogFormatException e) { + errMsg = errMsg + line + "\n"; + } + } + if (force) { + logger.log(Level.WARNING, errMsg + "(Continuing because of force.)"); + } else { + throw new IllegalArgumentException(errMsg); + } + } + + private File makeTempConfigDir(DeployLogger deployLogger) { + String name = "/tmp/deploy_ranksetup_" + System.currentTimeMillis() + "/"; + File tempDir = new File(name); + if (!tempDir.mkdir()) { + deployLogger.log(Level.WARNING, "Not able to create '" + name + "' when validating rank setup"); + return null; + } + return tempDir; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RestartConfigs.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RestartConfigs.java new file mode 100644 index 00000000000..1bf15d6dd4f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RestartConfigs.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.vespa.model.application.validation.change.ConfigValueChangeValidator; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Services should use this annotation to list all consumed configurations which contain definitions flagged with restart. + * This annotation is required for the {@link ConfigValueChangeValidator} + * to detect config changes that will require restart of some services. The {@link com.yahoo.config.ConfigInstance} + * values are inherited; any configs annotated on a service will be inherited to all sub-classes of that service. + * These sub-classes can supplement with more ConfigInstances (in addition to the inherited one) with the annotation. + * This is different the inheritance that {@link java.lang.annotation.Inherited} provides, where sub-classes can either + * inherit or override the annotation from the super class. + * + * NOTE: This annotation is will only have effect on subclasses of {@link com.yahoo.vespa.model.Service}. + * Do not use this annotation on other types config producers. + * + * @author bjorncs + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface RestartConfigs { + Class<? extends ConfigInstance>[] value() default {}; +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RoutingSelectorValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RoutingSelectorValidator.java new file mode 100644 index 00000000000..ad79b1eed45 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RoutingSelectorValidator.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.document.select.DocumentSelector; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.search.AbstractSearchCluster; +import com.yahoo.vespa.model.search.IndexedSearchCluster; + + +/** + * Validates routing selector for search and content clusters + * + */ +public class RoutingSelectorValidator extends Validator { + + @Override + public void validate(VespaModel model, DeployState deployState) { + for (AbstractSearchCluster cluster : model.getSearchClusters()) { + if (cluster instanceof IndexedSearchCluster) { + IndexedSearchCluster sc = (IndexedSearchCluster) cluster; + String routingSelector = sc.getRoutingSelector(); + if (routingSelector == null) continue; + try { + new DocumentSelector(routingSelector); + } catch (com.yahoo.document.select.parser.ParseException e) { + throw new IllegalArgumentException("Failed to parse routing selector for search cluster '" + + sc.getClusterName() + "'", e); + } + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RoutingValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RoutingValidator.java new file mode 100644 index 00000000000..6e94c39cc51 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RoutingValidator.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.vespa.model.VespaModel; + +import java.util.List; + +/** + * Validates routing + * + */ +public class RoutingValidator extends Validator { + + @Override + public void validate(VespaModel model, DeployState deployState) { + List<String> errors = model.getRouting().getErrors(); + if (!errors.isEmpty()) { + StringBuilder msg = new StringBuilder(); + msg.append("The routing specification contains ").append(errors.size()).append(" error(s):\n"); + for (int i = 0, len = errors.size(); i < len; ++i) { + msg.append(i + 1).append(". ").append(errors.get(i)).append("\n"); + } + throw new IllegalArgumentException(msg.toString()); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java new file mode 100644 index 00000000000..c03fb0617b8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java @@ -0,0 +1,86 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.document.*; +import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.search.AbstractSearchCluster; +import com.yahoo.vespa.model.search.SearchDefinition; + +import java.util.List; + +/** + * This Validator iterates through all search cluster in the given VespaModel to make sure that there are no custom + * structs defined in any of its search definitions. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SearchDataTypeValidator extends Validator { + + @Override + public void validate(VespaModel model, DeployState deployState) { + List<AbstractSearchCluster> clusters = model.getSearchClusters(); + for (AbstractSearchCluster cluster : clusters) { + if (cluster.isStreaming()) { + continue; + } + for (AbstractSearchCluster.SearchDefinitionSpec spec : cluster.getLocalSDS()) { + SDDocumentType docType = spec.getSearchDefinition().getSearch().getDocument(); + if (docType == null) { + continue; + } + validateDocument(cluster, spec.getSearchDefinition(), docType); + } + } + } + + private void validateDocument(AbstractSearchCluster cluster, SearchDefinition def, SDDocumentType doc) { + for (SDDocumentType child : doc.getTypes()) { + validateDocument(cluster, def, child); + } + for (Field field : doc.fieldSet()) { + DataType fieldType = field.getDataType(); + disallowIndexingOfMaps(cluster, def, field); + if (!validateDataType(fieldType)) { + throw new IllegalArgumentException("Field type '" + fieldType.getName() + "' is illegal for search " + + "clusters (field '" + field.getName() + "' in definition '" + + def.getName() + "' for cluster '" + cluster.getClusterName() + "')."); + } + } + } + + private boolean validateDataType(DataType dataType) { + if (dataType instanceof ArrayDataType || + dataType instanceof WeightedSetDataType) + { + return validateDataType(((CollectionDataType)dataType).getNestedType()); + } + if (dataType instanceof StructDataType) { + return true; // Struct will work for summary TODO maybe check individual fields + } + if (dataType instanceof MapDataType) { + return true; // Maps will work for summary, see disallowIndexingOfMaps() + } + return dataType.equals(DataType.INT) || + dataType.equals(DataType.FLOAT) || + dataType.equals(DataType.STRING) || + dataType.equals(DataType.RAW) || + dataType.equals(DataType.LONG) || + dataType.equals(DataType.DOUBLE) || + dataType.equals(DataType.URI) || + dataType.equals(DataType.BYTE) || + dataType.equals(DataType.PREDICATE) || + dataType.equals(DataType.TENSOR); + } + + private void disallowIndexingOfMaps(AbstractSearchCluster cluster, SearchDefinition def, Field field) { + DataType fieldType = field.getDataType(); + if ((fieldType instanceof MapDataType) && (((SDField) field).doesIndexing())) { + throw new IllegalArgumentException("Field type '" + fieldType.getName() + "' cannot be indexed for search " + + "clusters (field '" + field.getName() + "' in definition '" + + def.getName() + "' for cluster '" + cluster.getClusterName() + "')."); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java new file mode 100644 index 00000000000..9d6d10f2861 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import java.util.List; +import java.util.logging.Level; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.NumericDataType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.search.AbstractSearchCluster; +import com.yahoo.vespa.model.search.SearchCluster; + + +/** + * Validates streaming mode + */ +public class StreamingValidator extends Validator { + + @Override + public void validate(VespaModel model, DeployState deployState) { + List<AbstractSearchCluster> searchClusters = model.getSearchClusters(); + for (AbstractSearchCluster cluster : searchClusters) { + if (!cluster.isStreaming()) { + continue; + } + SearchCluster sc = (SearchCluster) cluster; + warnStreamingAttributes(sc, deployState.getDeployLogger()); + warnStreamingGramMatching(sc, deployState.getDeployLogger()); + } + } + + private void warnStreamingGramMatching(SearchCluster sc, DeployLogger logger) { + if (sc.getSdConfig() != null) { + for (SDField sd : sc.getSdConfig().getSearch().allFieldsList()) { + if (sd.getMatching().getType().equals(Matching.Type.GRAM)) { + logger.log(Level.WARNING, "For streaming search cluster '" + sc.getClusterName() + + "', SD field '" + sd.getName() + "': n-gram matching is not supported for streaming search."); + } + } + } + } + + /** + * Warn if one or more attributes are defined in a streaming search cluster SD. + * + * @param sc a search cluster to be checked for attributes in streaming search + * @param logger a DeployLogger + */ + private void warnStreamingAttributes(SearchCluster sc, DeployLogger logger) { + if (sc.getSdConfig() != null) { + for (SDField sd : sc.getSdConfig().getSearch().allFieldsList()) { + if (sd.doesAttributing()) { + warnStreamingAttribute(sc, sd, logger); + } + } + } + } + + private void warnStreamingAttribute(SearchCluster sc, SDField sd, DeployLogger logger) { + // If the field is numeric, we can't print this, because we may have converted the field to + // attribute indexing ourselves (IntegerIndex2Attribute) + if (sd.getDataType() instanceof NumericDataType) return; + logger.log(Level.WARNING, "For streaming search cluster '" + sc.getClusterName() + + "', SD field '" + sd.getName() + "': 'attribute' has same match semantics as 'index'."); + } +}
\ No newline at end of file diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java new file mode 100644 index 00000000000..0c74fddcc51 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.change.ChangeValidator; +import com.yahoo.vespa.model.application.validation.change.ClusterSizeReductionValidator; +import com.yahoo.vespa.model.application.validation.change.ConfigValueChangeValidator; +import com.yahoo.vespa.model.application.validation.change.ContainerRestartValidator; +import com.yahoo.vespa.model.application.validation.change.ContentClusterRemovalValidator; +import com.yahoo.vespa.model.application.validation.change.IndexedSearchClusterChangeValidator; +import com.yahoo.vespa.model.application.validation.change.IndexingModeChangeValidator; +import com.yahoo.vespa.model.application.validation.change.StartupCommandChangeValidator; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import static java.util.stream.Collectors.toList; + +/** + * Executor of validators. This defines the right order of validator execution. + * Validators that must be run after search cluster search definition deriving are + * defined in PostSdValidation. + * + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + * @since 2010-01-29 + */ +public class Validation { + + /** Validate everything */ + public static List<ConfigChangeAction> validate(VespaModel model, boolean force, DeployState deployState) { + return validate(model, true, force, deployState); + } + + /** + * Validate with optional checking of routing, which cannot always be valid in unit tests + * + * @return a list of required changes needed to make this configuration live + */ + public static List<ConfigChangeAction> validate(VespaModel model, boolean checkRouting, boolean force, DeployState deployState) { + if (checkRouting) { + new RoutingValidator().validate(model, deployState); + new RoutingSelectorValidator().validate(model, deployState); + } + new ComponentValidator().validate(model, deployState); + new SearchDataTypeValidator().validate(model, deployState); + new StreamingValidator().validate(model, deployState); + new RankSetupValidator(force).validate(model, deployState); + new NoPrefixForIndexes().validate(model, deployState); + new DeploymentFileValidator().validate(model, deployState); + + Optional<Model> currentActiveModel = deployState.getPreviousModel(); + if (currentActiveModel.isPresent() && (currentActiveModel.get() instanceof VespaModel)) + return validateChanges((VespaModel)currentActiveModel.get(), model, + deployState.validationOverrides(), deployState.getDeployLogger()); + else + return new ArrayList<>(); + } + + private static List<ConfigChangeAction> validateChanges(VespaModel currentModel, VespaModel nextModel, + ValidationOverrides overrides, DeployLogger logger) { + ChangeValidator[] validators = new ChangeValidator[] { + new IndexingModeChangeValidator(), + new IndexedSearchClusterChangeValidator(), + new ConfigValueChangeValidator(logger), + new StartupCommandChangeValidator(), + new ContentClusterRemovalValidator(), + new ClusterSizeReductionValidator(), + new ContainerRestartValidator(), + }; + return Arrays.stream(validators) + .flatMap(v -> v.validate(currentModel, nextModel, overrides).stream()) + .collect(toList()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationId.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationId.java new file mode 100644 index 00000000000..cb60e6fc77f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationId.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import java.util.Optional; + +/** + * Ids of validations that can be overridden + * + * @author bratseth + */ +public enum ValidationId { + + indexingChange("indexing-change"), + indexModeChange("indexing-mode-change"), + fieldTypeChange("field-type-change"), + clusterSizeReduction("cluster-size-reduction"), + contentClusterRemoval("content-cluster-removal"), + configModelVersionMismatch("config-model-version-mismatch"), + skipOldConfigModels("skip-old-config-models"); + + private final String id; + + ValidationId(String id) { this.id = id; } + + public String value() { return id; } + + @Override + public String toString() { return id; } + + /** + * Returns the validation id from this string. + * Use this instead of valueOf to match string on the (canonical) dash-separated form. + * + * @return the matching validation id or empty if none + */ + public static Optional<ValidationId> from(String id) { + for (ValidationId candidate : ValidationId.values()) + if (id.equals(candidate.toString())) return Optional.of(candidate); + return Optional.empty(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationOverrides.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationOverrides.java new file mode 100644 index 00000000000..6a12d206189 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ValidationOverrides.java @@ -0,0 +1,104 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.google.common.collect.ImmutableList; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Optional; + +/** + * A set of allows which suppresses specific validations in limited time periods. + * This is useful to be able to complete a deployment in cases where the application + * owner believes that the changes to be deployed have acceptable consequences. + * Immutable. + * + * @author bratseth + */ +public class ValidationOverrides { + + private final List<Allow> overrides; + + /** Instant to use as "now". This is a field to allow unit testing. */ + private final Instant now; + + /** Creates validation overrides for the current instant */ + public ValidationOverrides(List<Allow> overrides) { + this(overrides, Instant.now()); + } + + public ValidationOverrides(List<Allow> overrides, Instant now) { + this.overrides = ImmutableList.copyOf(overrides); + this.now = now; + for (Allow override : overrides) + if (now.plus(Duration.ofDays(30)).isBefore(override.until)) + throw new IllegalArgumentException(override + " is too far in the future: Max 30 days is allowed"); + } + + /** Throws a ValidationException unless this validation is overridden at this time */ + public void invalid(ValidationId validationId, String message) { + if ( ! allows(validationId)) + throw new ValidationException(validationId, message); + } + + public boolean allows(String validationIdString) { + Optional<ValidationId> validationId = ValidationId.from(validationIdString); + if ( ! validationId.isPresent()) return false; // unknown id -> not allowed + return allows(validationId.get()); + } + + /** Returns whether the given (assumed invalid) change is allowed by this at the moment */ + public boolean allows(ValidationId validationId) { + for (Allow override : overrides) + if (override.allows(validationId, now)) + return true; + return false; + } + + public static ValidationOverrides empty() { return new ValidationOverrides(ImmutableList.of()); } + + /** A validation override which allows a particular change. Immutable. */ + public static class Allow { + + private final ValidationId validationId; + private final Instant until; + + public Allow(ValidationId validationId, Instant until) { + this.validationId = validationId; + this.until = until; + } + + public boolean allows(ValidationId validationId, Instant now) { + return this.validationId.equals(validationId) && now.isBefore(until); + } + + @Override + public String toString() { return "allow '" + validationId + "' until " + until; } + + } + + /** + * A deployment validation exception. + * Deployment validations can be {@link ValidationOverrides overridden} based on their id. + * The purpose of this exception is to model that id as a separate field. + */ + public static class ValidationException extends IllegalArgumentException { + + private final ValidationId validationId; + + private ValidationException(ValidationId validationId, String message) { + super(message); + this.validationId = validationId; + } + + /** Returns the unique id of this validation, which can be used to {@link ValidationOverrides override} it */ + public ValidationId validationId() { return validationId; } + + /** Returns "validationId: message" */ + @Override + public String getMessage() { return validationId + ": " + super.getMessage(); } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validator.java new file mode 100644 index 00000000000..d92e15fea66 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validator.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.vespa.model.VespaModel; + +/** + * Abstract superclass of all application package validators. + * + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + * @since 2010-01-29 + */ +public abstract class Validator { + + /** + * Validates the input vespamodel + * + * @param model a VespaModel object + * @param deployState The {@link DeployState} built from building the model + */ + public abstract void validate(VespaModel model, DeployState deployState); + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ChangeValidator.java new file mode 100644 index 00000000000..f60b1871e59 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ChangeValidator.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.ValidationOverrides; + +import java.util.List; + +/** + * Interface for validating changes between a current active and next config model. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + * @since 2014-11-18 + */ +public interface ChangeValidator { + + /** + * Validates the current active vespa model with the next model. + * Both current and next should be non-null. + * + * @param current the current active model + * @param next the next model we would like to activate + * @param overrides validation overrides + * @return a list of actions specifying what needs to be done in order to activate the new model. + * Return an empty list if nothing needs to be done + */ + List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides); + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java new file mode 100644 index 00000000000..279f74db46d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ClusterSizeReductionValidator.java @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.ValidationId; +import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.content.cluster.ContentCluster; + +import java.util.Collections; +import java.util.List; + +/** + * Checks that no cluster sizes are reduced too much in one go. + * + * @author bratseth + */ +public class ClusterSizeReductionValidator implements ChangeValidator { + + @Override + public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides) { + for (ContainerCluster currentCluster : current.getContainerClusters().values()) { + ContainerCluster nextCluster = next.getContainerClusters().get(currentCluster.getName()); + if (nextCluster == null) continue; + validate(currentCluster.getContainers().size(), + nextCluster.getContainers().size(), + currentCluster.getName(), + overrides); + } + + for (ContentCluster currentCluster : current.getContentClusters().values()) { + ContentCluster nextCluster = next.getContentClusters().get(currentCluster.getName()); + if (nextCluster == null) continue; + validate(currentCluster.getSearch().getSearchNodes().size(), + nextCluster.getSearch().getSearchNodes().size(), + currentCluster.getName(), + overrides); + } + + return Collections.emptyList(); + } + + private void validate(int currentSize, int nextSize, String clusterName, ValidationOverrides overrides) { + // don't allow more than 50% reduction, but always allow to reduce size with 1 + if ( nextSize < ((double)currentSize) * 0.5 && nextSize != currentSize - 1) + overrides.invalid(ValidationId.clusterSizeReduction, + "Size reduction in '" + clusterName + "' is too large. Current size: " + currentSize + + ", new size: " + nextSize + ". New size must be at least 50% of the current size"); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java new file mode 100644 index 00000000000..06d7c358678 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java @@ -0,0 +1,162 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.ChangesRequiringRestart; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.producer.AbstractConfigProducerRoot; +import com.yahoo.vespa.model.Service; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.RestartConfigs; +import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.vespa.model.application.validation.change.ChangeValidator; +import com.yahoo.vespa.model.application.validation.change.VespaRestartAction; +import com.yahoo.vespa.model.utils.internal.ReflectionUtil; +import org.apache.commons.lang3.ClassUtils; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.logging.Level; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.joining; + +/** + * Compares the config instances in the current and next Vespa model to determine if some services will require restart. + * The configs used by a given service is deduced from the + * {@link com.yahoo.vespa.model.application.validation.RestartConfigs} annotation. + * + * @author bjorncs + */ +public class ConfigValueChangeValidator implements ChangeValidator { + + private final DeployLogger logger; + + public ConfigValueChangeValidator(DeployLogger logger) { + this.logger = logger; + } + + /** Inspects the configuration in the new and old Vespa model to determine which services that require restart */ + @Override + public List<ConfigChangeAction> validate(VespaModel currentModel, VespaModel nextModel, + ValidationOverrides overrides) { + return findConfigChangesFromModels(currentModel, nextModel).collect(Collectors.toList()); + } + + public Stream<ConfigChangeAction> findConfigChangesFromModels( + AbstractConfigProducerRoot currentModel, + AbstractConfigProducerRoot nextModel) { + return nextModel.getDescendantServices().stream() + .map(service -> findConfigChangeActionForService(service, currentModel, nextModel)) + .filter(Optional::isPresent) + .map(Optional::get); + } + + private Optional<ConfigChangeAction> findConfigChangeActionForService( + Service service, + AbstractConfigProducerRoot currentModel, + AbstractConfigProducerRoot nextModel) { + List<ChangesRequiringRestart> changes = findConfigChangesForService(service, currentModel, nextModel) + .collect(Collectors.toList()); + if (changes.isEmpty()) { + return Optional.empty(); + } + String description = createDescriptionOfConfigChanges(changes.stream()); + return Optional.of(new VespaRestartAction(description, service.getServiceInfo())); + } + + private Stream<ChangesRequiringRestart> findConfigChangesForService( + Service service, + AbstractConfigProducerRoot currentModel, + AbstractConfigProducerRoot nextModel) { + Class<? extends Service> serviceClass = service.getClass(); + if (!currentModel.getService(service.getConfigId()).isPresent()) { + // Service does not exist in the current model. + return Stream.empty(); + } + return getConfigInstancesFromServiceAnnotations(serviceClass) + .map(configClass -> compareConfigFromCurrentAndNextModel(service, configClass, currentModel, nextModel)) + .filter(Optional::isPresent) + .map(Optional::get) + .filter(ChangesRequiringRestart::needsRestart); + } + + private static String createDescriptionOfConfigChanges(Stream<ChangesRequiringRestart> changesStream) { + return changesStream + .map(changes -> changes.toString("")) + .collect(joining("\n")); + } + + /** + * Returns the ConfigInstances classes from the annotation on the given Service class , + * including the ConfigInstances annotated on any of its super classes. + * NOTE: Only the super classes that are subclass of Service are inspected. + */ + private static Stream<Class<? extends ConfigInstance>> getConfigInstancesFromServiceAnnotations(Class<? extends Service> serviceClass) { + List<Class<?>> classHierarchy = ClassUtils.getAllSuperclasses(serviceClass); + classHierarchy.add(serviceClass); + return classHierarchy.stream() + .filter(Service.class::isAssignableFrom) + .filter(clazz -> clazz.isAnnotationPresent(RestartConfigs.class)) + .map(clazz -> { + RestartConfigs annotation = clazz.getDeclaredAnnotation(RestartConfigs.class); + if (annotation.value().length == 0) { + throw new IllegalStateException(String.format( + "%s has a %s annotation with no ConfigInstances given as argument.", + clazz.getSimpleName(), RestartConfigs.class.getSimpleName())); + } + return annotation; + }) + .map(RestartConfigs::value) + .flatMap(Arrays::stream) + .distinct(); + } + + private Optional<ChangesRequiringRestart> compareConfigFromCurrentAndNextModel( + Service service, Class<? extends ConfigInstance> configClass, + AbstractConfigProducerRoot currentModel, AbstractConfigProducerRoot nextModel) { + + if (!hasConfigFieldsFlaggedWithRestart(configClass, service.getClass())) { + logger.log(Level.FINE, String.format("%s is listed in the annotation for %s, " + + "but does not have any restart flags in its config definition.", + configClass.getSimpleName(), service.getClass().getSimpleName())); + return Optional.empty(); + } + + Optional<ConfigInstance> nextConfig = getConfigFromModel(nextModel, configClass, service.getConfigId()); + if (!nextConfig.isPresent()) { + logger.log(Level.FINE, String.format( + "%s is listed as restart config for %s, but the config does not exist in the new model.", + configClass.getSimpleName(), service.getClass().getSimpleName())); + return Optional.empty(); + } + + Optional<ConfigInstance> currentConfig = getConfigFromModel(currentModel, configClass, service.getConfigId()); + if (!currentConfig.isPresent()) { + return Optional.empty(); + } + return Optional.of(ReflectionUtil.getChangesRequiringRestart(currentConfig.get(), nextConfig.get())); + } + + private static boolean hasConfigFieldsFlaggedWithRestart( + Class<? extends ConfigInstance> configClass, Class<? extends Service> serviceClass) { + if (!ReflectionUtil.hasRestartMethods(configClass)) { + throw new IllegalStateException(String.format( + "%s is listed as restart config for %s but does not contain any restart inspection methods.", + configClass.getSimpleName(), serviceClass.getSimpleName())); + } + return ReflectionUtil.containsFieldsFlaggedWithRestart(configClass); + } + + private static Optional<ConfigInstance> getConfigFromModel( + AbstractConfigProducerRoot configModel, Class<? extends ConfigInstance> configClass, String configKey) { + try { + return Optional.ofNullable(configModel.getConfig(configClass, configKey)); + } catch (Exception e) { + return Optional.empty(); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java new file mode 100644 index 00000000000..3d56fe200d8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContainerRestartValidator.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.container.QrConfig; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.vespa.model.container.Container; + +import java.util.List; + +import static java.util.stream.Collectors.toList; + +/** + * Returns a restart action for each container that has turned on {@link QrConfig#restartOnDeploy}. + * + * @author bjorncs + */ +public class ContainerRestartValidator implements ChangeValidator { + + @Override + public List<ConfigChangeAction> validate(VespaModel currentModel, VespaModel nextModel, ValidationOverrides ignored) { + return nextModel.getContainerClusters().values().stream() + .flatMap(cluster -> cluster.getContainers().stream()) + .filter(container -> isExistingContainer(container, currentModel)) + .filter(container -> shouldContainerRestartOnDeploy(container, nextModel)) + .map(ContainerRestartValidator::createConfigChangeAction) + .collect(toList()); + } + + private static ConfigChangeAction createConfigChangeAction(Container container) { + return new VespaRestartAction(createMessage(container), container.getServiceInfo()); + } + + private static String createMessage(Container container) { + return String.format("Container '%s' is configured to always restart on deploy.", container.getConfigId()); + } + + private static boolean shouldContainerRestartOnDeploy(Container container, VespaModel nextModel) { + QrConfig config = nextModel.getConfig(QrConfig.class, container.getConfigId()); + return config.restartOnDeploy(); + } + + private static boolean isExistingContainer(Container container, VespaModel currentModel) { + return currentModel.getService(container.getConfigId()).isPresent(); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java new file mode 100644 index 00000000000..57e9d124c77 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.ValidationId; +import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.vespa.model.content.cluster.ContentCluster; + +import java.util.Collections; +import java.util.List; + +/** + * Checks that this does not remove a content cluster (or changes its id) + * as that means losing all data of that cluster. + * + * @author bratseth + */ +public class ContentClusterRemovalValidator implements ChangeValidator { + + @Override + public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides) { + for (String currentClusterId : current.getContentClusters().keySet()) { + ContentCluster nextCluster = next.getContentClusters().get(currentClusterId); + if (nextCluster == null) + overrides.invalid(ValidationId.contentClusterRemoval, + "Content cluster '" + currentClusterId + "' is removed. " + + "This will cause loss of all data in this cluster"); + } + + return Collections.emptyList(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java new file mode 100644 index 00000000000..ab6bc5a1cd7 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexedSearchClusterChangeValidator.java @@ -0,0 +1,98 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.vespa.model.application.validation.change.search.DocumentDatabaseChangeValidator; +import com.yahoo.vespa.model.content.ContentSearchCluster; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.search.DocumentDatabase; +import com.yahoo.vespa.model.search.IndexedSearchCluster; + +import java.util.*; +import java.util.stream.Collectors; + +/** + * Validates the changes between all current and next indexed search clusters in a vespa model. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + * @since 2014-11-18 + */ +public class IndexedSearchClusterChangeValidator implements ChangeValidator { + + @Override + public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, ValidationOverrides overrides) { + List<ConfigChangeAction> result = new ArrayList<>(); + for (Map.Entry<String, ContentCluster> currentEntry : current.getContentClusters().entrySet()) { + ContentCluster nextCluster = next.getContentClusters().get(currentEntry.getKey()); + if (nextCluster != null && nextCluster.getSearch().hasIndexedCluster()) { + result.addAll(validateContentCluster(currentEntry.getValue(), nextCluster, overrides)); + } + } + return result; + } + + private static List<ConfigChangeAction> validateContentCluster(ContentCluster currentCluster, + ContentCluster nextCluster, + ValidationOverrides overrides) { + List<ConfigChangeAction> result = new ArrayList<>(); + result.addAll(validateDocumentDatabases(currentCluster, nextCluster, overrides)); + return result; + } + + private static List<ConfigChangeAction> validateDocumentDatabases(ContentCluster currentCluster, + ContentCluster nextCluster, + ValidationOverrides overrides) { + List<ConfigChangeAction> result = new ArrayList<>(); + for (DocumentDatabase currentDb : getDocumentDbs(currentCluster.getSearch())) { + String docTypeName = currentDb.getName(); + Optional<DocumentDatabase> nextDb = nextCluster.getSearch().getIndexed().getDocumentDbs().stream(). + filter(db -> db.getName().equals(docTypeName)).findFirst(); + if (nextDb.isPresent()) { + result.addAll(validateDocumentDatabase(currentCluster, nextCluster, docTypeName, + currentDb, nextDb.get(), overrides)); + } + } + return result; + } + + private static List<ConfigChangeAction> validateDocumentDatabase(ContentCluster currentCluster, + ContentCluster nextCluster, + String docTypeName, + DocumentDatabase currentDb, + DocumentDatabase nextDb, + ValidationOverrides overrides) { + NewDocumentType currentDocType = currentCluster.getDocumentDefinitions().get(docTypeName); + NewDocumentType nextDocType = nextCluster.getDocumentDefinitions().get(docTypeName); + List<VespaConfigChangeAction> result = + new DocumentDatabaseChangeValidator(currentDb, currentDocType, nextDb, nextDocType).validate(overrides); + + return modifyActions(result, getSearchNodeServices(nextCluster.getSearch().getIndexed()), docTypeName); + } + + private static List<DocumentDatabase> getDocumentDbs(ContentSearchCluster cluster) { + if (cluster.getIndexed() != null) { + return cluster.getIndexed().getDocumentDbs(); + } + return new ArrayList<>(); + } + + private static List<ServiceInfo> getSearchNodeServices(IndexedSearchCluster cluster) { + return cluster.getSearchNodes().stream(). + map(node -> node.getServiceInfo()). + collect(Collectors.toList()); + } + + private static List<ConfigChangeAction> modifyActions(List<VespaConfigChangeAction> result, + List<ServiceInfo> services, + String docTypeName) { + return result.stream(). + map(action -> action.modifyAction("Document type '" + docTypeName + "': " + action.getMessage(), + services, docTypeName)). + collect(Collectors.toList()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java new file mode 100644 index 00000000000..18683ce2411 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidator.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.vespa.model.application.validation.change.ChangeValidator; +import com.yahoo.vespa.model.application.validation.change.VespaRefeedAction; +import com.yahoo.vespa.model.content.cluster.ContentCluster; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Returns any change to the indexing mode of a cluster. + * + * @author musum + */ +public class IndexingModeChangeValidator implements ChangeValidator { + + @Override + public List<ConfigChangeAction> validate(VespaModel currentModel, VespaModel nextModel, ValidationOverrides overrides) { + List<ConfigChangeAction> actions = new ArrayList<>(); + for (Map.Entry<String, ContentCluster> currentEntry : currentModel.getContentClusters().entrySet()) { + ContentCluster nextCluster = nextModel.getContentClusters().get(currentEntry.getKey()); + if (nextCluster == null) continue; + + Optional<ConfigChangeAction> change = validateContentCluster(currentEntry.getValue(), nextCluster, overrides); + if (change.isPresent()) + actions.add(change.get()); + } + return actions; + } + + private Optional<ConfigChangeAction> validateContentCluster(ContentCluster currentCluster, ContentCluster nextCluster, + ValidationOverrides overrides) { + final boolean currentClusterIsIndexed = currentCluster.getSearch().hasIndexedCluster(); + final boolean nextClusterIsIndexed = nextCluster.getSearch().hasIndexedCluster(); + + if (currentClusterIsIndexed == nextClusterIsIndexed) return Optional.empty(); + + return Optional.of(VespaRefeedAction.of("indexing-mode-change", + overrides, + "Cluster '" + currentCluster.getName() + "' changed indexing mode from '" + + indexingMode(currentClusterIsIndexed) + "' to '" + indexingMode(nextClusterIsIndexed) + "'")); + } + + private String indexingMode(boolean isIndexed) { + return isIndexed ? "indexed" : "streaming"; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidator.java new file mode 100644 index 00000000000..fb90de60a2f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidator.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.producer.AbstractConfigProducerRoot; +import com.yahoo.vespa.model.Service; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.vespa.model.application.validation.change.ChangeValidator; +import com.yahoo.vespa.model.application.validation.change.VespaRestartAction; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Compares the startup command for the services in the next model with the ones in the old model. + * If the startup command has changes, a change entry is created and reported back. + * + * @author bjorncs + */ +public class StartupCommandChangeValidator implements ChangeValidator { + + @Override + public List<ConfigChangeAction> validate(VespaModel currentModel, VespaModel nextModel, + ValidationOverrides overrides) { + return findServicesWithChangedStartupCommmand(currentModel, nextModel).collect(Collectors.toList()); + } + + public Stream<ConfigChangeAction> findServicesWithChangedStartupCommmand( + AbstractConfigProducerRoot currentModel, AbstractConfigProducerRoot nextModel) { + return nextModel.getDescendantServices().stream() + .map(nextService -> currentModel.getService(nextService.getConfigId()) + .flatMap(currentService -> compareStartupCommand(currentService, nextService))) + .filter(Optional::isPresent) + .map(Optional::get); + } + + private Optional<ConfigChangeAction> compareStartupCommand(Service currentService, Service nextService) { + String currentCommand = currentService.getStartupCommand(); + String nextCommand = nextService.getStartupCommand(); + + // Objects.equals is null-aware + if (Objects.equals(currentCommand, nextCommand)) { + return Optional.empty(); + } + String message = String.format("Startup command for '%s' has changed.\nNew command: %s.\nOld command: %s.", + currentService.getServiceName(), nextCommand, currentCommand); + return Optional.of(new VespaRestartAction(message, currentService.getServiceInfo())); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaConfigChangeAction.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaConfigChangeAction.java new file mode 100644 index 00000000000..a8a334448ff --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaConfigChangeAction.java @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.api.ServiceInfo; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * Class containing the action to be performed on the given services to handle a config change + * between the current active vespa model and the next vespa model to prepare. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + * @since 2014-11-18 + */ +public abstract class VespaConfigChangeAction implements ConfigChangeAction { + + private final String message; + private final List<ServiceInfo> services; + + protected VespaConfigChangeAction(String message, List<ServiceInfo> services) { + this.message = message; + this.services = services; + } + + public abstract VespaConfigChangeAction modifyAction(String newMessage, List<ServiceInfo> newServices, String documentType); + + @Override + public String getMessage() { + return message; + } + + @Override + public List<ServiceInfo> getServices() { + return services; + } + + @Override + public String toString() { + return "type='" + getType() + "', message='" + message + "', services=[" + + services.stream(). + map(service -> service.getServiceName() + " '" + service.getConfigId() + "'"). + collect(Collectors.joining(", ")) + "]"; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof VespaConfigChangeAction)) { + return false; + } + VespaConfigChangeAction rhs = (VespaConfigChangeAction)o; + if (!getType().equals(rhs.getType())) return false; + if (!message.equals(rhs.message)) return false; + if (!services.equals(rhs.services)) return false; + return true; + } + + @Override + public int hashCode() { + int result = getType().hashCode(); + result = 31 * result + message.hashCode(); + result = 31 * result + services.hashCode(); + return result; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java new file mode 100644 index 00000000000..6b8d18fadea --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRefeedAction.java @@ -0,0 +1,84 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.model.api.ConfigChangeRefeedAction; +import com.yahoo.config.model.api.ServiceInfo; +import com.yahoo.vespa.model.application.validation.ValidationOverrides; + +import java.util.Collections; +import java.util.List; + +/** + * Represents an action to re-feed a document type in order to handle a config change. + * + * @author geirst + * @author bratseth + * @since 5.43 + */ +public class VespaRefeedAction extends VespaConfigChangeAction implements ConfigChangeRefeedAction { + + /** + * The name of this action, which must be a valid ValidationId. This is a string here because + * the validation ids belong to the Vespa model while these names are exposed to the config server, + * which is model version independent. + */ + private final String name; + + private final String documentType; + private final boolean allowed; + + private VespaRefeedAction(String name, String message, List<ServiceInfo> services, String documentType, boolean allowed) { + super(message, services); + this.name = name; + this.documentType = documentType; + this.allowed = allowed; + } + + /** Creates a refeed action with some missing information */ + // TODO: We should require document type or model its absence properly + public static VespaRefeedAction of(String name, ValidationOverrides overrides, String message) { + return new VespaRefeedAction(name, message, Collections.emptyList(), "", overrides.allows(name)); + } + + /** Creates a refeed action */ + public static VespaRefeedAction of(String name, ValidationOverrides overrides, String message, + List<ServiceInfo> services, String documentType) { + return new VespaRefeedAction(name, message, services, documentType, overrides.allows(name)); + } + + @Override + public VespaConfigChangeAction modifyAction(String newMessage, List<ServiceInfo> newServices, String documentType) { + return new VespaRefeedAction(name, newMessage, newServices, documentType, allowed); + } + + @Override + public String name() { return name; } + + @Override + public String getDocumentType() { return documentType; } + + @Override + public boolean allowed() { return allowed; } + + @Override + public String toString() { + return super.toString() + ", documentType='" + documentType + "'"; + } + + @Override + public boolean equals(Object o) { + if ( ! super.equals(o)) return false; + if ( ! (o instanceof VespaRefeedAction)) return false; + VespaRefeedAction other = (VespaRefeedAction)o; + if ( ! this.documentType.equals(other.documentType)) return false; + if ( ! this.name.equals(other.name)) return false; + if ( ! this.allowed == other.allowed) return false; + return true; + } + + @Override + public int hashCode() { + return 31 * super.hashCode() + 11 * name.hashCode() + documentType.hashCode(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java new file mode 100644 index 00000000000..1974847a83c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.model.api.ConfigChangeRestartAction; +import com.yahoo.config.model.api.ServiceInfo; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Represents an action to restart services in order to handle a config change. + * + * @author geirst + * @since 5.43 + */ +public class VespaRestartAction extends VespaConfigChangeAction implements ConfigChangeRestartAction { + + public VespaRestartAction(String message) { + super(message, new ArrayList<>()); + } + + public VespaRestartAction(String message, ServiceInfo service) { + super(message, Collections.singletonList(service)); + } + + public VespaRestartAction(String message, List<ServiceInfo> services) { + super(message, services); + } + + @Override + public VespaConfigChangeAction modifyAction(String newMessage, List<ServiceInfo> newServices, String documentType) { + return new VespaRestartAction(newMessage, newServices); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java new file mode 100644 index 00000000000..9e057c6cd28 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java @@ -0,0 +1,116 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change.search; + +import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.searchdefinition.derived.AttributeFields; +import com.yahoo.searchdefinition.derived.IndexSchema; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; +import com.yahoo.vespa.model.application.validation.change.VespaRestartAction; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +/** + * Validates the changes between the current and next set of attribute fields in a document database. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + * @since 2014-12-04 + */ +public class AttributeChangeValidator { + + private final AttributeFields currentFields; + private final IndexSchema currentIndexSchema; + private final NewDocumentType currentDocType; + private final AttributeFields nextFields; + private final IndexSchema nextIndexSchema; + private final NewDocumentType nextDocType; + + public AttributeChangeValidator(AttributeFields currentFields, + IndexSchema currentIndexSchema, + NewDocumentType currentDocType, + AttributeFields nextFields, + IndexSchema nextIndexSchema, + NewDocumentType nextDocType) { + this.currentFields = currentFields; + this.currentIndexSchema = currentIndexSchema; + this.currentDocType = currentDocType; + this.nextFields = nextFields; + this.nextIndexSchema = nextIndexSchema; + this.nextDocType = nextDocType; + } + + public List<VespaConfigChangeAction> validate() { + List<VespaConfigChangeAction> result = new ArrayList<>(); + result.addAll(validateAddAttributeAspect()); + result.addAll(validateRemoveAttributeAspect()); + result.addAll(validateAttributeSettings()); + return result; + } + + private List<VespaConfigChangeAction> validateAddAttributeAspect() { + return nextFields.attributes().stream(). + map(attr -> attr.getName()). + filter(attrName -> !currentFields.containsAttribute(attrName) && + currentDocType.containsField(attrName)). + map(attrName -> new VespaRestartAction(new ChangeMessageBuilder(attrName). + addChange("add attribute aspect").build())). + collect(Collectors.toList()); + } + + private List<VespaConfigChangeAction> validateRemoveAttributeAspect() { + return currentFields.attributes().stream(). + map(attr -> attr.getName()). + filter(attrName -> !nextFields.containsAttribute(attrName) && + nextDocType.containsField(attrName) && + !isIndexField(attrName)). + map(attrName -> new VespaRestartAction(new ChangeMessageBuilder(attrName). + addChange("remove attribute aspect").build())). + collect(Collectors.toList()); + } + + private boolean isIndexField(String fieldName) { + return currentIndexSchema.containsField(fieldName) && nextIndexSchema.containsField(fieldName); + } + + private List<VespaConfigChangeAction> validateAttributeSettings() { + List<VespaConfigChangeAction> result = new ArrayList<>(); + for (Attribute nextAttr : nextFields.attributes()) { + Attribute currAttr = currentFields.getAttribute(nextAttr.getName()); + if (currAttr != null) { + validateAttributeSetting(currAttr, nextAttr, Attribute::isFastSearch, "fast-search", result); + validateAttributeSetting(currAttr, nextAttr, Attribute::isFastAccess, "fast-access", result); + validateAttributeSetting(currAttr, nextAttr, Attribute::isHuge, "huge", result); + validateAttributeSetting(currAttr, nextAttr, Attribute::densePostingListThreshold, "dense-posting-list-threshold", result); + } + } + return result; + } + + private static void validateAttributeSetting(Attribute currentAttr, Attribute nextAttr, + Predicate<Attribute> predicate, String setting, + List<VespaConfigChangeAction> result) { + final boolean nextValue = predicate.test(nextAttr); + if (predicate.test(currentAttr) != nextValue) { + String change = nextValue ? "add" : "remove"; + result.add(new VespaRestartAction(new ChangeMessageBuilder(nextAttr.getName()). + addChange(change + " attribute '" + setting + "'").build())); + } + } + + private static <T> void validateAttributeSetting(Attribute currentAttr, Attribute nextAttr, + Function<Attribute, T> settingValueProvider, String setting, + List<VespaConfigChangeAction> result) { + T currentValue = settingValueProvider.apply(currentAttr); + T nextValue = settingValueProvider.apply(nextAttr); + if (!Objects.equals(currentValue, nextValue)) { + String message = String.format("change property '%s' from '%s' to '%s'", setting, currentValue, nextValue); + result.add(new VespaRestartAction(new ChangeMessageBuilder(nextAttr.getName()).addChange(message).build())); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/ChangeMessageBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/ChangeMessageBuilder.java new file mode 100644 index 00000000000..28b04d54307 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/ChangeMessageBuilder.java @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change.search; + +import java.util.ArrayList; +import java.util.List; + +/** + * Class used to build a message describing the changes in a given field. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + * @since 2014-12-09 + */ +public class ChangeMessageBuilder { + + private final String fieldName; + private final List<String> changes = new ArrayList<>(); + + public ChangeMessageBuilder(String fieldName) { + this.fieldName = fieldName; + } + + public String build() { + StringBuilder retval = new StringBuilder(); + retval.append("Field '" + fieldName + "' changed: "); + retval.append(String.join(", ", changes)); + return retval.toString(); + } + + public ChangeMessageBuilder addChange(String component, String from, String to) { + changes.add(component + ": '" + from + "' -> '" + to + "'"); + return this; + } + + public ChangeMessageBuilder addChange(String message) { + changes.add(message); + return this; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java new file mode 100644 index 00000000000..862b6caf0ca --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentDatabaseChangeValidator.java @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change.search; + +import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; +import com.yahoo.vespa.model.search.DocumentDatabase; + +import java.util.ArrayList; +import java.util.List; + +/** + * Validates the changes between a current and next document database that is part of an indexed search cluster. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + * @since 2014-11-18 + */ +public class DocumentDatabaseChangeValidator { + + private DocumentDatabase currentDatabase; + private NewDocumentType currentDocType; + private DocumentDatabase nextDatabase; + private NewDocumentType nextDocType; + + public DocumentDatabaseChangeValidator(DocumentDatabase currentDatabase, + NewDocumentType currentDocType, + DocumentDatabase nextDatabase, + NewDocumentType nextDocType) { + this.currentDatabase = currentDatabase; + this.currentDocType = currentDocType; + this.nextDatabase = nextDatabase; + this.nextDocType = nextDocType; + } + + public List<VespaConfigChangeAction> validate(ValidationOverrides overrides) { + List<VespaConfigChangeAction> result = new ArrayList<>(); + result.addAll(validateAttributeChanges()); + result.addAll(validateIndexingScriptChanges(overrides)); + result.addAll(validateDocumentTypeChanges(overrides)); + return result; + } + + private List<VespaConfigChangeAction> validateAttributeChanges() { + return new AttributeChangeValidator( + currentDatabase.getDerivedConfiguration().getAttributeFields(), + currentDatabase.getDerivedConfiguration().getIndexSchema(), currentDocType, + nextDatabase.getDerivedConfiguration().getAttributeFields(), + nextDatabase.getDerivedConfiguration().getIndexSchema(), nextDocType).validate(); + } + + private List<VespaConfigChangeAction> validateIndexingScriptChanges(ValidationOverrides overrides) { + return new IndexingScriptChangeValidator(currentDatabase.getDerivedConfiguration().getSearch(), + nextDatabase.getDerivedConfiguration().getSearch()).validate(overrides); + } + + private List<VespaConfigChangeAction> validateDocumentTypeChanges(ValidationOverrides overrides) { + return new DocumentTypeChangeValidator(currentDocType, nextDocType).validate(overrides); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java new file mode 100644 index 00000000000..4b0e350718c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidator.java @@ -0,0 +1,160 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change.search; + +import com.yahoo.document.StructDataType; +import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.document.Field; +import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; +import com.yahoo.vespa.model.application.validation.change.VespaRefeedAction; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * Validates the changes between a current and next document type used in a document database. + * + * @author <a href="mailto:Tor.Egge@yahoo-inc.com">Tor Egge</a> + * @since 2014-11-25 + */ +public class DocumentTypeChangeValidator { + + private NewDocumentType currentDocType; + private NewDocumentType nextDocType; + + private static abstract class FieldChange { + + protected final Field currentField; + protected final Field nextField; + + public FieldChange(Field currentField, Field nextField) { + this.currentField = currentField; + this.nextField = nextField; + } + + public String fieldName() { + return currentField.getName(); + } + + public boolean valid() { + return nextField != null; + } + + public abstract boolean changedType(); + public abstract String currentTypeName(); + public abstract String nextTypeName(); + } + + private static class SimpleFieldChange extends FieldChange { + + public SimpleFieldChange(Field currentField, Field nextField) { + super(currentField, nextField); + } + + public boolean changedType() { + return !currentField.getDataType().equals(nextField.getDataType()); + } + + public String currentTypeName() { + return currentField.getDataType().getName(); + } + + public String nextTypeName() { + return nextField.getDataType().getName(); + } + } + + private static class StructFieldChange extends FieldChange { + + private final StructDataType currentType; + private final StructDataType nextType; + + public StructFieldChange(Field currentField, Field nextField) { + super(currentField, nextField); + this.currentType = (StructDataType)currentField.getDataType(); + this.nextType = (StructDataType)nextField.getDataType(); + } + + public boolean changedType() { + return changedType(currentType, nextType); + } + + public String currentTypeName() { + return toString(currentType); + } + + public String nextTypeName() { + return toString(nextType); + } + + private static boolean changedType(StructDataType currentType, StructDataType nextType) { + for (Field currentField : currentType.getFields()) { + Field nextField = nextType.getField(currentField.getName()); + if (nextField != null) { + if (areStructFields(currentField, nextField)) { + if (changedType((StructDataType) currentField.getDataType(), + (StructDataType) nextField.getDataType())) { + return true; + } + } else { + if (!currentField.getDataType().equals(nextField.getDataType())) { + return true; + } + } + } + } + return false; + } + + private static String toString(StructDataType dataType) { + StringBuilder builder = new StringBuilder(); + builder.append(dataType.getName()).append(":{"); + boolean first = true; + for (Field field : dataType.getFields()) { + if (!first) { + builder.append(","); + } + if (field.getDataType() instanceof StructDataType) { + builder.append(toString((StructDataType) field.getDataType())); + } else { + builder.append(field.getName() + ":" + field.getDataType().getName()); + } + first = false; + } + builder.append("}"); + return builder.toString(); + } + } + + public DocumentTypeChangeValidator(NewDocumentType currentDocType, + NewDocumentType nextDocType) { + this.currentDocType = currentDocType; + this.nextDocType = nextDocType; + } + + public List<VespaConfigChangeAction> validate(ValidationOverrides overrides) { + return currentDocType.getAllFields().stream(). + map(field -> createFieldChange(field, nextDocType)). + filter(fieldChange -> fieldChange.valid() && fieldChange.changedType()). + map(fieldChange -> VespaRefeedAction.of("field-type-change", + overrides, + new ChangeMessageBuilder(fieldChange.fieldName()). + addChange("data type", fieldChange.currentTypeName(), + fieldChange.nextTypeName()).build())). + collect(Collectors.toList()); + } + + private static FieldChange createFieldChange(Field currentField, NewDocumentType nextDocType) { + Field nextField = nextDocType.getField(currentField.getName()); + if (nextField != null && areStructFields(currentField, nextField)) { + return new StructFieldChange(currentField, nextField); + } + return new SimpleFieldChange(currentField, nextField); + } + + private static boolean areStructFields(Field currentField, Field nextField) { + return (currentField.getDataType() instanceof StructDataType) && + (nextField.getDataType() instanceof StructDataType); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java new file mode 100644 index 00000000000..5d8121b6ef8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java @@ -0,0 +1,97 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change.search; + +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.NormalizeLevel; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.vespa.documentmodel.SummaryField; +import com.yahoo.vespa.documentmodel.SummaryTransform; + +/** + * Class used to build a message describing the usual field changes causing changes in the indexing script. + * This message should be more descriptive for the end-user than just seeing the changed indexing script. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + * @since 2014-12-09 + */ +public class IndexingScriptChangeMessageBuilder { + + private final Search currentSearch; + private final SDField currentField; + private final Search nextSearch; + private final SDField nextField; + + public IndexingScriptChangeMessageBuilder(Search currentSearch, SDField currentField, + Search nextSearch, SDField nextField) { + this.currentSearch = currentSearch; + this.currentField = currentField; + this.nextSearch = nextSearch; + this.nextField = nextField; + } + + public void populate(ChangeMessageBuilder builder) { + checkIndexing(builder); + checkMatching(builder); + checkStemming(builder); + checkNormalizing(builder); + checkSummaryTransform(builder); + } + + private void checkIndexing(ChangeMessageBuilder builder) { + if (currentField.doesIndexing() != nextField.doesIndexing()) { + String change = nextField.doesIndexing() ? "add" : "remove"; + builder.addChange(change + " index aspect"); + } + } + + private void checkMatching(ChangeMessageBuilder builder) { + Matching currentMatching = currentField.getMatching(); + Matching nextMatching = nextField.getMatching(); + if (!currentMatching.equals(nextMatching)) { + builder.addChange("matching", toString(currentMatching), toString(nextMatching)); + } + } + + private void checkStemming(ChangeMessageBuilder builder) { + Stemming currentStemming = currentField.getStemming(currentSearch); + Stemming nextStemming = nextField.getStemming(nextSearch); + if (!currentStemming.equals(nextStemming)) { + builder.addChange("stemming", currentStemming.getName(), nextStemming.getName()); + } + } + + private void checkNormalizing(ChangeMessageBuilder builder) { + NormalizeLevel.Level currentLevel = currentField.getNormalizing().getLevel(); + NormalizeLevel.Level nextLevel = nextField.getNormalizing().getLevel(); + if (!currentLevel.equals(nextLevel)) { + builder.addChange("normalizing", currentLevel.toString(), nextLevel.toString()); + } + } + + private void checkSummaryTransform(ChangeMessageBuilder builder) { + for (SummaryField nextSummaryField : nextField.getSummaryFields()) { + String fieldName = nextSummaryField.getName(); + SummaryField currentSummaryField = currentField.getSummaryField(fieldName); + if (currentSummaryField != null) { + SummaryTransform currentTransform = currentSummaryField.getTransform(); + SummaryTransform nextTransform = nextSummaryField.getTransform(); + if (!currentSummaryField.getTransform().equals(nextSummaryField.getTransform())) { + builder.addChange("summary field '" + fieldName + "' transform", + currentTransform.getName(), nextTransform.getName()); + } + } + } + } + + private static String toString(Matching matching) { + Matching.Type type = matching.getType(); + String retval = type.getName(); + if (type.equals(Matching.Type.GRAM)) { + retval += " (size " + matching.getGramSize() + ")"; + } + return retval; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java new file mode 100644 index 00000000000..f1043f14fdc --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java @@ -0,0 +1,89 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change.search; + +import com.yahoo.searchdefinition.Search; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; +import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; +import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; +import com.yahoo.vespa.model.application.validation.change.VespaRefeedAction; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * Validates the indexing script changes in all fields in the current and next search model. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + * @since 2014-12-08 + */ +public class IndexingScriptChangeValidator { + + private final Search currentSearch; + private final Search nextSearch; + + public IndexingScriptChangeValidator(Search currentSearch, Search nextSearch) { + this.currentSearch = currentSearch; + this.nextSearch = nextSearch; + } + + public List<VespaConfigChangeAction> validate(ValidationOverrides overrides) { + List<VespaConfigChangeAction> result = new ArrayList<>(); + for (SDField nextField : nextSearch.allFieldsList()) { + String fieldName = nextField.getName(); + SDField currentField = currentSearch.getField(fieldName); + if (currentField != null) { + validateScripts(currentField, nextField, overrides).ifPresent(r -> result.add(r)); + } + } + return result; + } + + private Optional<VespaConfigChangeAction> validateScripts(SDField currentField, SDField nextField, + ValidationOverrides overrides) { + ScriptExpression currentScript = currentField.getIndexingScript(); + ScriptExpression nextScript = nextField.getIndexingScript(); + if (!equalScripts(currentScript, nextScript)) { + ChangeMessageBuilder messageBuilder = new ChangeMessageBuilder(nextField.getName()); + new IndexingScriptChangeMessageBuilder(currentSearch, currentField, nextSearch, nextField).populate(messageBuilder); + messageBuilder.addChange("indexing script", currentScript.toString(), nextScript.toString()); + return Optional.of(VespaRefeedAction.of("indexing-change", overrides, messageBuilder.build())); + } + return Optional.empty(); + } + + static boolean equalScripts(ScriptExpression currentScript, + ScriptExpression nextScript) { + // Output expressions are specifying in which context a field value is used (attribute, index, summary), + // and do not affect how the field value is generated in the indexing doc proc. + // The output expressions are therefore removed before doing the comparison. + // Validating the addition / removal of attribute and index aspects are handled in other validators. + return removeOutputExpressions(currentScript).equals(removeOutputExpressions(nextScript)); + } + + private static ScriptExpression removeOutputExpressions(ScriptExpression script) { + ScriptExpression retval = (ScriptExpression) new OutputExpressionRemover().convert(script); + return retval; + } + + private static class OutputExpressionRemover extends ExpressionConverter { + + @Override + protected boolean shouldConvert(Expression exp) { + return exp instanceof OutputExpression; + } + + @Override + protected Expression doConvert(Expression exp) { + if (exp instanceof OutputExpression) { + return null; + } + return exp; + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/package-info.java new file mode 100644 index 00000000000..1e2df635ab6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/xml/ValidationOverridesXMLReader.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/xml/ValidationOverridesXMLReader.java new file mode 100644 index 00000000000..0bbc4e32010 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/xml/ValidationOverridesXMLReader.java @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.xml; + +import com.yahoo.text.XML; +import com.yahoo.vespa.model.application.validation.ValidationId; +import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import org.w3c.dom.Element; + +import java.io.Reader; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * Reader of the validation-allows.xml file in application packages. + * + * @author bratseth + */ +public class ValidationOverridesXMLReader { + + /** + * Returns a ValidationOverrides instance with the content of the given Reader. + * An empty ValidationOverrides is returned if the argument is empty. + * + * @param reader the reader which optionally contains a validation-overrides XML structure + * @param now the instant to use as "now", settable for unit testing + * @return a ValidationOverrides from the argument + * @throws IllegalArgumentException if the validation-allows.xml file exists but is invalid + */ + public ValidationOverrides read(Optional<Reader> reader, Instant now) { + if ( ! reader.isPresent()) return ValidationOverrides.empty(); + + try { + // Assume valid structure is ensured by schema validation + Element root = XML.getDocument(reader.get()).getDocumentElement(); + List<ValidationOverrides.Allow> overrides = new ArrayList<>(); + for (Element allow : XML.getChildren(root, "allow")) { + Instant until = LocalDate.parse(allow.getAttribute("until"), DateTimeFormatter.ISO_DATE) + .atStartOfDay().atZone(ZoneOffset.UTC).toInstant() + .plus(Duration.ofDays(1)); // Make the override valid *on* the "until" date + Optional<ValidationId> validationId = ValidationId.from(XML.getValue(allow)); + if (validationId.isPresent()) // skip unknonw ids as they may be valid for other model versions + overrides.add(new ValidationOverrides.Allow(validationId.get(), until)); + } + return new ValidationOverrides(overrides, now); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("validation-overrides is invalid", e); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java new file mode 100644 index 00000000000..612df56cfc9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java @@ -0,0 +1,77 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder; + +import com.yahoo.config.model.deploy.ConfigDefinitionStore; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.producer.UserConfigRepo; +import com.yahoo.log.LogLevel; +import com.yahoo.text.XML; +import com.yahoo.vespa.config.*; +import com.yahoo.vespa.model.builder.xml.dom.DomConfigPayloadBuilder; +import org.w3c.dom.Element; + +import java.util.*; +import java.util.logging.Logger; + +/** + * @author lulf + * @since 5.1 + */ +public class UserConfigBuilder { + + public static final Logger log = Logger.getLogger(UserConfigBuilder.class.getPackage().toString()); + + public static UserConfigRepo build(Element producerSpec, ConfigDefinitionStore configDefinitionStore, DeployLogger deployLogger) { + final Map<ConfigDefinitionKey, ConfigPayloadBuilder> builderMap = new LinkedHashMap<>(); + if (producerSpec == null) { + log.log(LogLevel.SPAM, "In getUserConfigs. producerSpec is null"); + } + log.log(LogLevel.DEBUG, "getUserConfigs for " + producerSpec); + for (Element configE : XML.getChildren(producerSpec, "config")) { + buildElement(configE, builderMap, configDefinitionStore, deployLogger); + } + return new UserConfigRepo(builderMap); + } + + + private static void buildElement(Element element, Map<ConfigDefinitionKey, ConfigPayloadBuilder> builderMap, ConfigDefinitionStore configDefinitionStore, DeployLogger logger) { + ConfigDefinitionKey key = DomConfigPayloadBuilder.parseConfigName(element); + log.log(LogLevel.SPAM, "Looking at " + key); + + ConfigDefinition def = getConfigDef(key, configDefinitionStore); + // TODO: Fail here unless deploying with :force true + if (def == null) { + logger.log(LogLevel.WARNING, "Unable to find config definition for config '" + key.getNamespace() + "." + key.getName() + + "'. Please ensure that the name is spelled correctly, and that the def file is included in a bundle."); + } + List<String> issuedWarnings = new ArrayList<>(); + for (String warning : issuedWarnings) { + logger.log(LogLevel.WARNING, warning); + } + ConfigPayloadBuilder payloadBuilder = new DomConfigPayloadBuilder(def).build(element, issuedWarnings); + log.log(LogLevel.SPAM, "configvalue=" + ConfigPayload.fromBuilder(payloadBuilder).toString()); + log.log(LogLevel.DEBUG, "Looking up key: " + key.toString()); + ConfigPayloadBuilder old = builderMap.get(key); + if (old != null) { + logger.log(LogLevel.WARNING, "Multiple overrides for " + key + " found. Applying in the order they are discovered"); + log.log(LogLevel.DEBUG, "old configvalue=" + old); + old.override(payloadBuilder); + } else { + builderMap.put(key, payloadBuilder); + } + } + + /** + * Returns the config definition matching the given name, or null if not found. + */ + private static ConfigDefinition getConfigDef(ConfigDefinitionKey configDefinitionKey, ConfigDefinitionStore configDefinitionStore) { + try { + return configDefinitionStore.getConfigDefinition(configDefinitionKey); + } catch (IllegalArgumentException e) { + log.log(LogLevel.DEBUG, "Unable to retrieve config definition: " + e.getMessage()); + return null; + } + } + +} + diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java new file mode 100644 index 00000000000..75e9caefbd5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder; + +import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.model.ApplicationConfigProducerRoot; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.vespa.model.generic.service.ServiceCluster; + +import java.util.List; + +/** + * Base class for classes capable of building vespa model. + * + * @author vegardh + */ +public abstract class VespaModelBuilder { + + + public abstract ApplicationConfigProducerRoot getRoot(String name, DeployState deployState, AbstractConfigProducer parent); + public abstract List<ServiceCluster> getClusters(ApplicationPackage pkg, AbstractConfigProducer parent); + + /** + * Processing that requires access across plugins + * @param producerRoot The root producer. + * @param configModelRepo a {@link com.yahoo.config.model.ConfigModelRepo instance} + */ + public abstract void postProc(AbstractConfigProducer producerRoot, ConfigModelRepo configModelRepo); +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryScaledAmountParser.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryScaledAmountParser.java new file mode 100644 index 00000000000..9fbdee30661 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryScaledAmountParser.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.binaryprefix.BinaryPrefix; +import com.yahoo.binaryprefix.BinaryScaledAmount; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @author tonytv + */ +public class BinaryScaledAmountParser { + //The pattern must match the one given in the schema + private static Pattern pattern = Pattern.compile("(\\d+(\\.\\d*)?)\\s*([kmgKMG])?"); + + public static BinaryScaledAmount parse(String valueString) { + Matcher matcher = pattern.matcher(valueString); + + if (!matcher.matches()) { + throw new RuntimeException("Pattern and schema is out of sync."); + } + + double amount = Double.valueOf(matcher.group(1)); + String binaryPrefixString = matcher.group(3); + + return new BinaryScaledAmount(amount, asBinaryPrefix(binaryPrefixString)); + } + + private static BinaryPrefix asBinaryPrefix(String binaryPrefixString) { + if (binaryPrefixString == null) { + return BinaryPrefix.unit; + } else { + return BinaryPrefix.fromSymbol(binaryPrefixString.toUpperCase().charAt(0)); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryUnit.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryUnit.java new file mode 100644 index 00000000000..38d919f0f23 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/BinaryUnit.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.yahoo.text.Lowercase.toLowerCase; + +/** + * @author tonytv + */ +public class BinaryUnit { + //The pattern must match the one given in the schema + private static Pattern pattern = Pattern.compile("(\\d+(\\.\\d*)?)\\s*([kmgKMG])?"); + + public static double valueOf(String valueString) { + Matcher matcher = pattern.matcher(valueString); + + matcher.matches(); + double value = Double.valueOf(matcher.group(1)); + String unit = matcher.group(3); + if (unit != null) { + value *= unitToValue(toLowerCase(unit).charAt(0)); + } + return value; + } + + private static double unitToValue(char unit) { + final char units[] = {'k', 'm', 'g'}; + for (int i=0; i<units.length; ++i) { + if (units[i] == unit) { + return Math.pow(2, 10*(i+1)); + } + } + + throw new RuntimeException("No such unit: '" + unit + "'"); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java new file mode 100644 index 00000000000..179f6bad0f3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminBuilderBase.java @@ -0,0 +1,101 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.config.model.api.ConfigServerSpec; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.log.LogLevel; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.Host; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.HostSystem; +import com.yahoo.vespa.model.admin.*; +import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; +import com.yahoo.config.application.api.FileRegistry; +import org.w3c.dom.Element; + +import java.util.*; + +/** + * A base class for admin model builders, to support common functionality across versions. + * + * @author lulf + * @author vegardh + * @since 5.12 + */ +public abstract class DomAdminBuilderBase extends VespaDomBuilder.DomConfigProducerBuilder<Admin> { + + private static final int DEFAULT_INTERVAL = 1; // in minutes + private static final String DEFAULT_CLUSTER_NAME = "vespa"; + + private final List<ConfigServerSpec> configServerSpecs; + private final FileRegistry fileRegistry; + protected final boolean multitenant; + + public DomAdminBuilderBase(FileRegistry fileRegistry, boolean multitenant, List<ConfigServerSpec> configServerSpecs) { + this.fileRegistry = fileRegistry; + this.multitenant = multitenant; + this.configServerSpecs = configServerSpecs; + } + + protected List<Configserver> getConfigServersFromSpec(AbstractConfigProducer parent) { + List<Configserver> configservers = new ArrayList<>(); + for (ConfigServerSpec spec : configServerSpecs) { + HostSystem hostSystem = parent.getHostSystem(); + HostResource host = new HostResource(Host.createMultitenantHost(hostSystem, spec.getHostName())); + hostSystem.addBoundHost(host); + Configserver configserver = new Configserver(parent, spec.getHostName()); + configserver.setHostResource(host); + configserver.setBasePort(configserver.getWantedPort()); + configserver.initService(); + configservers.add(configserver); + } + return configservers; + } + + @Override + protected Admin doBuild(AbstractConfigProducer parent, Element adminE) { + Yamas yamas = getYamas(XML.getChild(adminE, "yamas")); + Map<String, MetricsConsumer> metricsConsumers = DomMetricBuilderHelper.buildMetricsConsumers(XML.getChild(adminE, "metric-consumers")); + + Admin admin = new Admin(parent, yamas, metricsConsumers, multitenant); + + doBuildAdmin(admin, adminE); + + new ModelConfigProvider(admin); + + FileDistributionOptions fileDistributionOptions = new DomFileDistributionOptionsBuilder().build(XML.getChild(adminE, "filedistribution")); + admin.setFileDistribution(new FileDistributionConfigProducer.Builder(fileDistributionOptions).build(parent, fileRegistry)); + return admin; + } + + protected abstract void doBuildAdmin(Admin admin, Element adminE); + + private Yamas getYamas(Element yamasE) { + Yamas yamas; + if (yamasE == null) { + yamas = new Yamas(DEFAULT_CLUSTER_NAME, DEFAULT_INTERVAL); + } else { + Integer minutes = getMonitoringInterval(yamasE); + if (minutes == null) { + minutes = DEFAULT_INTERVAL; + } + yamas = new Yamas(yamasE.getAttribute("systemname"), minutes); + } + return yamas; + } + + private Integer getMonitoringInterval(Element monitoringE) { + Integer minutes = null; + String seconds = monitoringE.getAttribute("interval").trim(); + if (!seconds.isEmpty()) { + minutes = Integer.parseInt(seconds); + minutes = minutes / 60; + if (!(minutes == 1 || minutes == 5)) { + throw new IllegalArgumentException("The only allowed values for 'interval' attribute in '" + monitoringE.getTagName() + + "' element is 60 or 300."); + } + } + return minutes; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java new file mode 100644 index 00000000000..c45cbf8f0d2 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -0,0 +1,197 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.config.model.ConfigModelUtils; +import com.yahoo.config.model.api.ConfigServerSpec; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.text.XML; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.model.*; +import com.yahoo.vespa.model.admin.*; +import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerCluster; +import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder.DomConfigProducerBuilder; +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; +import com.yahoo.config.application.api.FileRegistry; +import org.w3c.dom.Element; + +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; + +/** + * Builds the admin model from a V2 admin XML tag. + * + * @author vegardh + */ +public class DomAdminV2Builder extends DomAdminBuilderBase { + + private static final String ATTRIBUTE_CLUSTER_CONTROLLER_STANDALONE_ZK = "standalone-zookeeper"; + + public DomAdminV2Builder(FileRegistry fileRegistry, boolean multitenant, List<ConfigServerSpec> configServerSpecs) { + super(fileRegistry, multitenant, configServerSpecs); + } + + @Override + protected void doBuildAdmin(Admin admin, Element adminE) { + List<Configserver> configservers = parseConfigservers(admin, adminE); + admin.setLogserver(parseLogserver(admin, adminE)); + admin.addConfigservers(configservers); + admin.addSlobroks(getSlobroks(admin, XML.getChild(adminE, "slobroks"))); + if ( ! admin.multitenant()) + admin.setClusterControllers(addConfiguredClusterControllers(admin, adminE)); + } + + private List<Configserver> parseConfigservers(Admin admin, Element adminE) { + List<Configserver> configservers; + if (multitenant) { + configservers = getConfigServersFromSpec(admin); + } else { + configservers = getConfigServers(admin, adminE); + } + int count = configservers.size(); + if (count % 2 == 0) { + admin.deployLogger().log(Level.WARNING, "An even number (" + count + ") of config servers have been configured. " + + "This is discouraged, see " + "" + + ConfigModelUtils.createDocLink("cloudconfig/configuration-server.html#fault-tolerance")); + } + return configservers; + } + + private Logserver parseLogserver(Admin admin, Element adminE) { + Element logserverE = XML.getChild(adminE, "logserver"); + if (logserverE == null) { + logserverE = XML.getChild(adminE, "adminserver"); + } + return new LogserverBuilder().build(admin, logserverE); + } + + private ContainerCluster addConfiguredClusterControllers(AbstractConfigProducer parent, Element admin) { + Element controllersElements = XML.getChild(admin, "cluster-controllers"); + if (controllersElements == null) return null; + + List<Element> controllers = XML.getChildren(controllersElements, "cluster-controller"); + if (controllers.isEmpty()) return null; + + boolean standaloneZooKeeper = "true".equals(controllersElements.getAttribute(ATTRIBUTE_CLUSTER_CONTROLLER_STANDALONE_ZK)) || multitenant; + if (standaloneZooKeeper) { + parent = new ClusterControllerCluster(parent, "standalone"); + } + ContainerCluster cluster = new ContainerCluster(parent, "cluster-controllers", "cluster-controllers"); + ContainerModelBuilder.addDefaultHandler_legacyBuilder(cluster); + + List<Container> containers = new ArrayList<>(); + + for (Element controller : controllers) { + ClusterControllerContainer clusterController = new ClusterControllerBuilder(containers.size(), standaloneZooKeeper).build(cluster, controller); + containers.add(clusterController); + } + + cluster.addContainers(containers); + return cluster; + } + + // Extra stupid because configservers tag is voluntary + private List<Configserver> getConfigServers(AbstractConfigProducer parent, Element adminE) { + SimpleConfigProducer configServers = new SimpleConfigProducer(parent, "configservers"); + List<Configserver> cfgs = new ArrayList<>(); + Element configserversE = XML.getChild(adminE, "configservers"); + if (configserversE == null) { + Element configserverE = XML.getChild(adminE, "configserver"); + if (configserverE == null) { + configserverE = XML.getChild(adminE, "adminserver"); + } else { + parent.deployLogger().log(LogLevel.INFO, "Specifying configserver without parent element configservers in services.xml is deprecated"); + } + Configserver cfgs0 = new ConfigserverBuilder(0).build(configServers, configserverE); + cfgs0.setProp("index", 0); + cfgs.add(cfgs0); + return cfgs; + } + // configservers tag in use + int i = 0; + for (Element configserverE : XML.getChildren(configserversE, "configserver")) { + Configserver cfgsrv = new ConfigserverBuilder(i).build(configServers, configserverE); + cfgsrv.setProp("index", i); + cfgs.add(cfgsrv); + i++; + } + return cfgs; + } + + private List<Slobrok> getSlobroks(AbstractConfigProducer parent, Element slobroksE) { + List<Slobrok> slobs = new ArrayList<>(); + if (slobroksE != null) { + slobs = getExplicitSlobrokSetup(parent, slobroksE); + } + return slobs; + } + + private List<Slobrok> getExplicitSlobrokSetup(AbstractConfigProducer parent, Element slobroksE) { + List<Slobrok> slobs = new ArrayList<>(); + List<Element> slobsE = XML.getChildren(slobroksE, "slobrok"); + int i = 0; + for (Element e : slobsE) { + Slobrok slob = new SlobrokBuilder(i).build(parent, e); + slobs.add(slob); + i++; + } + return slobs; + } + + private static class LogserverBuilder extends DomConfigProducerBuilder<Logserver> { + public LogserverBuilder() { + } + + @Override + protected Logserver doBuild(AbstractConfigProducer parent, Element producerSpec) { + return new Logserver(parent); + } + } + + private static class ConfigserverBuilder extends DomConfigProducerBuilder<Configserver> { + int i; + + public ConfigserverBuilder(int i) { + this.i = i; + } + + @Override + protected Configserver doBuild(AbstractConfigProducer parent, + Element spec) { + return new Configserver(parent, "configserver." + i); + } + } + + private static class SlobrokBuilder extends DomConfigProducerBuilder<Slobrok> { + int i; + + public SlobrokBuilder(int i) { + this.i = i; + } + + @Override + protected Slobrok doBuild(AbstractConfigProducer parent, + Element spec) { + return new Slobrok(parent, i); + } + } + + private static class ClusterControllerBuilder extends DomConfigProducerBuilder<ClusterControllerContainer> { + int i; + boolean runStandaloneZooKeeper; + + public ClusterControllerBuilder(int i, boolean runStandaloneZooKeeper) { + this.i = i; + this.runStandaloneZooKeeper = runStandaloneZooKeeper; + } + + @Override + protected ClusterControllerContainer doBuild(AbstractConfigProducer parent, + Element spec) { + return new ClusterControllerContainer(parent, i, runStandaloneZooKeeper); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java new file mode 100644 index 00000000000..1144ea775ab --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -0,0 +1,138 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.config.model.api.ConfigServerSpec; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.HostSystem; +import com.yahoo.vespa.model.admin.*; +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.vespa.model.admin.Admin; +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerModel; + +import org.w3c.dom.Element; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +/** + * Builds the admin model from a version 4 XML tag, or as a default when an admin 3 tag or no admin tag is used. + * + * @author bratseth + */ +public class DomAdminV4Builder extends DomAdminBuilderBase { + + private final Collection<ContainerModel> containerModels; + private final DeployLogger deployLogger; + + public DomAdminV4Builder(ConfigModelContext modelContext, boolean multitenant, List<ConfigServerSpec> configServerSpecs, Collection<ContainerModel> containerModels) { + super(modelContext.getDeployState().getFileRegistry(), multitenant, configServerSpecs); + this.containerModels = containerModels; + this.deployLogger = modelContext.getDeployLogger(); + } + + @Override + protected void doBuildAdmin(Admin admin, Element w3cAdminElement) { + ModelElement adminElement = new ModelElement(w3cAdminElement); + admin.addConfigservers(getConfigServersFromSpec(admin)); + + // Note: These two elements only exists in admin version 4.0 + // This build handles admin version 3.0 by ignoring its content (as the content is not useful) + Optional<NodesSpecification> requestedSlobroks = NodesSpecification.optionalDedicatedFromParent(adminElement.getChild("slobroks")); + Optional<NodesSpecification> requestedLogservers = NodesSpecification.optionalDedicatedFromParent(adminElement.getChild("logservers")); + assignSlobroks(requestedSlobroks.orElse(NodesSpecification.nonDedicated(3)), admin); + assignLogserver(requestedLogservers.orElse(NodesSpecification.nonDedicated(1)), admin); + } + + private void assignSlobroks(NodesSpecification nodesSpecification, Admin admin) { + if (nodesSpecification.isDedicated()) { + createSlobroks(admin, allocateHosts(admin.getHostSystem(), "slobroks", nodesSpecification)); + } + else { + createSlobroks(admin, pickContainerHosts(nodesSpecification.count())); + } + } + + private void assignLogserver(NodesSpecification nodesSpecification, Admin admin) { + if (nodesSpecification.count() > 1) throw new IllegalArgumentException("You can only request a single log server"); + + if (nodesSpecification.isDedicated()) { + createLogserver(admin, allocateHosts(admin.getHostSystem(), "logserver", nodesSpecification)); + } + else { + if (containerModels.iterator().hasNext()) + createLogserver(admin, sortedContainerHostsFrom(containerModels.iterator().next(), nodesSpecification.count(), false)); + } + } + + private Collection<HostResource> allocateHosts(HostSystem hostSystem, String clusterId, NodesSpecification nodesSpecification) { + return nodesSpecification.provision(hostSystem, ClusterSpec.Type.admin, ClusterSpec.Id.from(clusterId), Optional.empty(), deployLogger).keySet(); + } + + /** + * Returns a list of container hosts to use for an auxiliary cluster. + * The list returns the same nodes on each invocation given the same available nodes. + * + * @param count the desired number of nodes. More nodes may be returned to ensure a smooth transition + * on topology changes, and less nodes may be returned if fewer are available + */ + private List<HostResource> pickContainerHosts(int count) { + // Pick from all container clusters to make sure we don't lose all nodes at once if some clusters are removed. + // This will overshoot the desired size (due to ceil and picking at least one node per cluster). + List<HostResource> picked = new ArrayList<>(); + for (ContainerModel containerModel : containerModels) + picked.addAll(pickContainerHostsFrom(containerModel, + (int)Math.max(1, Math.ceil((double)count/containerModels.size())))); + return picked; + } + + private List<HostResource> pickContainerHostsFrom(ContainerModel model, int count) { + boolean retired = true; + List<HostResource> picked = sortedContainerHostsFrom(model, count, !retired); + + // if we can return multiple hosts, include retired nodes which would have been picked before + // (probably - assuming all previous nodes were retired, which is always true for a single cluster + // at the moment (Sept 2015)) // to ensure a smoother transition between the old and new topology + // by including both new and old nodes during the retirement period + picked.addAll(sortedContainerHostsFrom(model, count, retired)); + + return picked; + } + + /** Returns the count first containers in the current model having isRetired set to the given value */ + private List<HostResource> sortedContainerHostsFrom(ContainerModel model, int count, boolean retired) { + List<HostResource> hosts = new ArrayList<>(); + for (Container container : model.getCluster().getContainers()) + if (retired == container.isRetired()) + hosts.add(container.getHostResource()); + Collections.sort(hosts); + return hosts.subList(0, Math.min(count, hosts.size())); + } + + private void createLogserver(Admin admin, Collection<HostResource> hosts) { + if (hosts.isEmpty()) return; // No log server can be created (and none is needed) + Logserver logserver = new Logserver(admin); + logserver.setHostResource(hosts.iterator().next()); + admin.setLogserver(logserver); + logserver.initService(); + } + + private void createSlobroks(Admin admin, Collection<HostResource> hosts) { + if (hosts.isEmpty()) return; // No slobroks can be created (and none are needed) + List<Slobrok> slobroks = new ArrayList<>(); + int index = 0; + for (HostResource host : hosts) { + Slobrok slobrok = new Slobrok(admin, index++); + slobrok.setHostResource(host); + slobroks.add(slobrok); + slobrok.initService(); + } + admin.addSlobroks(slobroks); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java new file mode 100644 index 00000000000..56415c7232d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.text.XML; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.Handler; +import org.w3c.dom.Element; + +/** + * @author gjoranv + * @since 5.1.6 + */ +public class DomClientProviderBuilder extends DomHandlerBuilder { + + @Override + protected Handler doBuild(AbstractConfigProducer ancestor, Element clientElement) { + Handler<? super Component<?, ?>> client = getHandler(clientElement); + + for (Element binding : XML.getChildren(clientElement, "binding")) + client.addClientBindings(XML.getValue(binding)); + + for (Element serverBinding : XML.getChildren(clientElement, "serverBinding")) + client.addServerBindings(XML.getValue(serverBinding)); + + DomComponentBuilder.addChildren(ancestor, clientElement, client); + + return client; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientsBuilder.java new file mode 100644 index 00000000000..0e0c94abae1 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientsBuilder.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.builder.xml.ConfigModelId; +import com.yahoo.vespa.model.clients.*; +import org.w3c.dom.Element; + +import java.util.Arrays; +import java.util.List; + +/** + * Builds the Clients plugin + * + * @author musum + */ +public class DomClientsBuilder extends LegacyConfigModelBuilder<Clients> { + + public DomClientsBuilder() { + super(Clients.class); + } + + @Override + public List<ConfigModelId> handlesElements() { + return Arrays.asList(ConfigModelId.fromNameAndVersion("clients", "2.0")); + } + + @Override + public void doBuild(Clients clients, Element clientsE, ConfigModelContext modelContext) { + String version = clientsE.getAttribute("version"); + if (version.startsWith("2.")) { + DomV20ClientsBuilder parser = new DomV20ClientsBuilder(clients, version); + parser.build(clientsE); + } else { + throw new IllegalArgumentException("Version '" + version + "' of 'clients' not supported."); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java new file mode 100644 index 00000000000..17a59bd7590 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomComponentBuilder.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.component.ComponentId; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder; +import org.w3c.dom.Element; + +/** + * @author gjoranv + * @author tonytv + */ +public class DomComponentBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Component> { + + public static final String elementName = "component" ; + + private final ComponentId namespace; + + public DomComponentBuilder() { + this(null); + } + + public DomComponentBuilder(ComponentId namespace) { + this.namespace = namespace; + } + + protected Component doBuild(AbstractConfigProducer ancestor, Element spec) { + Component component = buildComponent(spec); + addChildren(ancestor, spec, component); + return component; + } + + private Component buildComponent(Element spec) { + BundleInstantiationSpecification bundleSpec = + BundleInstantiationSpecificationBuilder.build(spec, false).nestInNamespace(namespace); + + return new Component<Component<?, ?>, ComponentModel>(new ComponentModel(bundleSpec)); + } + + public static void addChildren(AbstractConfigProducer ancestor, Element componentNode, Component<? super Component<?, ?>, ?> component) { + for (Element childNode : XML.getChildren(componentNode, elementName)) { + addAndInjectChild(ancestor, component, childNode); + } + } + + private static void addAndInjectChild(AbstractConfigProducer ancestor, Component<? super Component<?, ?>, ?> component, Element childNode) { + Component<?, ?> child = new DomComponentBuilder(component.getComponentId()).build(ancestor, childNode); + component.addComponent(child); + component.inject(child); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java new file mode 100644 index 00000000000..a2478f80397 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java @@ -0,0 +1,243 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.collections.Tuple2; +import com.yahoo.config.ConfigurationRuntimeException; +import com.yahoo.config.codegen.CNode; +import com.yahoo.log.LogLevel; +import com.yahoo.yolean.Exceptions; +import com.yahoo.text.XML; +import com.yahoo.vespa.config.*; + +import com.yahoo.vespa.config.util.ConfigUtils; +import org.w3c.dom.Element; + +import java.util.*; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +/** + * Builder that transforms xml config to a slime tree representation of the config. The root element of the xml config + * must be named 'config' and have a 'name' attribute that matches the name of the {@link ConfigDefinition}. The values + * are not validated against their types. That task is moved to the builders. + * + * @author lulf + */ +public class DomConfigPayloadBuilder { + + private static final Logger log = Logger.getLogger(DomConfigPayloadBuilder.class.getPackage().toString()); + + private static final Pattern namePattern = ConfigDefinition.namePattern; + private static final Pattern namespacePattern = ConfigDefinition.namespacePattern; + + private final ConfigDefinition configDefinition; + + public DomConfigPayloadBuilder(ConfigDefinition configDefinition) { + this.configDefinition = configDefinition; + } + + /** + * Builds a {@link ConfigPayloadBuilder} representing the input 'config' xml element. + * + * @param configE The 'config' xml element + * @param issuedWarnings a list that will be populated with issued warnings when building the config payload + * @return a new payload builder built from xml. + */ + public ConfigPayloadBuilder build(Element configE, List<String> issuedWarnings) { + parseConfigName(configE); + + ConfigPayloadBuilder payloadBuilder = new ConfigPayloadBuilder(configDefinition, issuedWarnings); + for (Element child : XML.getChildren(configE)) { + parseElement(child, payloadBuilder, null); + } + return payloadBuilder; + } + + public static ConfigDefinitionKey parseConfigName(Element configE) { + if (!configE.getNodeName().equals("config")) { + throw new ConfigurationRuntimeException("The root element must be 'config', but was '" + + configE.getNodeName() + "'."); + } + if (!configE.hasAttribute("name")) { + throw new ConfigurationRuntimeException + ("The 'config' element must have a 'name' attribute that matches the name of the config definition."); + } + + String xmlName = configE.getAttribute("name"); + final boolean xmlNamespaceAttributeExists = configE.hasAttribute("namespace"); + + String xmlNamespace = null; + // If name contains dots, rewrite to name and namespace + if (xmlName.contains(".")) { + Tuple2<String, String> t = ConfigUtils.getNameAndNamespaceFromString(xmlName); + xmlName = t.first; + xmlNamespace = t.second; + } else { + if (!xmlNamespaceAttributeExists) { + log.log(LogLevel.WARNING, "No namespace in 'config name=" + xmlName + "', please specify one"); + } + } + + if (!validName(xmlName)) { + throw new ConfigurationRuntimeException("The config name '" + xmlName + + "' contains illegal characters. Only names with the pattern " + namePattern.toString() + " are legal."); + } + + if (xmlNamespace == null) { + xmlNamespace = configE.getAttribute("namespace"); + if (xmlNamespace == null || xmlNamespace.isEmpty()) { + xmlNamespace = CNode.DEFAULT_NAMESPACE; + } + } + if (!validNamespace(xmlNamespace)) { + throw new ConfigurationRuntimeException("The config namespace '" + xmlNamespace + + "' contains illegal characters. Only namespaces with the pattern " + namespacePattern.toString() + " are legal."); + } + return new ConfigDefinitionKey(xmlName, xmlNamespace); + } + + private static boolean validName(String name) { + Matcher m = namePattern.matcher(name); + return m.matches(); + } + + private static boolean validNamespace(String namespace) { + Matcher m = namespacePattern.matcher(namespace); + return m.matches(); + } + + private String extractName(Element element) { + String initial = element.getNodeName(); + if (initial.indexOf('-') < 0) { + return initial; + } + StringBuilder buf = new StringBuilder(); + boolean upcase = false; + for (char ch : initial.toCharArray()) { + if (ch == '-') { + upcase = true; + } else if (upcase && ch >= 'a' && ch <= 'z') { + buf.append((char)('A' + ch - 'a')); + upcase = false; + } else { + buf.append(ch); + upcase = false; + } + } + return buf.toString(); + } + + /** + * Parse leaf value in an xml tree + */ + private void parseLeaf(Element element, ConfigPayloadBuilder payloadBuilder, String parentName) { + String name = extractName(element); + String value = XML.getValue(element); + if (value == null) { + throw new ConfigurationRuntimeException("Element '" + name + "' must have either children or a value"); + } + + + if (element.hasAttribute("index")) { + // Check for legacy (pre Vespa 6) usage + throw new IllegalArgumentException("The 'index' attribute on config elements is not supported - use <item>"); + } else if (element.hasAttribute("operation")) { + // leaf array, currently the only supported operation is 'append' + verifyLegalOperation(element); + ConfigPayloadBuilder.Array a = payloadBuilder.getArray(name); + a.append(value); + } else if ("item".equals(name)) { + if (parentName == null) + throw new ConfigurationRuntimeException("<item> is a reserved keyword for array and map elements"); + if (element.hasAttribute("key")) { + payloadBuilder.getMap(parentName).put(element.getAttribute("key"), value); + } else { + payloadBuilder.getArray(parentName).append(value); + } + } else { + // leaf scalar, e.g. <intVal>3</intVal> + payloadBuilder.setField(name, value); + } + } + + private void parseComplex(Element element, List<Element> children, ConfigPayloadBuilder payloadBuilder, String parentName) { + String name = extractName(element); + // Inner value + if (element.hasAttribute("index")) { + // Check for legacy (pre Vespa 6) usage + throw new IllegalArgumentException("The 'index' attribute on config elements is not supported - use <item>"); + } else if (element.hasAttribute("operation")) { + // inner array, currently the only supported operation is 'append' + verifyLegalOperation(element); + ConfigPayloadBuilder childPayloadBuilder = payloadBuilder.getArray(name).append(); + //Cursor array = node.setArray(name); + for (Element child : children) { + //Cursor struct = array.addObject(); + parseElement(child, childPayloadBuilder, name); + } + } else if ("item".equals(name)) { + // Reserved item means array/map element as struct + if (element.hasAttribute("key")) { + ConfigPayloadBuilder childPayloadBuilder = payloadBuilder.getMap(parentName).get(element.getAttribute("key")); + for (Element child : children) { + parseElement(child, childPayloadBuilder, parentName); + } + } else { + ConfigPayloadBuilder.Array array = payloadBuilder.getArray(parentName); + ConfigPayloadBuilder childPayloadBuilder = array.append(); + for (Element child : children) { + parseElement(child, childPayloadBuilder, parentName); + } + } + } else { + int numMatching = 0; + for (Element child : children) { + numMatching += ("item".equals(child.getTagName())) ? 1 : 0; + } + + if (numMatching == 0) { + // struct, e.g. <basicStruct> + ConfigPayloadBuilder p = payloadBuilder.getObject(name); + //Cursor struct = node.setObject(name); + for (Element child : children) + parseElement(child, p, name); + } else if (numMatching == children.size()) { + // Array with <item elements> + for (Element child : children) { + parseElement(child, payloadBuilder, name); + } + } else { + throw new ConfigurationRuntimeException("<item> is a reserved keyword for array and map elements"); + } + } + } + + /** + * Adds the values and children (recursively) in the given xml element to the given {@link ConfigPayloadBuilder}. + * @param currElem The element representing a config parameter. + * @param payloadBuilder The builder to use when adding elements. + */ + private void parseElement(Element currElem, ConfigPayloadBuilder payloadBuilder, String parentName) { + List<Element> children = XML.getChildren(currElem); + try { + if (children.isEmpty()) { + parseLeaf(currElem, payloadBuilder, parentName); + } else { + parseComplex(currElem, children, payloadBuilder, parentName); + } + } catch (Exception exception) { + throw new ConfigurationRuntimeException("Error parsing element at " + XML.getNodePath(currElem, " > ") + ": " + + Exceptions.toMessageString(exception)); + } + } + + private void verifyLegalOperation(Element currElem) { + String operation = currElem.getAttribute("operation"); + if (! operation.equalsIgnoreCase("append")) + throw new ConfigurationRuntimeException("The only supported array operation is 'append', got '" + + operation + "' at XML node '" + XML.getNodePath(currElem, " > ") + "'."); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomContainerClusterBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomContainerClusterBuilder.java new file mode 100644 index 00000000000..fe339835c78 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomContainerClusterBuilder.java @@ -0,0 +1,109 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.text.XML; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.Handler; +import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; +import com.yahoo.vespa.model.container.search.searchchain.SearchChains; +import org.w3c.dom.Element; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author gjoranv + */ +public abstract class DomContainerClusterBuilder<CLUSTER extends ContainerCluster> + extends VespaDomBuilder.DomConfigProducerBuilder<CLUSTER> { + + protected final Element outerChainsElem; + + public DomContainerClusterBuilder(Element outerChainsElem) { + this.outerChainsElem = outerChainsElem; + } + + protected void buildAndAddUserConfiguredComponents(ContainerCluster cluster, Element spec) { + buildAndAddConfiguredHandlers(cluster, spec); + buildAndAddClientProviders(cluster, spec); + buildAndAddServerProviders(cluster, spec); + buildAndAddGenericComponents(cluster, spec); + buildAndAddRenderers(cluster, spec); + buildAndAddFilters(cluster, spec); + } + + public void addSpecialHandlers(ContainerCluster cluster) { + ContainerModelBuilder.addDefaultHandler_legacyBuilder(cluster); + } + + private void buildAndAddConfiguredHandlers(ContainerCluster cluster, Element spec) { + List<Handler> handlers = buildConfiguredHandlers(new DomHandlerBuilder(true), cluster, spec, "handler"); + + for (Handler handler : handlers) { + // TODO: hack to avoid adding a simple Handler for an explicitly declared SearchHandler + if (handler.getClassId().getName().equals("com.yahoo.search.handler.SearchHandler")) { + final ProcessingHandler<SearchChains> searchHandler = new ProcessingHandler<>( + cluster.getSearch().getChains(), "com.yahoo.search.handler.SearchHandler"); + searchHandler.addServerBindings("http://*/search/*"); + cluster.addComponent(searchHandler); + } else + cluster.addComponent(handler); + } + } + + private void buildAndAddClientProviders(ContainerCluster cluster, Element spec) { + List<Handler> clients = buildConfiguredHandlers(new DomClientProviderBuilder(), cluster, spec, "client"); + + for (Handler client : clients) { + cluster.addComponent(client); + } + } + + private void buildAndAddServerProviders(ContainerCluster cluster, Element spec) { + ContainerModelBuilder.addConfiguredComponents(cluster, spec, "server"); + } + + private void buildAndAddGenericComponents(ContainerCluster cluster, Element spec) { + ContainerModelBuilder.addConfiguredComponents(cluster, spec, DomComponentBuilder.elementName); + } + + private void buildAndAddFilters(ContainerCluster cluster, Element spec) { + for (Component component : buildConfiguredFilters(cluster, spec, "filter")) { + cluster.addComponent(component); + } + } + + private List<Component> buildConfiguredFilters(AbstractConfigProducer ancestor, + Element spec, + String componentName) { + List<Component> components = new ArrayList<>(); + + for (Element node : XML.getChildren(spec, componentName)) { + components.add(new DomFilterBuilder().build(ancestor, node)); + } + return components; + } + + private List<Handler> buildConfiguredHandlers(DomHandlerBuilder builder, + AbstractConfigProducer ancestor, + Element spec, + String componentName) { + List<Handler> handlers = new ArrayList<>(); + + for (Element node : XML.getChildren(spec, componentName)) { + handlers.add(builder.build(ancestor, node)); + } + return handlers; + } + + protected void buildAndAddRenderers(ContainerCluster cluster, Element spec) { + ContainerModelBuilder.addConfiguredComponents(cluster, spec, "renderer"); + } + + protected void buildAndAddProcessingRenderers(ContainerCluster cluster, Element spec) { + ContainerModelBuilder.addConfiguredComponents(cluster, spec, "renderer"); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomContentBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomContentBuilder.java new file mode 100644 index 00000000000..930e7105c2f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomContentBuilder.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.config.model.builder.xml.ConfigModelId; +import com.yahoo.vespa.model.admin.Admin; +import com.yahoo.vespa.model.content.Content; +import com.yahoo.vespa.model.content.cluster.ContentCluster; + +import org.w3c.dom.Element; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * @author balder + */ +public class DomContentBuilder extends ConfigModelBuilder<Content> { + + public static final List<ConfigModelId> configModelIds = Collections.singletonList(ConfigModelId.fromName("content")); + + public DomContentBuilder() { + super(Content.class); + } + + @Override + public List<ConfigModelId> handlesElements() { + return configModelIds; + } + + @Override + public void doBuild(Content content, Element xml, ConfigModelContext modelContext) { + Admin admin = content.adminModel() != null ? content.adminModel().getAdmin() : null; // This is null in tests only + ContentCluster cluster = new ContentCluster.Builder(admin, modelContext.getDeployLogger()).build(modelContext.getParentProducer(), xml); + content.setCluster(cluster, modelContext); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java new file mode 100644 index 00000000000..434c57fdcb3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFileDistributionOptionsBuilder.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.binaryprefix.BinaryScaledAmount; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.admin.FileDistributionOptions; +import org.w3c.dom.Element; + +/** + * Builds a file distribution options. + * @author tonytv + */ +public class DomFileDistributionOptionsBuilder { + private static void throwExceptionForElementInFileDistribution(String subElement, String reason) { + throw new RuntimeException("In element '" + subElement + "' contained in 'filedistribution': " + reason); + } + + private static void callSetter(FileDistributionOptions options, String name, BinaryScaledAmount amount) { + try { + options.getClass().getMethod(name, BinaryScaledAmountParser.class).invoke(options, amount); + } catch (IllegalArgumentException e) { + throwExceptionForElementInFileDistribution(name, e.getMessage()); + } + catch (Exception e) { + if (e instanceof RuntimeException) + throw (RuntimeException)e; + else + throw new RuntimeException(e); + } + } + + private static void setIfPresent(FileDistributionOptions options, String name, Element fileDistributionElement) { + try { + Element optionElement = XML.getChild(fileDistributionElement, name); + if (optionElement != null) { + String valueString = XML.getValue(optionElement); + BinaryScaledAmount amount = BinaryScaledAmountParser.parse(valueString); + callSetter(options, name, amount); + } + } catch (NumberFormatException e) { + throwExceptionForElementInFileDistribution(name, "Expected a valid number. (Message = " + e.getMessage() + ")."); + } + } + + public FileDistributionOptions build(Element fileDistributionElement) { + FileDistributionOptions options = FileDistributionOptions.defaultOptions(); + if (fileDistributionElement != null) { + setIfPresent(options, "uploadbitrate", fileDistributionElement); + setIfPresent(options, "downloadbitrate", fileDistributionElement); + } + return options; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFilterBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFilterBuilder.java new file mode 100644 index 00000000000..741b6f99944 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomFilterBuilder.java @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.HttpFilter; +import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder; +import org.w3c.dom.Element; + +/** + * @author tonytv + */ +public class DomFilterBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Component> { + @Override + protected Component doBuild(AbstractConfigProducer ancestor, Element element) { + return new HttpFilter(BundleInstantiationSpecificationBuilder.build(element, false)); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java new file mode 100644 index 00000000000..39a3ab553d2 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.text.XML; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.Handler; +import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder; +import org.w3c.dom.Element; + +/** + * @author gjoranv + * @since 5.1.6 + */ +public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Handler> { + private final boolean legacyMode; + + public DomHandlerBuilder(boolean legacyMode) { + this.legacyMode = legacyMode; + } + + public DomHandlerBuilder() { + this(false); + } + + @Override + protected Handler doBuild(AbstractConfigProducer ancestor, Element handlerElement) { + Handler<? super Component<?, ?>> handler = getHandler(handlerElement); + + for (Element binding : XML.getChildren(handlerElement, "binding")) + handler.addServerBindings(XML.getValue(binding)); + + for (Element clientBinding : XML.getChildren(handlerElement, "clientBinding")) + handler.addClientBindings(XML.getValue(clientBinding)); + + DomComponentBuilder.addChildren(ancestor, handlerElement, handler); + + return handler; + } + + protected Handler<? super Component<?, ?>> getHandler(Element handlerElement) { + BundleInstantiationSpecification bundleSpec = BundleInstantiationSpecificationBuilder.build(handlerElement, legacyMode); + return new Handler<>( + new ComponentModel(bundleSpec)); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomMetricBuilderHelper.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomMetricBuilderHelper.java new file mode 100644 index 00000000000..49d475ec057 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomMetricBuilderHelper.java @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.text.XML; +import com.yahoo.vespa.model.admin.Metric; +import com.yahoo.vespa.model.admin.MetricsConsumer; +import org.w3c.dom.Element; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Helper class for parsing yamasmetric config. + * + * @author trygve + * @since 5.1 + */ +public class DomMetricBuilderHelper { + + + /** + * Build metricConsumer config + * + * @param spec xml element + * @return a map from metric name to a {@link MetricsConsumer} + */ + protected static Map<String, MetricsConsumer> buildMetricsConsumers(Element spec) { + Map<String, MetricsConsumer> metricsConsumers = new LinkedHashMap<>(); + List<Element> consumersElem = XML.getChildren(spec, "consumer"); + for (Element consumer : consumersElem) { + String consumerName = consumer.getAttribute("name"); + Map<String, Metric> metrics = new LinkedHashMap<>(); + List<Element> metricsEl = XML.getChildren(consumer, "metric"); + if (metricsEl != null) { + for (Element metric : metricsEl) { + String metricName = metric.getAttribute("name"); + String outputName = metric.getAttribute("output-name"); + metrics.put(metricName, new Metric(metricName, outputName)); + } + } + MetricsConsumer metricsConsumer = new MetricsConsumer(consumerName, metrics); + metricsConsumers.put(consumerName, metricsConsumer); + } + return metricsConsumers; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomRoutingBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomRoutingBuilder.java new file mode 100644 index 00000000000..dad00f529fb --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomRoutingBuilder.java @@ -0,0 +1,138 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.config.application.Xml; +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.config.model.builder.xml.ConfigModelId; +import com.yahoo.messagebus.routing.*; +import com.yahoo.text.XML; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.vespa.model.routing.Routing; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import java.util.Arrays; +import java.util.List; + +/** + * Builds the Routing plugin + * + * @author vegardh + */ +public class DomRoutingBuilder extends ConfigModelBuilder<Routing> { + + public DomRoutingBuilder() { + super(Routing.class); + } + + @Override + public List<ConfigModelId> handlesElements() { + return Arrays.asList(ConfigModelId.fromName("routing")); + } + + // Overrides ConfigModelBuilder. + @Override + public void doBuild(Routing plugin, Element spec, ConfigModelContext modelContext) { + ApplicationSpec app = null; + RoutingSpec routing = null; + if (spec != null) { + app = new ApplicationSpec(); + for (Element node : Xml.mergeElems(spec, "services", modelContext.getApplicationPackage(), ApplicationPackage.ROUTINGTABLES_DIR)) { + addServices(app, node); + } + routing = new RoutingSpec(); + for (Element node : Xml.mergeElems(spec, "routingtable", modelContext.getApplicationPackage(), ApplicationPackage.ROUTINGTABLES_DIR)) { + addRoutingTable(routing, node); + } + } + plugin.setExplicitApplicationSpec(app); + plugin.setExplicitRoutingSpec(routing); + } + + /** + * This function updates the given application with the data contained in the given xml element. + * + * @param app The application to update. + * @param element The element to base the services on. + */ + private static void addServices(ApplicationSpec app, Element element) { + String protocol = element.getAttribute("protocol"); + for (Element node : XML.getChildren(element, "service")) { + app.addService(protocol, node.getAttribute("name")); + } + } + + /** + * This function updates the given routing spec with the data contained in the given xml element. + * + * @param routing The routing spec to update. + * @param element The element to base the route config on. + */ + private static void addRoutingTable(RoutingSpec routing, Element element) { + boolean verify = element.hasAttribute("verify") ? Boolean.valueOf(element.getAttribute("verify")) : true; + RoutingTableSpec table = new RoutingTableSpec(element.getAttribute("protocol"), verify); + + NodeList children = element.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + Node node = children.item(i); + if ("hop".equals(node.getNodeName())) { + table.addHop(createHopSpec((Element)node)); + } else if ("route".equals(node.getNodeName())) { + table.addRoute(createRouteSpec((Element)node)); + } + } + + routing.addTable(table); + } + + /** + * This function creates a route from the content of the given xml element. + * + * @param element The element to base the route config on. + * @return The corresponding route spec. + */ + private static RouteSpec createRouteSpec(Element element) { + boolean verify = element.hasAttribute("verify") ? Boolean.valueOf(element.getAttribute("verify")) : true; + RouteSpec route = new RouteSpec(element.getAttribute("name"), verify); + String hops = element.getAttribute("hops"); + int from = 0; + for (int to = 0, depth = 0, len = hops.length(); to < len; ++to) { + if (hops.charAt(to) == '[') { + ++depth; + } else if (hops.charAt(to) == ']') { + --depth; + } else if (hops.charAt(to) == ' ' && depth == 0) { + if (to > from) { + route.addHop(hops.substring(from, to)); + } + from = to + 1; + } + } + if (from < hops.length()) { + route.addHop(hops.substring(from)); + } + return route; + } + + /** + * This function creates a hop from the content of the given xml element. + * + * @param element The element to base the hop config on. + * @return The corresponding hop spec. + */ + private static HopSpec createHopSpec(Element element) { + boolean verify = element.hasAttribute("verify") ? Boolean.valueOf(element.getAttribute("verify")) : true; + HopSpec hop = new HopSpec(element.getAttribute("name"), element.getAttribute("selector"), verify); + if (Boolean.valueOf(element.getAttribute("ignore-result"))) { + hop.setIgnoreResult(true); + } + NodeList children = element.getElementsByTagName("recipient"); + for (int i = 0; i < children.getLength(); i++) { + Element node = (Element)children.item(i); + hop.addRecipient(node.getAttribute("session")); + } + return hop; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java new file mode 100644 index 00000000000..aa2dfd84eb5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java @@ -0,0 +1,247 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.text.XML; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.search.Tuning; +import org.w3c.dom.Element; + +/** + * Builder for the tuning config for a search cluster. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Tuning> { + + @Override + protected Tuning doBuild(AbstractConfigProducer parent, Element spec) { + Tuning tuning = new Tuning(parent); + for (Element e : XML.getChildren(spec)) { + if (equals("dispatch", e)) { + handleDispatch(e, tuning); + } else if (equals("searchnode", e)) { + handleSearchNode(e, tuning); + } + } + return tuning; + } + + private static boolean equals(String name, Element e) { + return name.equals(e.getNodeName()); + } + + private static String asString(Element e) { + return e.getFirstChild().getNodeValue(); + } + + private static Long asLong(Element e) { + return Long.parseLong(e.getFirstChild().getNodeValue()); + } + + private static Integer asInt(Element e) { + return Integer.parseInt(e.getFirstChild().getNodeValue()); + } + + private static Double asDouble(Element e) { + return Double.parseDouble(e.getFirstChild().getNodeValue()); + } + + private void handleDispatch(Element spec, Tuning t) { + t.dispatch = new Tuning.Dispatch(); + for (Element e : XML.getChildren(spec)) { + if (equals("max-hits-per-partition", e)) { + t.dispatch.maxHitsPerPartition = asInt(e); + } + } + } + + private void handleSearchNode(Element spec, Tuning t) { + t.searchNode = new Tuning.SearchNode(); + for (Element e : XML.getChildren(spec)) { + if (equals("requestthreads", e)) { + handleRequestThreads(e, t.searchNode); + } else if (equals("flushstrategy", e)) { + handleFlushStrategy(e, t.searchNode); + } else if (equals("resizing", e)) { + handleResizing(e, t.searchNode); + } else if (equals("index", e)) { + handleIndex(e, t.searchNode); + } else if (equals("attribute", e)) { + handleAttribute(e, t.searchNode); + } else if (equals("summary", e)) { + handleSummary(e, t.searchNode); + } + } + } + + private void handleRequestThreads(Element spec, Tuning.SearchNode sn) { + sn.threads = new Tuning.SearchNode.RequestThreads(); + Tuning.SearchNode.RequestThreads rt = sn.threads; + for (Element e : XML.getChildren(spec)) { + if (equals("search", e)) { + rt.numSearchThreads = asInt(e); + } else if (equals("persearch", e)) { + rt.numThreadsPerSearch = asInt(e); + } else if (equals("summary", e)) { + rt.numSummaryThreads = asInt(e); + } + } + } + + private void handleFlushStrategy(Element spec, Tuning.SearchNode sn) { + for (Element e : XML.getChildren(spec)) { + if (equals("native", e)) { + handleNativeStrategy(e, sn); + } + } + } + + private void handleNativeStrategy(Element spec, Tuning.SearchNode sn) { + sn.strategy = new Tuning.SearchNode.FlushStrategy(); + Tuning.SearchNode.FlushStrategy fs = sn.strategy; + for (Element e : XML.getChildren(spec)) { + if (equals("total", e)) { + for (Element e2 : XML.getChildren(e)) { + if (equals("maxmemorygain", e2)) { + fs.totalMaxMemoryGain = asLong(e2); + } else if (equals("diskbloatfactor", e2)) { + fs.totalDiskBloatFactor = asDouble(e2); + } + } + } else if (equals("component", e)) { + for (Element e2 : XML.getChildren(e)) { + if (equals("maxmemorygain", e2)) { + fs.componentMaxMemoryGain = asLong(e2); + } else if (equals("diskbloatfactor", e2)) { + fs.componentDiskBloatFactor = asDouble(e2); + } else if (equals("maxage", e2)) { + fs.componentMaxage = asDouble(e2); + } + } + } else if (equals("transactionlog", e)) { + for (Element e2 : XML.getChildren(e)) { + if (equals("maxentries", e2)) { + fs.transactionLogMaxEntries = asLong(e2); + } else if (equals("maxsize", e2)) { + fs.transactionLogMaxSize = asLong(e2); + } + } + } + } + } + + private void handleResizing(Element spec, Tuning.SearchNode sn) { + sn.resizing = new Tuning.SearchNode.Resizing(); + for (Element e : XML.getChildren(spec)) { + if (equals("initialdocumentcount", e)) { + sn.resizing.initialDocumentCount = asInt(e); + } + } + } + + private void handleIndex(Element spec, Tuning.SearchNode sn) { + sn.index = new Tuning.SearchNode.Index(); + for (Element e : XML.getChildren(spec)) { + if (equals("io", e)) { + sn.index.io = new Tuning.SearchNode.Index.Io(); + Tuning.SearchNode.Index.Io io = sn.index.io; + for (Element e2 : XML.getChildren(e)) { + if (equals("write", e2)) { + io.write = Tuning.SearchNode.IoType.fromString(asString(e2)); + } else if (equals("read", e2)) { + io.read = Tuning.SearchNode.IoType.fromString(asString(e2)); + } else if (equals("search", e2)) { + io.search = Tuning.SearchNode.IoType.fromString(asString(e2)); + } + } + } + } + } + + private void handleAttribute(Element spec, Tuning.SearchNode sn) { + sn.attribute = new Tuning.SearchNode.Attribute(); + for (Element e : XML.getChildren(spec)) { + if (equals("io", e)) { + sn.attribute.io = new Tuning.SearchNode.Attribute.Io(); + for (Element e2 : XML.getChildren(e)) { + if (equals("write", e2)) { + sn.attribute.io.write = Tuning.SearchNode.IoType.fromString(asString(e2)); + } + } + } + } + } + + private void handleSummary(Element spec, Tuning.SearchNode sn) { + sn.summary = new Tuning.SearchNode.Summary(); + for (Element e : XML.getChildren(spec)) { + if (equals("io", e)) { + sn.summary.io = new Tuning.SearchNode.Summary.Io(); + for (Element e2 : XML.getChildren(e)) { + if (equals("write", e2)) { + sn.summary.io.write = Tuning.SearchNode.IoType.fromString(asString(e2)); + } else if (equals("read", e2)) { + sn.summary.io.read = Tuning.SearchNode.IoType.fromString(asString(e2)); + } + } + } else if (equals("store", e)) { + handleSummaryStore(e, sn.summary); + } + } + } + + private void handleSummaryStore(Element spec, Tuning.SearchNode.Summary s) { + s.store = new Tuning.SearchNode.Summary.Store(); + for (Element e : XML.getChildren(spec)) { + if (equals("cache", e)) { + s.store.cache = new Tuning.SearchNode.Summary.Store.Component(); + handleSummaryStoreComponent(e, s.store.cache); + } else if (equals("logstore", e)) { + handleSummaryLogStore(e, s.store); + } + } + } + + private void handleSummaryStoreComponent(Element spec, Tuning.SearchNode.Summary.Store.Component c) { + for (Element e : XML.getChildren(spec)) { + if (equals("maxsize", e)) { + c.maxSize = asLong(e); + } else if (equals("maxentries", e)) { + c.maxEntries = asLong(e); + } else if (equals("initialentries", e)) { + c.initialEntries = asLong(e); + } else if (equals("compression", e)) { + c.compression = new Tuning.SearchNode.Summary.Store.Compression(); + handleSummaryStoreCompression(e, c.compression); + } + } + } + + private void handleSummaryStoreCompression(Element spec, Tuning.SearchNode.Summary.Store.Compression c) { + for (Element e : XML.getChildren(spec)) { + if (equals("type", e)) { + c.type = Tuning.SearchNode.Summary.Store.Compression.Type.fromString(asString(e)); + } else if (equals("level", e)) { + c.level = asInt(e); + } + } + } + + private void handleSummaryLogStore(Element spec, Tuning.SearchNode.Summary.Store s) { + s.logStore = new Tuning.SearchNode.Summary.Store.LogStore(); + for (Element e : XML.getChildren(spec)) { + if (equals("maxfilesize", e)) { + s.logStore.maxFileSize = asLong(e); + } else if (equals("maxdiskbloatfactor", e)) { + s.logStore.maxDiskBloatFactor = asDouble(e); + } else if (equals("minfilesizefactor", e)) { + s.logStore.minFileSizeFactor = asDouble(e); + } else if (equals("numthreads", e)) { + s.logStore.numThreads = asInt(e); + } else if (equals("chunk", e)) { + s.logStore.chunk = new Tuning.SearchNode.Summary.Store.Component(true); + handleSummaryStoreComponent(e, s.logStore.chunk); + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java new file mode 100644 index 00000000000..65519637bd5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java @@ -0,0 +1,576 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.Phase; +import com.yahoo.component.chain.dependencies.Dependencies; +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.component.chain.model.ChainedComponentModel; +import com.yahoo.config.model.ConfigModelUtils; +import com.yahoo.vespa.config.content.spooler.SpoolerConfig; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.text.XML; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.SimpleConfigProducer; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder.DomConfigProducerBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.docproc.DomDocprocChainsBuilder; +import com.yahoo.vespa.model.clients.Clients; +import com.yahoo.vespa.model.clients.HttpGatewayOwner; +import com.yahoo.vespa.model.clients.VespaSpoolMaster; +import com.yahoo.vespa.model.clients.VespaSpooler; +import com.yahoo.vespa.model.clients.VespaSpoolerProducer; +import com.yahoo.vespa.model.clients.VespaSpoolerService; +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.Handler; +import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; +import com.yahoo.vespa.model.container.docproc.ContainerDocproc; +import com.yahoo.vespa.model.container.docproc.DocprocChains; +import com.yahoo.vespa.model.container.search.ContainerHttpGateway; +import com.yahoo.vespa.model.container.search.ContainerSearch; +import com.yahoo.vespa.model.container.search.searchchain.SearchChain; +import com.yahoo.vespa.model.container.search.searchchain.SearchChains; +import com.yahoo.vespa.model.container.search.searchchain.Searcher; +import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; +import com.yahoo.vespaclient.config.FeederConfig; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.logging.Level; + +/** + * Builds the Clients plugin + * + * @author vegardh + */ +public class DomV20ClientsBuilder { + + public static final String vespaClientBundleSpecification = "vespaclient-container-plugin"; + + // The parent docproc plugin to register data with. + private final Clients clients; + + DomV20ClientsBuilder(Clients clients, String version) { + this.clients = clients; + if (!version.equals("2.0")) { + throw new IllegalArgumentException("Version '" + version + "' of 'clients' not supported."); + } + } + + public void build(Element spec) { + NodeList children = spec.getElementsByTagName("gateways"); + if (children.getLength() > 0 && clients.getConfigProducer()!=null) + clients.getConfigProducer().deployLogger().log(Level.WARNING, "The 'gateways' element is deprecated, and will be disallowed in a " + + "later version of Vespa. Use 'document-api' under 'jdisc' instead, see: " + + ConfigModelUtils.createDocLink("reference/services-jdisc.html")); + for (int i = 0; i < children.getLength(); i++) { + createGateways(clients.getConfigProducer(), (Element) children.item(i), clients); + } + + children = spec.getElementsByTagName("spoolers"); + for (int i = 0; i < children.getLength(); i++) { + createSpoolers(clients.getConfigProducer(), (Element) children.item(i), clients); + } + + children = spec.getElementsByTagName("load-types"); + for (int i = 0; i < children.getLength(); i++) { + createLoadTypes((Element) children.item(i), clients); + } + } + + static Boolean getBooleanNodeValue(Node node) { + return Boolean.valueOf(node.getFirstChild().getNodeValue()); + } + + static boolean getHttpFileServerEnabled(Element parentHttpFileServer, Element httpFileServer) { + boolean ret=false; + if (parentHttpFileServer != null) { + for (Element child : XML.getChildren(parentHttpFileServer)) { + if ("enabled".equals(child.getNodeName())) { + ret = getBooleanNodeValue(child); + } + } + } + if (httpFileServer != null) { + for (Element child : XML.getChildren(httpFileServer)) { + if ("enabled".equals(child.getNodeName())) { + ret = getBooleanNodeValue(child); + } + } + } + return ret; + } + + static String getHttpFileServerRootDir(Element parentHttpFileServer, Element httpFileServer) { + String ret=""; + if (parentHttpFileServer != null) { + for (Element child : XML.getChildren(parentHttpFileServer)) { + if ("rootdir".equals(child.getNodeName())) { + ret = child.getFirstChild().getNodeValue(); + } + } + } + if (httpFileServer != null) { + for (Element child : XML.getChildren(httpFileServer)) { + if ("rootdir".equals(child.getNodeName())) { + ret = child.getFirstChild().getNodeValue(); + } + } + } + return ret; + } + + private void createLoadTypes(Element element, Clients clients) { + for (Element e : XML.getChildren(element, "type")) { + String priority = e.getAttribute("default-priority"); + clients.getLoadTypes().addType(e.getAttribute("name"), priority.length() > 0 ? priority : null); + } + } + + /** + * Creates HttpGateway objects using the given xml Element. + * + * @param pcp AbstractConfigProducer + * @param element The xml Element + */ + private void createGateways(AbstractConfigProducer pcp, Element element, Clients clients) { + String jvmArgs = null; + if (element.hasAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME)) jvmArgs=element.getAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME); + + Element gatewaysFeederOptions = findFeederOptions(element); + + HttpGatewayOwner owner = new HttpGatewayOwner(pcp, getFeederConfig(null, gatewaysFeederOptions)); + ContainerCluster cluster = new ContainerHttpGatewayClusterBuilder().build(owner, element); + + int index = 0; + for (Element e : XML.getChildren(element, "gateway")) { + ContainerHttpGateway qrs = new ContainerHttpGatewayBuilder(cluster, index).build(cluster, e); + + if ("".equals(qrs.getJvmArgs()) && jvmArgs!=null) qrs.setJvmArgs(jvmArgs); + index++; + } + clients.setContainerHttpGateways(cluster); + } + + /** + * Creates VespaSpooler objects using the given xml Element. + */ + private void createSpoolers(AbstractConfigProducer pcp, Element element, Clients clients) { + String jvmArgs = null; + if (element.hasAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME)) jvmArgs=element.getAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME); + SimpleConfigProducer spoolerCfg = new VespaDomBuilder.DomSimpleConfigProducerBuilder(element.getNodeName()). + build(pcp, element); + Element spoolersFeederOptions = findFeederOptions(element); + createSpoolMasters(spoolerCfg, element); + for (Element e : XML.getChildren(element, "spooler")) { + String configId = e.getAttribute("id").trim(); + FeederConfig.Builder feederConfig = getFeederConfig(spoolersFeederOptions, e); + SpoolerConfig.Builder spoolConfig = getSpoolConfig(e); + if (configId.length() == 0) { + int index = clients.getVespaSpoolers().size(); + VespaSpoolerService spoolerService = new VespaSpoolerServiceBuilder(index, new VespaSpooler(feederConfig, spoolConfig)). + build(spoolerCfg, e); + if ("".equals(spoolerService.getJvmArgs()) && jvmArgs!=null) spoolerService.setJvmArgs(jvmArgs); + spoolerService.setProp("index", String.valueOf(index)); + clients.getVespaSpoolers().add(spoolerService); + } else { + new VespaSpoolerProducerBuilder(configId, new VespaSpooler(feederConfig, spoolConfig)). + build(spoolerCfg, e); + } + } + } + + private void createSpoolMasters(SimpleConfigProducer producer, + Element element) { + int i=0; + for (Element e : XML.getChildren(element, "spoolmaster")) { + VespaSpoolMaster master = new VespaSpoolMasterBuilder(i).build(producer, e); + i++; + } + } + + private SpoolerConfig.Builder getSpoolConfig(Element conf) { + SpoolerConfig.Builder builder = new SpoolerConfig.Builder(); + if (conf.getAttributes().getNamedItem("directory") != null) { + builder.directory(Defaults.getDefaults().underVespaHome(conf.getAttributes().getNamedItem("directory").getNodeValue())); + } + if (conf.getAttributes().getNamedItem("keepsuccess") != null) { + builder.keepsuccess(getBooleanFromAttribute(conf, "keepsuccess")); + } + if (conf.getAttributes().getNamedItem("maxfailuresize") != null) { + builder.maxfailuresize(getIntegerFromAttribute(conf, "maxfailuresize")); + } + if (conf.getAttributes().getNamedItem("maxfatalfailuresize") != null) { + builder.maxfatalfailuresize(getIntegerFromAttribute(conf, "maxfatalfailuresize")); + } + if (conf.getAttributes().getNamedItem("threads") != null) { + builder.threads(getIntegerFromAttribute(conf, "threads")); + } + if (conf.getAttributes().getNamedItem("maxretries") != null) { + builder.maxretries(getIntegerFromAttribute(conf, "maxretries")); + } + + NodeList children = conf.getElementsByTagName("parsers"); + if (children.getLength() == 1) { + children = ((Element)children.item(0)).getElementsByTagName("parser"); + } + + for (int i=0; i < children.getLength(); i++) { + Element e = (Element)children.item(i); + + String type = e.getAttributes().getNamedItem("type").getNodeValue(); + NodeList params = e.getElementsByTagName("parameter"); + + SpoolerConfig.Parsers.Builder parserBuilder = new SpoolerConfig.Parsers.Builder(); + parserBuilder.classname(type); + if (params.getLength() > 0) { + List<SpoolerConfig.Parsers.Parameters.Builder> parametersBuilders = new ArrayList<>(); + for (int j = 0; j < params.getLength(); j++) { + SpoolerConfig.Parsers.Parameters.Builder parametersBuilder = new SpoolerConfig.Parsers.Parameters.Builder(); + Element p = (Element) params.item(j); + parametersBuilder.key(getStringFromAttribute(p, "key")); + parametersBuilder.value(getStringFromAttribute(p, "value")); + parametersBuilders.add(parametersBuilder); + } + parserBuilder.parameters(parametersBuilders); + } + + builder.parsers.add(parserBuilder); + } + return builder; + } + + Boolean getBooleanFromAttribute(Element e, String attributeName) { + return Boolean.parseBoolean(e.getAttributes().getNamedItem(attributeName).getNodeValue()); + } + + Integer getIntegerFromAttribute(Element e, String attributeName) { + return Integer.parseInt(e.getAttributes().getNamedItem(attributeName).getNodeValue()); + } + + String getStringFromAttribute(Element e, String attributeName) { + return e.getAttributes().getNamedItem(attributeName).getNodeValue(); + } + + private FeederConfig.Builder getFeederConfig(Element gatewaysFeederOptions, Element e) { + FeederOptionsParser foParser = new FeederOptionsParser(); + if (gatewaysFeederOptions!=null) { + foParser.parseFeederOptions(gatewaysFeederOptions).getFeederConfig(); + } + foParser.parseFeederOptions(e); + return foParser.getFeederConfig(); + } + + /** + * Finds the feederoptions subelement in the given xml Element. + * + * @param element The xml Element + * @return The feederoptions xml Element + */ + private Element findFeederOptions(Element element) { + for (Element child : XML.getChildren(element)) { + if (child.getNodeName().equals("feederoptions")) { + return child; + } + } + return null; + } + + private static class VespaSpoolerServiceBuilder extends DomConfigProducerBuilder<VespaSpoolerService> { + private int index; + private VespaSpooler spoolerConfig; + + public VespaSpoolerServiceBuilder(int index, VespaSpooler spoolerConfig) { + this.index = index; + this.spoolerConfig = spoolerConfig; + } + + @Override + protected VespaSpoolerService doBuild(AbstractConfigProducer parent, + Element spec) { + return new VespaSpoolerService(parent, index, spoolerConfig); + } + } + + private static class VespaSpoolerProducerBuilder extends DomConfigProducerBuilder<VespaSpoolerProducer> { + private String name=null; + private VespaSpooler spooler; + + public VespaSpoolerProducerBuilder(String name, VespaSpooler spooler) { + this.name = name; + this.spooler = spooler; + } + + @Override + protected VespaSpoolerProducer doBuild(AbstractConfigProducer parent, + Element producerSpec) { + return new VespaSpoolerProducer(parent, name, spooler); + } + } + + private static class VespaSpoolMasterBuilder extends DomConfigProducerBuilder<VespaSpoolMaster> { + int index; + + public VespaSpoolMasterBuilder(int index) { + super(); + this.index = index; + } + + @Override + protected VespaSpoolMaster doBuild(AbstractConfigProducer parent, + Element spec) { + return new VespaSpoolMaster(parent, index); + } + } + + public static class ContainerHttpGatewayClusterBuilder extends DomConfigProducerBuilder<ContainerCluster> { + @Override + protected ContainerCluster doBuild(AbstractConfigProducer parent, + Element spec) { + + ContainerCluster cluster = new ContainerCluster(parent, "gateway", "gateway"); + + SearchChains searchChains = new SearchChains(cluster, "searchchain"); + Set<ComponentSpecification> inherited = new TreeSet<>(); + //inherited.add(new ComponentSpecification("vespa", null, null)); + { + SearchChain mySearchChain = new SearchChain(new ChainSpecification(new ComponentId("vespaget"), + new ChainSpecification.Inheritance(inherited, null), new ArrayList<>(), new TreeSet<>())); + Searcher getComponent = newVespaClientSearcher("com.yahoo.storage.searcher.GetSearcher"); + mySearchChain.addInnerComponent(getComponent); + searchChains.add(mySearchChain); + } + { + SearchChain mySearchChain = new SearchChain(new ChainSpecification(new ComponentId("vespavisit"), + new ChainSpecification.Inheritance(inherited, null), new ArrayList<>(), new TreeSet<>())); + Searcher getComponent = newVespaClientSearcher("com.yahoo.storage.searcher.VisitSearcher"); + mySearchChain.addInnerComponent(getComponent); + searchChains.add(mySearchChain); + } + + ContainerSearch containerSearch = new ContainerSearch(cluster, searchChains, new ContainerSearch.Options()); + cluster.setSearch(containerSearch); + + cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandler", "http://*/feed")); + cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerRemove", "http://*/remove")); + cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation", "http://*/removelocation")); + cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerGet", "http://*/get")); + cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerVisit", "http://*/visit")); + cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerCompatibility", "http://*/document")); + cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerStatus", "http://*/feedstatus")); + final ProcessingHandler<SearchChains> searchHandler = new ProcessingHandler<>( + cluster.getSearch().getChains(), "com.yahoo.search.handler.SearchHandler"); + searchHandler.addServerBindings("http://*/search/*"); + cluster.addComponent(searchHandler); + + ContainerModelBuilder.addDefaultHandler_legacyBuilder(cluster); + + //BEGIN HACK for docproc chains: + DocprocChains docprocChains = getDocprocChains(cluster, spec); + if (docprocChains != null) { + ContainerDocproc containerDocproc = new ContainerDocproc(cluster, docprocChains); + cluster.setDocproc(containerDocproc); + } + //END HACK + + return cluster; + } + + private Handler newVespaClientHandler(String componentId, String binding) { + Handler<AbstractConfigProducer<?>> handler = new Handler<>(new ComponentModel( + BundleInstantiationSpecification.getFromStrings(componentId, null, vespaClientBundleSpecification), "")); + handler.addServerBindings(binding); + handler.addServerBindings(binding + '/'); + return handler; + } + + private Searcher newVespaClientSearcher(String componentSpec) { + return new Searcher<>(new ChainedComponentModel( + BundleInstantiationSpecification.getFromStrings(componentSpec, null, vespaClientBundleSpecification), + new Dependencies(null, null, null))); + } + + //BEGIN HACK for docproc chains: + private DocprocChains getDocprocChains(AbstractConfigProducer qrs, Element gateways) { + Element clients = (Element) gateways.getParentNode(); + Element services = (Element) clients.getParentNode(); + if (services == null) { + return null; + } + + Element docproc = XML.getChild(services, "docproc"); + if (docproc == null) { + return null; + } + + String version = docproc.getAttribute("version"); + if (version.startsWith("1.")) { + return null; + } else if (version.startsWith("2.")) { + return null; + } else if (version.startsWith("3.")) { + return getDocprocChainsV3(qrs, docproc); + } else { + throw new IllegalArgumentException("Docproc version " + version + " unknown."); + } + } + + private DocprocChains getDocprocChainsV3(AbstractConfigProducer qrs, Element docproc) { + Element docprocChainsElem = XML.getChild(docproc, "docprocchains"); + if (docprocChainsElem == null) { + return null; + } + return new DomDocprocChainsBuilder(null, true).build(qrs, docprocChainsElem); + } + //END HACK + } + + public static class ContainerHttpGatewayBuilder extends DomConfigProducerBuilder<ContainerHttpGateway> { + int index; + ContainerCluster cluster; + + public ContainerHttpGatewayBuilder(ContainerCluster cluster, int index) { + this.index = index; + this.cluster = cluster; + } + + @Override + protected ContainerHttpGateway doBuild(AbstractConfigProducer parent, + Element spec) { + // TODO: remove port handling + int port = 19020; + if (spec != null && spec.hasAttribute("baseport")) { + port = Integer.parseInt(spec.getAttribute("baseport")); + } + ContainerHttpGateway httpGateway = new ContainerHttpGateway(cluster, "" + index, port); + List<Container> containers = new ArrayList<>(); + containers.add(httpGateway); + + cluster.addContainers(containers); + return httpGateway; + } + } + + /** + * This class parses the feederoptions xml tag and produces Vespa config output. + * + * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a> + */ + private class FeederOptionsParser implements Serializable { + private static final long serialVersionUID = 1L; + // All member variables are objects so that we can switch on null values. + private Boolean abortondocumenterror = null; + private String route = null; + private Integer maxpendingdocs = null; + private Integer maxpendingbytes = null; + private Boolean retryenabled = null; + private Double retrydelay = null; + private Double timeout = null; + private Integer tracelevel = null; + private Integer mbusport = null; + private String docprocChain = null; + + /** + * Constructs an empty feeder options object with all members set to null. + */ + public FeederOptionsParser() { + // empty + } + + /** + * Parses the content of the given XML element as feeder options. + * + * @param conf The XML element to parse. + */ + public FeederOptionsParser parseFeederOptions(Element conf) { + for (Node node : XML.getChildren(conf)) { + String nodename = node.getNodeName(); + Node firstchild = node.getFirstChild(); + String childval = (firstchild != null) ? firstchild.getNodeValue() : null; + + switch (nodename) { + case "abortondocumenterror": + abortondocumenterror = Boolean.valueOf(childval); + break; + case "maxpendingdocs": + maxpendingdocs = new Integer(childval); + break; + case "maxpendingbytes": + maxpendingbytes = new Integer(childval); + break; + case "retryenabled": + retryenabled = Boolean.valueOf(childval); + break; + case "retrydelay": + retrydelay = new Double(childval); + break; + case "timeout": + timeout = new Double(childval); + break; + case "route": + route = childval; + break; + case "tracelevel": + tracelevel = new Integer(childval); + break; + case "mbusport": + mbusport = new Integer(childval); + break; + case "docprocchain": + docprocChain = childval; + break; + } + } + return this; + } + + /** + * Returns a feeder options config string of the content of this. + * + * @return A config string. + */ + public FeederConfig.Builder getFeederConfig() { + FeederConfig.Builder builder = new FeederConfig.Builder(); + if (abortondocumenterror != null) { + builder.abortondocumenterror(abortondocumenterror); + } + if (route != null && route.length() > 0) { + builder.route(route); + } + if (maxpendingdocs != null) { + builder.maxpendingdocs(maxpendingdocs); + } + if (maxpendingbytes != null) { + builder.maxpendingbytes(maxpendingbytes); + } + if (retryenabled != null) { + builder.retryenabled(retryenabled); + } + if (retrydelay != null) { + builder.retrydelay(retrydelay); + } + if (timeout != null) { + builder.timeout(timeout); + } + if (tracelevel != null) { + builder.tracelevel(tracelevel); + } + if (mbusport != null) { + builder.mbusport(mbusport); + } + if (docprocChain != null && docprocChain.length() > 0) { + builder.docprocchain(docprocChain); + } + return builder; + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilder.java new file mode 100644 index 00000000000..87a88fae176 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/LegacyConfigModelBuilder.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.config.model.ConfigModel; +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.config.model.ConfigModelInstanceFactory; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import org.w3c.dom.Element; + +/** + * A model builder that can be used to deal with toplevel config overrides and create another + * producer in between. This should not be used by new model plugins. + * + * @author lulf + * @since 5.1 + */ +public abstract class LegacyConfigModelBuilder<MODEL extends ConfigModel> extends ConfigModelBuilder<MODEL> { + + public LegacyConfigModelBuilder(Class<MODEL> configModelClass) { + super(configModelClass); + } + + @Override + public MODEL build(ConfigModelInstanceFactory<MODEL> factory, Element spec, ConfigModelContext context) { + VespaDomBuilder.DomSimpleConfigProducerBuilder builder = new VespaDomBuilder.DomSimpleConfigProducerBuilder(context.getProducerId()); + AbstractConfigProducer producer = builder.build(context.getParentProducer(), spec); + return super.build(factory, spec, context.modifyParent(producer)); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java new file mode 100644 index 00000000000..a60e0dd01e4 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ModelElement.java @@ -0,0 +1,211 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.text.XML; +import com.yahoo.vespa.model.utils.Duration; + +import org.w3c.dom.Element; + +import java.util.ArrayList; +import java.util.List; +import java.util.StringTokenizer; + +/** + * A w3c Element wrapper whith a better API. + * + * Author unknown. + */ +public class ModelElement { + + private final Element xml; + + public ModelElement(Element xml) { + this.xml = xml; + if (xml == null) { + throw new NullPointerException("Can not create ModelElement with null element"); + } + if (xml.getNodeName() == null) { + throw new NullPointerException("Can not create ModelElement with unnamed element"); + } + } + + public Element getXml() { + return xml; + } + + /** + * If not found, return null. + */ + public ModelElement getChild(String name) { + Element e = XML.getChild(xml, name); + + if (e != null) { + return new ModelElement(e); + } + + return null; + } + + public ModelElement getChildByPath(String path) { + StringTokenizer tokenizer = new StringTokenizer(path, "."); + ModelElement curElem = this; + while (tokenizer.hasMoreTokens() && curElem != null) { + String pathElem = tokenizer.nextToken(); + ModelElement child = curElem.getChild(pathElem); + if (!tokenizer.hasMoreTokens()) { + if (child != null) { + return child; + } + } + curElem = child; + } + return null; + } + + public String childAsString(String path) { + StringTokenizer tokenizer = new StringTokenizer(path, "."); + ModelElement curElem = this; + while (tokenizer.hasMoreTokens() && curElem != null) { + String pathElem = tokenizer.nextToken(); + ModelElement child = curElem.getChild(pathElem); + if (!tokenizer.hasMoreTokens()) { + String attr = curElem.getStringAttribute(pathElem); + if (attr != null) { + return attr; + } else if (child != null) { + return child.asString(); + } + } + curElem = child; + } + return null; + } + + public String asString() { + return xml.getFirstChild().getTextContent(); + } + + public double asDouble() { + return Double.parseDouble(asString()); + } + + public long asLong() { + return (long) BinaryUnit.valueOf(asString()); + } + + public Duration asDuration() { + return new Duration(asString()); + } + + public Long childAsLong(String path) { + String child = childAsString(path); + if (child == null) { + return null; + } + return Long.parseLong(child.trim()); + } + + public Integer childAsInteger(String path) { + String child = childAsString(path); + if (child == null) { + return null; + } + return Integer.parseInt(child.trim()); + } + + public Double childAsDouble(String path) { + String child = childAsString(path); + if (child == null) { + return null; + } + return Double.parseDouble(child.trim()); + } + + public Boolean childAsBoolean(String path) { + String child = childAsString(path); + if (child == null) { + return null; + } + return Boolean.parseBoolean(child.trim()); + } + + public Duration childAsDuration(String path) { + String child = childAsString(path); + if (child == null) { + return null; + } + return new Duration(child); + } + + /** Returns the given attribute or throws IllegalArgumentException if not present */ + public int requiredIntegerAttribute(String name) { + if (getStringAttribute(name) == null) + throw new IllegalArgumentException("Required attribute '" + name + "' is missing"); + return getIntegerAttribute(name, null); + } + + /** Returns the value of this attribute or null if not present */ + public Integer getIntegerAttribute(String name) { + return getIntegerAttribute(name, null); + } + + public Integer getIntegerAttribute(String name, Integer defaultValue) { + String value = getStringAttribute(name); + if (value == null) { + return defaultValue; + } + return (int) BinaryUnit.valueOf(value); + } + + public boolean getBooleanAttribute(String name) { + return getBooleanAttribute(name, false); + } + + public boolean getBooleanAttribute(String name, boolean defaultValue) { + String value = getStringAttribute(name); + if (value == null) { + return defaultValue; + } + return Boolean.parseBoolean(value); + } + + public Long getLongAttribute(String name) { + String value = getStringAttribute(name); + if (value == null) { + return null; + } + return (long) BinaryUnit.valueOf(value); + } + + public Double getDoubleAttribute(String name) { + String value = getStringAttribute(name); + if (value == null) { + return null; + } + return Double.parseDouble(value); + } + + public String getStringAttribute(String name) { + if (!xml.hasAttribute(name)) { + return null; + } + + return xml.getAttribute(name); + } + + public List<ModelElement> subElements(String name) { + List<Element> elements = XML.getChildren(xml, name); + + List<ModelElement> helpers = new ArrayList<>(); + for (Element e : elements) { + helpers.add(new ModelElement(e)); + } + + return helpers; + } + + @Override + public String toString() { + return xml.getNodeName(); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java new file mode 100644 index 00000000000..a27739b42ef --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -0,0 +1,105 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.HostSystem; + +import java.util.Map; +import java.util.Optional; + +/** + * A common utility class to represent a requirement for some nodes during model building. + * Such a requirement is commonly specified as a <code>nodes</code> element. + * + * @author bratseth + */ + // TODO: Use this for all nodes tags and unify with NodesUtil +public class NodesSpecification { + + private final boolean dedicated; + + private final int count; + + private final int groups; + + private final Optional<String> flavor; + + private final Optional<String> dockerImage; + + private NodesSpecification(boolean dedicated, int count, int groups, Optional<String> flavor, Optional<String> dockerImage) { + this.dedicated = dedicated; + this.count = count; + this.groups = groups; + this.flavor = flavor; + this.dockerImage = dockerImage; + } + + private NodesSpecification(boolean dedicated, ModelElement nodesElement) { + this(dedicated, + nodesElement.requiredIntegerAttribute("count"), + nodesElement.getIntegerAttribute("groups", 1), + Optional.ofNullable(nodesElement.getStringAttribute("flavor")), + Optional.ofNullable(nodesElement.getStringAttribute("docker-image"))); + } + + /** + * Returns a requirement for dedicated nodes taken from the given <code>nodes</code> element + */ + public static NodesSpecification from(ModelElement nodesElement) { + return new NodesSpecification(true, nodesElement); + } + + /** + * Returns a requirement for dedicated nodes taken from the <code>nodes</code> element + * contained in the given parent element, or empty if the parent element is null, or the nodes elements + * is not present. + */ + public static Optional<NodesSpecification> fromParent(ModelElement parentElement) { + if (parentElement == null) return Optional.empty(); + ModelElement nodesElement = parentElement.getChild("nodes"); + if (nodesElement == null) return Optional.empty(); + return Optional.of(from(nodesElement)); + } + + /** + * Returns a requirement for undedicated or dedicated nodes taken from the <code>nodes</code> element + * contained in the given parent element, or empty if the parent element is null, or the nodes elements + * is not present. + */ + public static Optional<NodesSpecification> optionalDedicatedFromParent(ModelElement parentElement) { + if (parentElement == null) return Optional.empty(); + ModelElement nodesElement = parentElement.getChild("nodes"); + if (nodesElement == null) return Optional.empty(); + return Optional.of(new NodesSpecification(nodesElement.getBooleanAttribute("dedicated", false), nodesElement)); + } + + /** Returns a requirement from <code>count</code> nondedicated nodes in one group */ + public static NodesSpecification nonDedicated(int count) { + return new NodesSpecification(false, count, 1, Optional.empty(), Optional.empty()); + } + + /** + * Returns whether this requires dedicated nodes. + * Otherwise the model encountering this request should reuse nodes requested for other purposes whenever possible. + */ + public boolean isDedicated() { return dedicated; } + + /** Returns the number of nodes required */ + public int count() { return count; } + + /** Returns the number of host groups this specifies. Default is 1 */ + public int groups() { return groups; } + + public Map<HostResource, ClusterMembership> provision(HostSystem hostSystem, ClusterSpec.Type clusterType, ClusterSpec.Id clusterId, Optional<ClusterSpec.Group> clusterGroup, DeployLogger logger) { + if (clusterGroup.isPresent() && groups > 1) + throw new IllegalArgumentException("Cannot both specify a group and request multiple groups"); + ClusterSpec cluster = ClusterSpec.from(clusterType, clusterId, clusterGroup, dockerImage); + return hostSystem.allocateHosts(cluster, Capacity.fromNodeCount(count, flavor), groups, logger); + } + + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ServletBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ServletBuilder.java new file mode 100644 index 00000000000..147f6517754 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/ServletBuilder.java @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.container.component.Servlet; +import com.yahoo.vespa.model.container.component.ServletProvider; +import com.yahoo.vespa.model.container.component.SimpleComponent; +import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder; +import org.w3c.dom.Element; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author stiankri + * @since 5.32 + */ +public class ServletBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Servlet> { + @Override + protected ServletProvider doBuild(AbstractConfigProducer ancestor, Element servletElement) { + SimpleComponent servlet = createServletComponent(servletElement); + ServletProvider servletProvider = createServletProvider(servletElement, servlet); + + return servletProvider; + } + + private SimpleComponent createServletComponent(Element servletElement) { + ComponentModel componentModel = new ComponentModel(BundleInstantiationSpecificationBuilder.build(servletElement, false)); + return new SimpleComponent(componentModel); + } + + private ServletProvider createServletProvider(Element servletElement, SimpleComponent servlet) { + Map<String, String> servletConfig = getServletConfig(servletElement); + return new ServletProvider(servlet, getPath(servletElement), servletConfig); + } + + private String getPath(Element servletElement) { + Element pathElement = XML.getChild(servletElement, "path"); + return XML.getValue(pathElement); + } + + private Map<String, String> getServletConfig(Element servletElement) { + Map<String, String> servletConfig = new HashMap<>(); + + Element servletConfigElement = XML.getChild(servletElement, "servlet-config"); + XML.getChildren(servletConfigElement).forEach( parameter -> + servletConfig.put(parameter.getTagName(), XML.getValue(parameter)) + ); + + return servletConfig; + } +} + diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java new file mode 100644 index 00000000000..08c3db81091 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java @@ -0,0 +1,301 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.config.model.*; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.model.producer.UserConfigRepo; +import com.yahoo.log.LogLevel; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.*; +import com.yahoo.vespa.model.builder.UserConfigBuilder; +import com.yahoo.vespa.model.builder.VespaModelBuilder; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.ContainerModel; +import com.yahoo.vespa.model.container.docproc.ContainerDocproc; +import com.yahoo.vespa.model.content.Content; +import com.yahoo.vespa.model.generic.builder.DomServiceClusterBuilder; +import com.yahoo.vespa.model.generic.service.ServiceCluster; +import com.yahoo.vespa.model.search.AbstractSearchCluster; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.logging.Logger; + +/** + * Builds Vespa model components using the w3c dom api + * + * @author vegardh + */ +public class VespaDomBuilder extends VespaModelBuilder { + + public static final String JVMARGS_ATTRIB_NAME="jvmargs"; + public static final String PRELOAD_ATTRIB_NAME="preload"; + public static final String MMAP_NOCORE_LIMIT="mmap-nocore-limit"; + private static final String CPU_SOCKET_ATTRIB_NAME = "cpu-socket"; + public static final String CPU_SOCKET_AFFINITY_ATTRIB_NAME = "cpu-socket-affinity"; + + public static final Logger log = Logger.getLogger(VespaDomBuilder.class.getPackage().toString()); + + /** + * Get all aliases for one host from a list of 'alias' xml nodes. + * + * @param hostAliases List of xml nodes, each representing one hostalias + * @return a list of alias strings. + */ + // TODO Move and change scope + public static List<String> getHostAliases(NodeList hostAliases) { + List<String> aliases = new LinkedList<>(); + for (int i=0; i < hostAliases.getLength(); i++) { + Node n = hostAliases.item(i); + if (! (n instanceof Element)) { + continue; + } + Element e = (Element)n; + if (! e.getNodeName().equals("alias")) { + throw new RuntimeException("Unexpected tag: '" + e.getNodeName() + "' at node " + + XML.getNodePath(e, " > ") + ", expected 'alias'."); + } + String alias = e.getFirstChild().getNodeValue(); + if ((alias == null) || (alias.equals(""))) { + throw new RuntimeException("Missing value for the alias tag at node " + + XML.getNodePath(e, " > ") + "'."); + } + aliases.add(alias); + } + return aliases; + } + + + @Override + public ApplicationConfigProducerRoot getRoot(String name, DeployState deployState, AbstractConfigProducer parent) { + try { + return new DomRootBuilder(name, deployState). + build(parent, XmlHelper.getDocument(deployState.getApplicationPackage().getServices()).getDocumentElement()); + } catch (Exception e) { + throw new IllegalArgumentException(e); + } + } + + /** + * @param spec The element containing the xml specification for this Service. + * @return the user's desired port, which is retrieved from the xml spec. + */ + public static int getXmlWantedPort(Element spec) { + return getXmlIntegerAttribute(spec, "baseport"); + } + + /** + * Base class for builders of producers using DOM. The purpose is to always + * include hostalias, baseport and user config overrides generically. + * + * @param <T> an {@link com.yahoo.config.model.producer.AbstractConfigProducer} + * @author vegardh + */ + public static abstract class DomConfigProducerBuilder<T extends AbstractConfigProducer> { + + // TODO: find good way to provide access to app package + public final T build(AbstractConfigProducer ancestor, Element producerSpec) { + T t = doBuild(ancestor, producerSpec); + + if (t instanceof AbstractService) { + initializeService((AbstractService)t, ancestor, producerSpec); + } else { + initializeProducer(t, ancestor, producerSpec); + } + + return t; + } + + protected abstract T doBuild(AbstractConfigProducer ancestor, Element producerSpec); + + private void initializeProducer(AbstractConfigProducer child, + AbstractConfigProducer ancestor, + Element producerSpec) { + UserConfigRepo userConfigs = UserConfigBuilder.build(producerSpec, ancestor.getRoot().getDeployState(), ancestor.getRoot().deployLogger()); + // TODO: must be made to work: + //userConfigs.applyWarnings(child); + log.log(LogLevel.DEBUG, "Adding user configs " + userConfigs + " for " + producerSpec); + child.mergeUserConfigs(userConfigs); + } + + private void initializeService(AbstractService t, + AbstractConfigProducer ancestor, + Element producerSpec) { + initializeProducer(t, ancestor, producerSpec); + if (producerSpec != null) { + if (producerSpec.hasAttribute(JVMARGS_ATTRIB_NAME)) { + t.appendJvmArgs(producerSpec.getAttribute(JVMARGS_ATTRIB_NAME)); + } + if (producerSpec.hasAttribute(PRELOAD_ATTRIB_NAME)) { + t.setPreLoad(producerSpec.getAttribute(PRELOAD_ATTRIB_NAME)); + } + if (producerSpec.hasAttribute(MMAP_NOCORE_LIMIT)) { + t.setMMapNoCoreLimit(Long.parseLong(producerSpec.getAttribute(MMAP_NOCORE_LIMIT))); + } + if (producerSpec.hasAttribute(CPU_SOCKET_ATTRIB_NAME)) { + t.setAffinity(new Affinity.Builder().cpuSocket(Integer.parseInt(producerSpec.getAttribute(CPU_SOCKET_ATTRIB_NAME))).build()); + } + int port = getXmlWantedPort(producerSpec); + if (port > 0) { + t.setBasePort(port); + } + allocateHost(t, ancestor.getHostSystem(), producerSpec); + } + // This depends on which constructor in AbstractService is used, but the best way + // is to let this method do initialize. + if (!t.isInitialized()) { + t.initService(); + } + } + + /** + * Allocates a host to the service using host file or create service spec for provisioner to use later + * Pre-condition: producerSpec is non-null + * @param service the service to allocate a host for + * @param hostSystem a {@link HostSystem} + * @param producerSpec xml element for the service + */ + private void allocateHost(final AbstractService service, HostSystem hostSystem, Element producerSpec) { + // TODO store service on something else than HostSystem, to not make that overloaded + service.setHostResource(hostSystem.getHost(producerSpec.getAttribute("hostalias"))); + } + } + + /** + * The SimpleConfigProducer is the producer for elements such as qrservers, topleveldispatchers, gateways. + * Must support overrides for that too, hence this builder + * + * @author vegardh + */ + public static class DomSimpleConfigProducerBuilder extends DomConfigProducerBuilder<SimpleConfigProducer> { + private String configId = null; + + public DomSimpleConfigProducerBuilder(String configId) { + this.configId = configId; + } + + @Override + protected SimpleConfigProducer doBuild(AbstractConfigProducer parent, + Element producerSpec) { + return new SimpleConfigProducer(parent, configId); + } + } + + public static class DomRootBuilder extends VespaDomBuilder.DomConfigProducerBuilder<ApplicationConfigProducerRoot> { + private final String name; + private final DeployState deployState; + + /** + * @param name The name of the Vespa to create. Usually 'root' when there is only one. + */ + public DomRootBuilder(String name, DeployState deployState) { + this.name = name; + this.deployState = deployState; + } + + @Override + protected ApplicationConfigProducerRoot doBuild(AbstractConfigProducer parent, Element producerSpec) { + ApplicationConfigProducerRoot root = new ApplicationConfigProducerRoot(parent, name, + deployState.getDocumentModel(), deployState.getProperties().vespaVersion(), deployState.getProperties().applicationId()); + root.setHostSystem(new HostSystem(root, "hosts", deployState.getProvisioner())); + new Client(root); + return root; + } + } + + /** + * Gets the index from a service's spec + * + * @param spec The element containing the xml specification for this Service. + * @return the index of the service, which is retrieved from the xml spec. + */ + static int getIndex(Element spec) { + return getXmlIntegerAttribute(spec, "index"); + } + + /** + * Gets an integer attribute value from a service's spec + * + * @param spec XML element + * @param attributeName nam of attribute to get value from + * @return value of attribute, or 0 if it does not exist or is empty + */ + static int getXmlIntegerAttribute(Element spec, String attributeName) { + String value = (spec == null) ? null : spec.getAttribute(attributeName); + if (value == null || value.equals("")) { + return 0; + } else { + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + throw new IllegalArgumentException + ("Illegal format for attribute '" + attributeName + "' at node " + + XML.getNodePath(spec, " > ") + ", must be an integer", e); + } + } + } + + /** + * Processing that requires access across different plugins + * + * @param root root config producer + * @param configModelRepo a {@link ConfigModelRepo} + */ + public void postProc(AbstractConfigProducer root, ConfigModelRepo configModelRepo) { + createTlds(configModelRepo); + setContentSearchClusterIndexes(configModelRepo); + createDocprocMBusServersAndClients(configModelRepo); + } + + private void createDocprocMBusServersAndClients(ConfigModelRepo pc) { + for (ContainerCluster cluster: ContainerModel.containerClusters(pc)) { + addServerAndClientsForChains(cluster.getDocproc()); + } + } + + private void addServerAndClientsForChains(ContainerDocproc docproc) { + if (docproc != null) + docproc.getChains().addServersAndClientsForChains(); + } + + private void createTlds(ConfigModelRepo pc) { + for (ConfigModel p : pc.asMap().values()) { + if (p instanceof Content) { + ((Content)p).createTlds(pc); + } + } + } + + /** + * For some reason, search clusters need to be enumerated. + * @param configModelRepo a {@link ConfigModelRepo} + */ + private void setContentSearchClusterIndexes(ConfigModelRepo configModelRepo) { + int index = 0; + for (AbstractSearchCluster sc : Content.getSearchClusters(configModelRepo)) { + sc.setClusterIndex(index++); + } + } + + @Override + public List<ServiceCluster> getClusters(ApplicationPackage pkg, + AbstractConfigProducer parent) { + List<ServiceCluster> clusters = new ArrayList<>(); + Document services = XmlHelper.getDocument(pkg.getServices()); + for (Element clusterSpec : XML.getChildren(services.getDocumentElement(), "cluster")) { + DomServiceClusterBuilder clusterBuilder = new DomServiceClusterBuilder(clusterSpec.getAttribute("name")); + clusters.add(clusterBuilder.build(parent.getRoot(), clusterSpec)); + } + return clusters; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainSpecificationBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainSpecificationBuilder.java new file mode 100644 index 00000000000..9dac8a23587 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainSpecificationBuilder.java @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.Phase; +import com.yahoo.component.chain.dependencies.Dependencies; +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.text.XML; +import com.yahoo.config.model.builder.xml.XmlHelper; +import org.w3c.dom.Element; + +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.Set; + + +/** + * Creates a partial ChainSpecification without inner components + * @author tonytv + */ +public class ChainSpecificationBuilder { + private final ComponentId componentId; + private final ChainSpecification.Inheritance inheritance; + private final Collection<Phase> phases; + + public ChainSpecificationBuilder(Element chainElement) { + componentId = readComponentId(chainElement); + inheritance = readInheritance(chainElement); + phases = readPhases(chainElement); + } + + private Set<Phase> readPhases(Element parentElement) { + Set<Phase> phases = new LinkedHashSet<>(); + + for (Element phaseSpec : XML.getChildren(parentElement, "phase")) { + String name = XmlHelper.getIdString(phaseSpec); + Dependencies dependencies = new DependenciesBuilder(phaseSpec).build(); + phases.add(new Phase(name, dependencies)); + } + return phases; + } + + private ComponentId readComponentId(Element spec) { + return XmlHelper.getId(spec); + } + + private ChainSpecification.Inheritance readInheritance(Element spec) { + return new InheritanceBuilder(spec).build(); + } + + public ChainSpecification build(Set<ComponentSpecification> outerComponentReferences) { + return new ChainSpecification(componentId, inheritance, phases, outerComponentReferences); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainedComponentModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainedComponentModelBuilder.java new file mode 100644 index 00000000000..7414723c164 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainedComponentModelBuilder.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains; + +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.component.chain.model.ChainedComponentModel; +import com.yahoo.vespa.model.container.xml.BundleInstantiationSpecificationBuilder; +import org.w3c.dom.Element; + +/** + * Builds a regular ChainedComponentModel from an element. + * @author tonytv + */ +public class ChainedComponentModelBuilder extends GenericChainedComponentModelBuilder { + protected final BundleInstantiationSpecification bundleInstantiationSpec; + + public ChainedComponentModelBuilder(Element spec) { + super(spec); + bundleInstantiationSpec = BundleInstantiationSpecificationBuilder.build(spec, false); + } + + public ChainedComponentModel build() { + return new ChainedComponentModel(bundleInstantiationSpec, dependencies); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainsBuilder.java new file mode 100644 index 00000000000..9c848e3f284 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ChainsBuilder.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains; + +import com.yahoo.text.XML; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.container.component.chain.Chain; +import com.yahoo.vespa.model.container.component.chain.ChainedComponent; +import org.w3c.dom.Element; + +import java.util.*; + +/** + * @author tonytv + * @author gjoranv + */ +public class ChainsBuilder<COMPONENT extends ChainedComponent<?>, CHAIN extends Chain<COMPONENT>> { + + protected final List<CHAIN> chains = new ArrayList<>(); + private final Map<String, Class<? extends DomChainBuilderBase<? extends COMPONENT, ? extends CHAIN>>> chainType2BuilderClass; + + // NOTE: The chain type string (key in chainType2BuilderClass) must match the xml tag name for the chain. + public ChainsBuilder(AbstractConfigProducer ancestor, List<Element> chainsElems, + Map<String, ComponentsBuilder.ComponentType> outerComponentTypeByComponentName, + Map<String, Class<? extends DomChainBuilderBase<? extends COMPONENT, ? extends CHAIN>>> chainType2BuilderClass) { + + this.chainType2BuilderClass = chainType2BuilderClass; + readChains(ancestor, chainsElems, outerComponentTypeByComponentName); + } + + public Collection<CHAIN> getChains() { + return Collections.unmodifiableCollection(chains); + } + + private void readChains(AbstractConfigProducer ancestor, List<Element> chainsElems, + Map<String, ComponentsBuilder.ComponentType> outerSearcherTypeByComponentName) { + + for (Map.Entry<String, Class<? extends DomChainBuilderBase<? extends COMPONENT, ? extends CHAIN>>> + chainType : chainType2BuilderClass.entrySet()) { + for (Element elemContainingChainElems : chainsElems) { + for (Element chainElem : XML.getChildren(elemContainingChainElems, chainType.getKey())) { + readChain(ancestor, chainElem, chainType.getValue(), outerSearcherTypeByComponentName); + } + } + } + } + + private void readChain(AbstractConfigProducer ancestor, Element chainElem, + Class<? extends DomChainBuilderBase<? extends COMPONENT, ? extends CHAIN>> builderClass, + Map<String, ComponentsBuilder.ComponentType> outerSearcherTypeByComponentName) { + + DomChainBuilderBase<? extends COMPONENT, ? extends CHAIN> builder = + DomBuilderCreator.create(builderClass, outerSearcherTypeByComponentName); + chains.add(builder.build(ancestor, chainElem)); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ComponentsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ComponentsBuilder.java new file mode 100644 index 00000000000..b7aef6b9b1f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/ComponentsBuilder.java @@ -0,0 +1,156 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.text.XML; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.docproc.DomDocumentProcessorBuilder; +import com.yahoo.vespa.model.container.http.xml.FilterBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.processing.DomProcessorBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.search.DomFederationSearcherBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.search.DomSearcherBuilder; +import com.yahoo.vespa.model.container.component.chain.ChainedComponent; +import com.yahoo.vespa.model.container.http.Filter; +import com.yahoo.vespa.model.container.processing.Processor; +import com.yahoo.vespa.model.container.docproc.DocumentProcessor; +import com.yahoo.vespa.model.container.search.searchchain.Searcher; +import org.w3c.dom.Element; + +import java.util.*; + +/** + * Creates component models and component references from xml for a given scope. + * @author tonytv + */ +public class ComponentsBuilder<T extends ChainedComponent<?>> { + + // NOTE: the 'name' string must match the xml tag name for the component in services. + public static class ComponentType<T extends ChainedComponent<?>> { + static ArrayList<ComponentType> values = new ArrayList<>(); + public static final ComponentType<DocumentProcessor> documentprocessor = new ComponentType<>("documentprocessor", DomDocumentProcessorBuilder.class); + public static final ComponentType<Searcher<?>> searcher = new ComponentType<>("searcher", DomSearcherBuilder.class); + public static final ComponentType<Processor> processor = new ComponentType<>("processor", DomProcessorBuilder.class); + public static final ComponentType<Searcher<?>> federation = new ComponentType<>("federation", DomFederationSearcherBuilder.class); + public static final ComponentType<Filter> filter = new ComponentType<>("filter", FilterBuilder.class); + + + final String name; + + private final Class<? extends VespaDomBuilder.DomConfigProducerBuilder<T>> builderClass; + + private ComponentType(String name, Class<? extends VespaDomBuilder.DomConfigProducerBuilder<T>> builderClass) { + this.name = name; + this.builderClass = builderClass; + values.add(this); + } + + public VespaDomBuilder.DomConfigProducerBuilder<T> createBuilder() { + return DomBuilderCreator.create(builderClass); + } + } + + private final Set<ComponentSpecification> outerComponentReferences = new LinkedHashSet<>(); + private final List<T> componentDefinitions = new ArrayList<>(); + private final Map<String, ComponentType> componentTypesByComponentName = new LinkedHashMap<>(); + + /** + * @param ancestor The parent config producer + * @param componentTypes The allowed component types for 'elementContainingComponentElements' - MUST match <T> + * @param elementsContainingComponentElems All elements containing elements with name matching ComponentType.name + * @param outerComponentTypeByComponentName Use null if this is the outermost scope, i.e. + * every component is a definition, not a reference. + */ + ComponentsBuilder(AbstractConfigProducer ancestor, + Collection<ComponentType<T>> componentTypes, + List<Element> elementsContainingComponentElems, + Map<String, ComponentType> outerComponentTypeByComponentName) { + + readComponents(ancestor, componentTypes, elementsContainingComponentElems, unmodifiable(outerComponentTypeByComponentName)); + } + + private void readComponents(AbstractConfigProducer ancestor, + Collection<ComponentType<T>> componentTypes, + List<Element> elementsContainingComponentElems, + Map<String, ComponentType> outerComponentTypeByComponentName) { + + for (ComponentType<T> componentType : componentTypes) { + for (Element elemContainingComponentElems : elementsContainingComponentElems) { + for (Element componentElement : XML.getChildren(elemContainingComponentElems, componentType.name)) { + readComponent(ancestor, componentElement, componentType, outerComponentTypeByComponentName); + } + } + } + } + + private void readComponent(AbstractConfigProducer ancestor, + Element componentElement, + ComponentType<T> componentType, + Map<String, ComponentType> outerComponentTypeByComponentName) { + + ComponentSpecification componentSpecification = XmlHelper.getIdRef(componentElement); + + if (outerComponentTypeByComponentName.containsKey(componentSpecification.getName())) { + readComponentReference(componentElement, componentType, componentSpecification, outerComponentTypeByComponentName); + } else { + readComponentDefinition(ancestor, componentElement, componentType); + } + } + + private void readComponentReference(Element componentElement, ComponentType componentType, + ComponentSpecification componentSpecification, + Map<String, ComponentType> outerComponentTypeByComponentName) { + + String componentName = componentSpecification.getName(); + ensureTypesMatch(componentType, outerComponentTypeByComponentName.get(componentName), componentName); + ensureNotDefinition(componentName, componentElement); + outerComponentReferences.add(componentSpecification); + } + + private void readComponentDefinition(AbstractConfigProducer ancestor, Element componentElement, ComponentType<T> componentType) { + T component = componentType.createBuilder().build(ancestor, componentElement); + componentDefinitions.add(component); + updateComponentTypes(component.getComponentId(), componentType); + } + + private void updateComponentTypes(ComponentId componentId, ComponentType componentType) { + ComponentType oldType = componentTypesByComponentName.put(componentId.getName(), componentType); + if (oldType != null) { + ensureTypesMatch(componentType, oldType, componentId.getName()); + } + } + + private void ensureNotDefinition(String componentName, Element componentSpec) { + if (componentSpec.getAttributes().getLength() > 1 || !XML.getChildren(componentSpec).isEmpty()) + throw new RuntimeException("Expecting " + componentName + + " to be a reference to a global component with the same name," + + " so no additional attributes or nested elements are allowed"); + } + + private void ensureTypesMatch(ComponentType type1, ComponentType type2, String componentName) { + if (!type1.equals(type2)) { + throw new RuntimeException("Two different types declared for the component with name '" + componentName + "' (" + + type1.name + " != " + type2.name + ")."); + } + } + + private Map<String, ComponentType> unmodifiable(Map<String, ComponentType> outerComponentTypeByComponentName) { + return (outerComponentTypeByComponentName != null)? + Collections.unmodifiableMap(outerComponentTypeByComponentName): + Collections.<String, ComponentType>emptyMap(); + } + + public Collection<T> getComponentDefinitions() { + return Collections.unmodifiableCollection(componentDefinitions); + } + + public Map<String, ComponentType> getComponentTypeByComponentName() { + return Collections.unmodifiableMap(componentTypesByComponentName); + } + + public Set<ComponentSpecification> getOuterComponentReferences() { + return Collections.unmodifiableSet(outerComponentReferences); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DependenciesBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DependenciesBuilder.java new file mode 100644 index 00000000000..85ed36d0356 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DependenciesBuilder.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains; + +import com.yahoo.component.chain.dependencies.Dependencies; +import com.yahoo.config.model.builder.xml.XmlHelper; +import org.w3c.dom.Element; + +import java.util.HashSet; +import java.util.Set; + +/** + * Builds Dependencies (provides, before, after) from an element. + * @author tonytv + */ +public class DependenciesBuilder { + private final Dependencies dependencies; + + public DependenciesBuilder(Element spec) { + + Set<String> provides = read(spec, "provides"); + Set<String> before = read(spec, "before"); + Set<String> after = read(spec, "after"); + + dependencies = new Dependencies(provides, before, after); + } + + public Dependencies build() { + return dependencies; + } + + private Set<String> read(Element spec, String name) { + Set<String> symbols = new HashSet<>(); + symbols.addAll(XmlHelper.valuesFromElements(spec, name)); + symbols.addAll(XmlHelper.spaceSeparatedSymbolsFromAttribute(spec, name)); + + return symbols; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomBuilderCreator.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomBuilderCreator.java new file mode 100644 index 00000000000..1cf91f11ed1 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomBuilderCreator.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; + +/** + * Utility class for instantiating a builder using reflection. + * @author tonytv + */ +public class DomBuilderCreator { + public static <T> T create(Class<T> builderClass, Object... parameters) { + try { + return getConstructor(builderClass).newInstance(parameters); + } catch (InstantiationException | InvocationTargetException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + @SuppressWarnings("unchecked") + private static <T> Constructor<T> getConstructor(Class<T> builderClass) { + Constructor<?>[] constructors = builderClass.getConstructors(); + assert(constructors.length == 1); + return (Constructor<T>) constructors[0]; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomChainBuilderBase.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomChainBuilderBase.java new file mode 100644 index 00000000000..3233424f4ac --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomChainBuilderBase.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains; + +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.container.component.chain.Chain; +import com.yahoo.vespa.model.container.component.chain.ChainedComponent; +import org.w3c.dom.Element; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; + +/** + * Base functionality for all chain builders (docprocChain, searchChain, provider, source) + * @author tonytv + */ +public abstract class DomChainBuilderBase<COMPONENT extends ChainedComponent<?>, CHAIN extends Chain<COMPONENT>> + extends VespaDomBuilder.DomConfigProducerBuilder<CHAIN> { + + private Collection<ComponentsBuilder.ComponentType<COMPONENT>> allowedComponentTypes; + protected final Map<String, ComponentsBuilder.ComponentType> outerComponentTypeByComponentName; + + public DomChainBuilderBase(Collection<ComponentsBuilder.ComponentType<COMPONENT>> allowedComponentTypes, + Map<String, ComponentsBuilder.ComponentType> outerComponentTypeByComponentName) { + this.allowedComponentTypes = allowedComponentTypes; + this.outerComponentTypeByComponentName = outerComponentTypeByComponentName; + } + + public final CHAIN doBuild(AbstractConfigProducer ancestor, Element producerSpec) { + ComponentsBuilder<COMPONENT> componentsBuilder = + new ComponentsBuilder<>(ancestor, allowedComponentTypes, Arrays.asList(producerSpec), outerComponentTypeByComponentName); + ChainSpecification specWithoutInnerComponents = + new ChainSpecificationBuilder(producerSpec).build(componentsBuilder.getOuterComponentReferences()); + + CHAIN chain = buildChain(ancestor, producerSpec, specWithoutInnerComponents); + addInnerComponents(chain, componentsBuilder.getComponentDefinitions()); + + return chain; + } + + private void addInnerComponents(CHAIN chain, Collection<COMPONENT> componentDefinitions) { + for (COMPONENT innerComponent : componentDefinitions) { + chain.addInnerComponent(innerComponent); + } + } + + protected abstract CHAIN buildChain(AbstractConfigProducer ancestor, Element producerSpec, + ChainSpecification specWithoutInnerComponents); +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomChainsBuilder.java new file mode 100644 index 00000000000..9555d916d2e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/DomChainsBuilder.java @@ -0,0 +1,102 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains; + +import com.yahoo.config.application.Xml; +import com.yahoo.text.XML; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder.ComponentType; +import com.yahoo.vespa.model.container.component.chain.Chain; +import com.yahoo.vespa.model.container.component.chain.ChainedComponent; +import com.yahoo.vespa.model.container.component.chain.Chains; +import org.w3c.dom.Element; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * NOTE: This class _must_ be abstract, due to calling subclass method in ctor. + * @author tonytv + * @author gjoranv + */ +public abstract +class DomChainsBuilder<COMPONENT extends ChainedComponent<?>, CHAIN extends Chain<COMPONENT>, CHAINS extends Chains<CHAIN>> + extends VespaDomBuilder.DomConfigProducerBuilder<CHAINS> { + + private final Collection<ComponentType<COMPONENT>> allowedComponentTypes; + private final String appPkgChainsDir; + private final Element outerChainsElem; + + protected DomChainsBuilder(Element outerChainsElem, + Collection<ComponentType<COMPONENT>> allowedComponentTypes, + String appPkgChainsDir) { + + this.outerChainsElem = outerChainsElem; + + // XXX: hack to prevent 'config' in the outer chains elem. + // TODO: Remove when this can be handled by schema, i.e. when we don't need the non-cluster qrservers syntax anymore + if (outerChainsElem != null) { + if (XML.getChildren(outerChainsElem, "config").size() > 0) { + throw new RuntimeException("At node " + XML.getNodePath(outerChainsElem, " > ") + ": " + + "'config' is not allowed in the outer chains element, please move it up one level!"); + } + } + + this.allowedComponentTypes = new ArrayList<>(allowedComponentTypes); + this.appPkgChainsDir = appPkgChainsDir; + } + + protected abstract CHAINS newChainsInstance(AbstractConfigProducer parent); + + @Override + protected final CHAINS doBuild(AbstractConfigProducer parent, Element chainsElement) { + CHAINS chains = newChainsInstance(parent); + + List<Element> allChainElements = allChainElements(parent, chainsElement); + if (! allChainElements.isEmpty()) { + ComponentsBuilder<COMPONENT> outerComponentsBuilder = readOuterComponents(chains, allChainElements); + ChainsBuilder<COMPONENT, CHAIN> chainsBuilder = readChains(chains, allChainElements, + outerComponentsBuilder.getComponentTypeByComponentName()); + + addOuterComponents(chains, outerComponentsBuilder); + addChains(chains, chainsBuilder); + } + return chains; + } + + private List<Element> allChainElements(AbstractConfigProducer ancestor, Element chainsElement) { + List<Element> chainsElements = new ArrayList<>(); + if (outerChainsElem != null) + chainsElements.add(outerChainsElem); + chainsElements.add(chainsElement); + + if (appPkgChainsDir != null) + chainsElements.addAll(Xml.allElemsFromPath(ancestor.getRoot().getDeployState().getApplicationPackage(), appPkgChainsDir)); + + return chainsElements; + } + + private ComponentsBuilder<COMPONENT> readOuterComponents(AbstractConfigProducer ancestor, List<Element> chainsElems) { + return new ComponentsBuilder<>(ancestor, allowedComponentTypes, chainsElems, null); + } + + protected abstract + ChainsBuilder<COMPONENT, CHAIN> readChains(AbstractConfigProducer ancestor, List<Element> allChainsElems, + Map<String, ComponentsBuilder.ComponentType> outerComponentTypeByComponentName); + + private void addOuterComponents(CHAINS chains, ComponentsBuilder<COMPONENT> outerComponentsBuilder) { + assert (outerComponentsBuilder.getOuterComponentReferences().isEmpty()); + + for (ChainedComponent outerComponent : outerComponentsBuilder.getComponentDefinitions()) { + chains.add(outerComponent); + } + } + + private void addChains(CHAINS chains, ChainsBuilder<COMPONENT, CHAIN> chainsBuilder) { + for (CHAIN chain : chainsBuilder.getChains()) { + chains.add(chain); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/GenericChainedComponentModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/GenericChainedComponentModelBuilder.java new file mode 100644 index 00000000000..a7f5a306f8b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/GenericChainedComponentModelBuilder.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains; + +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.dependencies.Dependencies; +import com.yahoo.component.chain.model.ChainedComponentModel; +import com.yahoo.config.model.builder.xml.XmlHelper; +import org.w3c.dom.Element; + +/** + * reads the common attributes and elements of all chained component elements. + * @author tonytv + */ +public abstract class GenericChainedComponentModelBuilder { + //The componentId might be used as a spec later(for example as class or + //bundle), so we must treat it as a specification until then. + protected final ComponentSpecification componentId; + protected final Dependencies dependencies; + + public GenericChainedComponentModelBuilder(Element spec) { + componentId = readComponentId(spec); + dependencies = readDependencies(spec); + } + + private Dependencies readDependencies(Element spec) { + return new DependenciesBuilder(spec).build(); + } + + protected ComponentSpecification readComponentId(Element spec) { + return XmlHelper.getIdRef(spec); + } + + protected abstract ChainedComponentModel build(); + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/InheritanceBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/InheritanceBuilder.java new file mode 100644 index 00000000000..e617ea93405 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/InheritanceBuilder.java @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains; + +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.text.XML; +import org.w3c.dom.Element; + +import java.util.*; + +/** + * Build an Inheritance object from an inheritance section. + * @author tonytv + */ +public class InheritanceBuilder { + final ChainSpecification.Inheritance inheritance; + + public InheritanceBuilder(Element spec) { + inheritance = new ChainSpecification.Inheritance( + // XXX: for this to work, the tagname in the spec must match the tagname inside the 'inherits' elem, e.g. 'searchchain->inherits->searchchain' + read(spec, "inherits", spec.getTagName()), + read(spec, "excludes", "exclude")); + } + + public ChainSpecification.Inheritance build() { + return inheritance; + } + + private Set<ComponentSpecification> read(Element spec, String attributeName, String elementName) { + Set<ComponentSpecification> componentSpecifications = new LinkedHashSet<>(); + + componentSpecifications.addAll(spaceSeparatedComponentSpecificationsFromAttribute(spec, attributeName)); + + // TODO: the 'inherits' element is undocumented, and can be removed in an upcoming version of Vespa + componentSpecifications.addAll(idRefFromElements(XML.getChild(spec, "inherits"), elementName)); + + + return componentSpecifications; + } + + private Collection<ComponentSpecification> idRefFromElements(Element spec, String elementName) { + Collection<ComponentSpecification> result = new ArrayList<>(); + if (spec == null) + return result; + + for (Element element : XML.getChildren(spec, elementName)) { + result.add(XmlHelper.getIdRef(element)); + } + return result; + } + + private Collection<ComponentSpecification> spaceSeparatedComponentSpecificationsFromAttribute(Element spec, String attributeName) { + return toComponentSpecifications(XmlHelper.spaceSeparatedSymbolsFromAttribute(spec, attributeName)); + } + + private Set<ComponentSpecification> toComponentSpecifications(Collection<String> symbols) { + Set<ComponentSpecification> specifications = new LinkedHashSet<>(); + for (String symbol : symbols) { + specifications.add(new ComponentSpecification(symbol)); + } + return specifications; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DocprocChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DocprocChainsBuilder.java new file mode 100644 index 00000000000..8aabea965eb --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DocprocChainsBuilder.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains.docproc; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.builder.xml.dom.chains.ChainsBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainBuilderBase; +import com.yahoo.vespa.model.container.docproc.DocprocChain; +import com.yahoo.vespa.model.container.docproc.DocumentProcessor; +import org.w3c.dom.Element; + +import java.util.*; + +/** + * Creates all docproc chains from xml. + * + * @author gjoranv + */ +public class DocprocChainsBuilder extends ChainsBuilder<DocumentProcessor, DocprocChain> { + + private static final Map<String, Class<? extends DomChainBuilderBase<? extends DocumentProcessor, ? extends DocprocChain>>> + chainType2builderClass = Collections.unmodifiableMap( + new LinkedHashMap<String, Class<? extends DomChainBuilderBase<? extends DocumentProcessor, ? extends DocprocChain>>>() {{ + put("docprocchain", DomDocprocChainBuilder.class); + put("chain", DomDocprocChainBuilder.class); + }}); + + public DocprocChainsBuilder(AbstractConfigProducer ancestor, List<Element> docprocChainsElements, + Map<String, ComponentsBuilder.ComponentType> outerSearcherTypeByComponentName) { + super(ancestor, docprocChainsElements, outerSearcherTypeByComponentName, chainType2builderClass); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DocumentProcessorModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DocumentProcessorModelBuilder.java new file mode 100644 index 00000000000..e2c5afb3c86 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DocumentProcessorModelBuilder.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains.docproc; + +import com.yahoo.collections.Pair; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.builder.xml.dom.chains.ChainedComponentModelBuilder; +import com.yahoo.vespa.model.container.docproc.model.DocumentProcessorModel; +import org.w3c.dom.Element; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class DocumentProcessorModelBuilder extends ChainedComponentModelBuilder { + + private Map<Pair<String, String>, String> fieldNameSchemaMap = new HashMap<>(); + + public DocumentProcessorModelBuilder(Element spec) { + super(spec); + readFieldNameSchemaMap(spec); + } + + private void readFieldNameSchemaMap(Element spec) { + fieldNameSchemaMap = parseFieldNameSchemaMap(spec); + } + + @Override + public DocumentProcessorModel build() { + return new DocumentProcessorModel(bundleInstantiationSpec, dependencies, fieldNameSchemaMap); + } + + /** + * Parses a schemamapping element and generates a map of field mappings + * + * @param e a schemamapping element + * @return doctype,in-document → in-processor + */ + public static Map<Pair<String,String>, String> parseFieldNameSchemaMap(Element e) { + Map<Pair<String, String>, String> ret = new HashMap<>(); + for (Element sm : XML.getChildren(e, "map")) { + for (Element fm : XML.getChildren(sm, "field")) { + String from = fm.getAttribute("in-document"); + String to = fm.getAttribute("in-processor"); + String doctype = fm.getAttribute("doctype"); + if ("".equals(doctype)) doctype=null; + ret.put(new Pair<>(doctype, from), to); + } + } + return ret; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocprocChainBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocprocChainBuilder.java new file mode 100644 index 00000000000..476275ca290 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocprocChainBuilder.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains.docproc; + +import com.yahoo.collections.Pair; +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainBuilderBase; +import com.yahoo.vespa.model.container.docproc.DocprocChain; +import com.yahoo.vespa.model.container.docproc.DocumentProcessor; +import org.w3c.dom.Element; + +import java.util.Arrays; +import java.util.Map; + +/** + * Builds a docproc chain from xml + * + * @author gjoranv + */ +public class DomDocprocChainBuilder extends DomChainBuilderBase<DocumentProcessor, DocprocChain> { + + public DomDocprocChainBuilder(Map<String, ComponentsBuilder.ComponentType> outerComponentTypeByComponentName) { + super(Arrays.asList(ComponentsBuilder.ComponentType.documentprocessor), outerComponentTypeByComponentName); + } + + protected DocprocChain buildChain(AbstractConfigProducer ancestor, Element producerSpec, + ChainSpecification specWithoutInnerComponents) { + Map<Pair<String, String>, String> fieldNameSchemaMap = DocumentProcessorModelBuilder.parseFieldNameSchemaMap(producerSpec); + return new DocprocChain(specWithoutInnerComponents, fieldNameSchemaMap); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocprocChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocprocChainsBuilder.java new file mode 100644 index 00000000000..07dc6e337fd --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocprocChainsBuilder.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains.docproc; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder.ComponentType; +import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainsBuilder; +import com.yahoo.vespa.model.container.docproc.DocprocChains; +import com.yahoo.vespa.model.container.docproc.DocprocChain; +import com.yahoo.vespa.model.container.docproc.DocumentProcessor; +import org.w3c.dom.Element; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * Builds the docproc chains model from xml. + * + * @author gjoranv + */ +public class DomDocprocChainsBuilder extends DomChainsBuilder<DocumentProcessor, DocprocChain, DocprocChains> { + public DomDocprocChainsBuilder(Element outerChainsElem, boolean supportDocprocChainsDir) { + super(outerChainsElem, Arrays.asList(ComponentType.documentprocessor), + supportDocprocChainsDir ? ApplicationPackage.DOCPROCCHAINS_DIR: null); + } + + @Override + protected DocprocChains newChainsInstance(AbstractConfigProducer parent) { + return new DocprocChains(parent, "docprocchains"); + } + + @Override + protected DocprocChainsBuilder readChains(AbstractConfigProducer ancestor, List<Element> docprocChainsElements, + Map<String, ComponentType> outerComponentTypeByComponentName) { + return new DocprocChainsBuilder(ancestor, docprocChainsElements, outerComponentTypeByComponentName); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocumentProcessorBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocumentProcessorBuilder.java new file mode 100644 index 00000000000..43b652ccff5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/docproc/DomDocumentProcessorBuilder.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains.docproc; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.container.docproc.DocumentProcessor; +import org.w3c.dom.Element; + +/** + * Builds a DocumentProcessor from XML. + * + * @author gjoranv + */ +public class DomDocumentProcessorBuilder extends VespaDomBuilder.DomConfigProducerBuilder<DocumentProcessor> { + + protected DocumentProcessor doBuild(AbstractConfigProducer ancestor, Element documentProcessorElement) { + DocumentProcessorModelBuilder modelBuilder = new DocumentProcessorModelBuilder(documentProcessorElement); + return new DocumentProcessor(modelBuilder.build()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessingBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessingBuilder.java new file mode 100644 index 00000000000..4533b078b28 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessingBuilder.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains.processing; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainsBuilder; +import com.yahoo.vespa.model.container.processing.ProcessingChain; +import com.yahoo.vespa.model.container.processing.ProcessingChains; +import com.yahoo.vespa.model.container.processing.Processor; +import org.w3c.dom.Element; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * Root builder of the processing model + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + * @since 5.1.6 + */ +public class DomProcessingBuilder extends DomChainsBuilder<Processor, ProcessingChain, ProcessingChains> { + + public DomProcessingBuilder(Element outerChainsElem) { + super(outerChainsElem, Arrays.asList(ComponentsBuilder.ComponentType.processor), ApplicationPackage.PROCESSORCHAINS_DIR); + } + + @Override + protected ProcessingChains newChainsInstance(AbstractConfigProducer parent) { + return new ProcessingChains(parent, "processing"); + } + + @Override + protected ProcessingChainsBuilder readChains(AbstractConfigProducer ancestor, List<Element> processingChainsElements, + Map<String, ComponentsBuilder.ComponentType> outerComponentTypeByComponentName) { + return new ProcessingChainsBuilder(ancestor, processingChainsElements, outerComponentTypeByComponentName); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessingChainBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessingChainBuilder.java new file mode 100644 index 00000000000..a3cbff2910f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessingChainBuilder.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains.processing; + +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainBuilderBase; +import com.yahoo.vespa.model.container.processing.ProcessingChain; +import com.yahoo.vespa.model.container.processing.Processor; +import org.w3c.dom.Element; + +import java.util.Arrays; +import java.util.Map; + +/** + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + * @since 5.1.6 + */ +public class DomProcessingChainBuilder extends DomChainBuilderBase<Processor, ProcessingChain> { + + public DomProcessingChainBuilder(Map<String, ComponentsBuilder.ComponentType> outerComponentTypeByComponentName) { + super(Arrays.asList(ComponentsBuilder.ComponentType.processor), outerComponentTypeByComponentName); + } + + protected ProcessingChain buildChain(AbstractConfigProducer ancestor, Element producerSpec, + ChainSpecification specWithoutInnerComponents) { + return new ProcessingChain(specWithoutInnerComponents); + } + + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessorBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessorBuilder.java new file mode 100644 index 00000000000..b48d3dcab2f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/DomProcessorBuilder.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains.processing; + +import com.yahoo.vespa.model.builder.xml.dom.chains.ChainedComponentModelBuilder; +import com.yahoo.vespa.model.container.processing.Processor; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import org.w3c.dom.Element; + +/** + * Builds a processor from XML. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + * @since 5.1.6 + */ +public class DomProcessorBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Processor> { + + protected Processor doBuild(AbstractConfigProducer ancestor, Element processorElement) { + ChainedComponentModelBuilder modelBuilder = new ChainedComponentModelBuilder(processorElement); + return new Processor(modelBuilder.build()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/ProcessingChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/ProcessingChainsBuilder.java new file mode 100644 index 00000000000..5806de9343e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/processing/ProcessingChainsBuilder.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains.processing; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.builder.xml.dom.chains.ChainsBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainBuilderBase; +import com.yahoo.vespa.model.container.processing.ProcessingChain; +import com.yahoo.vespa.model.container.processing.Processor; +import org.w3c.dom.Element; + +import java.util.*; + +/** + * Creates all processing chains from xml. + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + * @since 5.1.6 + */ +public class ProcessingChainsBuilder extends ChainsBuilder<Processor, ProcessingChain> { + + private static final Map<String, Class<? extends DomChainBuilderBase<? extends Processor, ? extends ProcessingChain>>> + chainType2builderClass = Collections.unmodifiableMap( + new LinkedHashMap<String, Class<? extends DomChainBuilderBase<? extends Processor, ? extends ProcessingChain>>>() {{ + put("chain", DomProcessingChainBuilder.class); + }}); + + public ProcessingChainsBuilder(AbstractConfigProducer ancestor, List<Element> processingChainsElements, + Map<String, ComponentsBuilder.ComponentType> outerSearcherTypeByComponentName) { + super(ancestor, processingChainsElements, outerSearcherTypeByComponentName, chainType2builderClass); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilder.java new file mode 100644 index 00000000000..c06d62b7d43 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomFederationSearcherBuilder.java @@ -0,0 +1,88 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains.search; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import com.yahoo.search.searchchain.model.federation.FederationSearcherModel; +import com.yahoo.text.XML; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.builder.xml.dom.DomComponentBuilder; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.GenericChainedComponentModelBuilder; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.search.searchchain.FederationSearcher; +import com.yahoo.vespa.model.container.search.searchchain.Searcher; +import org.w3c.dom.Element; +import scala.Option; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * Builds a federation searcher config producer from an element. + * @author tonytv + */ +public class DomFederationSearcherBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Searcher<?>> { + static class FederationSearcherModelBuilder extends GenericChainedComponentModelBuilder { + private final List<FederationSearcherModel.TargetSpec> sources; + private final boolean inheritDefaultSources; + + FederationSearcherModelBuilder(Element searcherSpec) { + super(searcherSpec); + sources = readSources(searcherSpec); + inheritDefaultSources = readSourceSet(searcherSpec); + } + + private boolean readSourceSet(Element searcherSpec) { + return XML.getChild(searcherSpec, "source-set") != null; + } + + + private List<FederationSearcherModel.TargetSpec> readSources(Element searcherSpec) { + List<FederationSearcherModel.TargetSpec> sources = new ArrayList<>(); + for (Element source : XML.getChildren(searcherSpec, "source")) { + sources.add(readSource(source)); + } + return sources; + } + + private FederationSearcherModel.TargetSpec readSource(Element source) { + ComponentSpecification componentSpecification = XmlHelper.getIdRef(source); + + FederationOptions federationOptions = + readFederationOptions(XML.getChild(source, FederationOptionsBuilder.federationOptionsElement)); + + return new FederationSearcherModel.TargetSpec(componentSpecification, federationOptions); + } + + private FederationOptions readFederationOptions(Element federationOptionsElement) { + if (federationOptionsElement == null) { + return new FederationOptions(); + } else { + return new FederationOptionsBuilder(federationOptionsElement).build(); + } + } + + protected FederationSearcherModel build() { + return new FederationSearcherModel(componentId, dependencies, sources, inheritDefaultSources); + } + } + + protected FederationSearcher doBuild(AbstractConfigProducer ancestor, Element searcherElement) { + FederationSearcherModel model = new FederationSearcherModelBuilder(searcherElement).build(); + Optional<Component> targetSelector = buildTargetSelector(ancestor, searcherElement, model.getComponentId()); + + return new FederationSearcher(model, targetSelector); + } + + private Optional<Component> buildTargetSelector(AbstractConfigProducer ancestor, Element searcherElement, ComponentId namespace) { + Element targetSelectorElement = XML.getChild(searcherElement, "target-selector"); + if (targetSelectorElement == null) + return Optional.empty(); + + return Optional.of(new DomComponentBuilder(namespace).build(ancestor, targetSelectorElement)); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomGenericTargetBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomGenericTargetBuilder.java new file mode 100644 index 00000000000..59ecbe99b40 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomGenericTargetBuilder.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains.search; + +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainBuilderBase; +import com.yahoo.vespa.model.container.search.searchchain.GenericTarget; +import com.yahoo.vespa.model.container.search.searchchain.Searcher; +import org.w3c.dom.Element; + +import java.util.Arrays; +import java.util.Map; + +/** + * @author tonytv + * @author gjoranv + * Base functionality for all target chain builders (provider, source) + */ +abstract public class DomGenericTargetBuilder<T extends GenericTarget> extends DomChainBuilderBase<Searcher<?>, T> { + + DomGenericTargetBuilder(Map<String, ComponentsBuilder.ComponentType> outerSearcherTypeByComponentName) { + super(Arrays.asList(ComponentsBuilder.ComponentType.searcher, ComponentsBuilder.ComponentType.federation), + outerSearcherTypeByComponentName); + } + + protected static FederationOptions readFederationOptions(Element sourceElement) { + Element optionsElement = XML.getChild(sourceElement, FederationOptionsBuilder.federationOptionsElement); + if (optionsElement == null) { + return new FederationOptions(); + } else { + return new FederationOptionsBuilder(optionsElement).build(); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilder.java new file mode 100644 index 00000000000..bfd7dbc3a3c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomProviderBuilder.java @@ -0,0 +1,283 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains.search; + +import com.yahoo.binaryprefix.BinaryPrefix; +import com.yahoo.component.chain.dependencies.Dependencies; +import com.yahoo.component.chain.model.ChainedComponentModel; +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import com.yahoo.search.searchchain.model.federation.HttpProviderSpec; +import com.yahoo.search.searchchain.model.federation.LocalProviderSpec; +import com.yahoo.text.XML; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.builder.xml.dom.BinaryScaledAmountParser; +import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder; +import com.yahoo.vespa.model.container.search.searchchain.HttpProvider; +import com.yahoo.vespa.model.container.search.searchchain.HttpProviderSearcher; +import com.yahoo.vespa.model.container.search.searchchain.LocalProvider; +import com.yahoo.vespa.model.container.search.searchchain.Provider; +import com.yahoo.vespa.model.container.search.searchchain.Source; +import org.w3c.dom.Element; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * Builds a provider from xml. + * The demangling of provider types is taken care of here, + * since the mangling is an intrinsic of the configuration language, + * not the model itself. + * + * @author tonytv + */ +public class DomProviderBuilder extends DomGenericTargetBuilder<Provider> { + /** + * Retrieves all possible provider specific parameters + */ + private static class ProviderReader { + final String type; + final String path; + final Double cacheWeight; + final Integer retries; + final Double readTimeout; + final Double connectionTimeout; + final Double connectionPoolTimeout; + final String clusterName; + final List<HttpProviderSpec.Node> nodes; + final String ycaApplicationId; + final Integer ycaTtl; + final Integer ycaRetryWait; + final HttpProviderSpec.Node ycaProxy; // Just re-using the Node class, as it matches our needs + final Integer cacheSizeMB; + + ProviderReader(Element providerElement) { + type = readType(providerElement); + path = readPath(providerElement); + cacheWeight = readCacheWeight(providerElement); + cacheSizeMB = readCacheSize(providerElement); + clusterName = readCluster(providerElement); + readTimeout = readReadTimeout(providerElement); + connectionTimeout = readConnectionTimeout(providerElement); + connectionPoolTimeout = readConnectionPoolTimeout(providerElement); + retries = readRetries(providerElement); + nodes = readNodes(providerElement); + ycaApplicationId = readYcaApplicationId(providerElement); + ycaTtl = readYcaTtl(providerElement); + ycaRetryWait = readYcaRetryWait(providerElement); + ycaProxy = readYcaProxy(providerElement); + } + + + private String getAttributeOrNull(Element element, String name) { + String value = element.getAttribute(name); + return value.isEmpty() ? null : value; + } + + private String readPath(Element providerElement) { + return getAttributeOrNull(providerElement, "path"); + } + + private String readCluster(Element providerElement) { + return getAttributeOrNull(providerElement, "cluster"); + } + + private Double readCacheWeight(Element providerElement) { + String cacheWeightString = getAttributeOrNull(providerElement, "cacheweight"); + return (cacheWeightString == null)? null : Double.parseDouble(cacheWeightString); + } + + private Integer readCacheSize(Element providerElement) { + String cacheSize = getAttributeOrNull(providerElement, "cachesize"); + return (cacheSize == null)? null : (int)BinaryScaledAmountParser.parse(cacheSize).as(BinaryPrefix.mega); + } + + private Integer readRetries(Element providerElement) { + String retriesString = getAttributeOrNull(providerElement, "retries"); + return (retriesString == null) ? null : Integer.parseInt(retriesString); + } + + private Double readReadTimeout(Element providerElement) { + String timeoutString = getAttributeOrNull(providerElement, "readtimeout"); + return (timeoutString == null) ? null : TimeParser.seconds(timeoutString); + } + + private Double readConnectionTimeout(Element providerElement) { + String timeoutString = getAttributeOrNull(providerElement, "connectiontimeout"); + return (timeoutString == null) ? null : TimeParser.seconds(timeoutString); + } + + private Double readConnectionPoolTimeout(Element providerElement) { + String timeoutString = getAttributeOrNull(providerElement, "connectionpooltimeout"); + return (timeoutString == null) ? null : TimeParser.seconds(timeoutString); + } + + private String readYcaApplicationId(Element providerElement) { + return getAttributeOrNull(providerElement, "yca-application-id"); + } + + private Integer readYcaTtl(Element providerElement) { + String x = getAttributeOrNull(providerElement, "yca-cache-ttl"); + return (x == null) ? null : TimeParser.seconds(x).intValue(); + } + + private Integer readYcaRetryWait(Element providerElement) { + String x = getAttributeOrNull(providerElement, "yca-cache-retry-wait"); + return (x == null) ? null : TimeParser.seconds(x).intValue(); + } + + private HttpProviderSpec.Node readYcaProxy(Element providerElement) { + Element ycaProxySpec = XML.getChild(providerElement, "yca-proxy"); + if (ycaProxySpec == null) { + return null; // no proxy + } + if(getAttributeOrNull(ycaProxySpec, "host") == null) { + return new HttpProviderSpec.Node(null, 0); // default proxy + } + return readNode(ycaProxySpec); + } + + private List<HttpProviderSpec.Node> readNodes(Element providerElement) { + Element nodesSpec = XML.getChild(providerElement, "nodes"); + if (nodesSpec == null) { + return null; + } + + List<HttpProviderSpec.Node> nodes = new ArrayList<>(); + for (Element nodeSpec : XML.getChildren(nodesSpec, "node")) { + nodes.add(readNode(nodeSpec)); + } + return nodes; + } + + private HttpProviderSpec.Node readNode(Element nodeElement) { + String host = getAttributeOrNull(nodeElement, "host"); + // The direct calls to parse methods below works because the schema + // guarantees us no null references + int port = Integer.parseInt(getAttributeOrNull(nodeElement, "port")); + return new HttpProviderSpec.Node(host, port); + } + + private String readType(Element providerElement) { + return getAttributeOrNull(providerElement, "type"); + } + } + + public DomProviderBuilder(Map<String, ComponentsBuilder.ComponentType> outerSearcherTypeByComponentName) { + super(outerSearcherTypeByComponentName); + } + + @Override + protected Provider buildChain(AbstractConfigProducer ancestor, Element providerElement, + ChainSpecification specWithoutInnerComponents) { + + ProviderReader providerReader = new ProviderReader(providerElement); + if (providerReader.ycaApplicationId == null && providerReader.ycaProxy != null) { + throw new IllegalArgumentException( + "Provider '" + specWithoutInnerComponents.componentId + + "' must have a YCA application ID, since a YCA proxy is given"); + } + + FederationOptions federationOptions = readFederationOptions(providerElement); + + Provider provider = buildProvider(specWithoutInnerComponents, providerReader, federationOptions); + + Collection<Source> sources = buildSources(ancestor, providerElement); + addSources(provider, sources); + + return provider; + } + + + private Collection<Source> buildSources(AbstractConfigProducer ancestor, Element providerElement) { + List<Source> sources = new ArrayList<>(); + for (Element sourceElement : XML.getChildren(providerElement, "source")) { + sources.add(new DomSourceBuilder(outerComponentTypeByComponentName).build(ancestor, sourceElement)); + } + return sources; + } + + private void addSources(Provider provider, Collection<Source> sources) { + for (Source source : sources) { + provider.addSource(source); + } + } + + private Provider buildProvider(ChainSpecification specWithoutInnerSearchers, + ProviderReader providerReader, FederationOptions federationOptions) { + + if (providerReader.type == null) { + return buildEmptyHttpProvider(specWithoutInnerSearchers, providerReader, federationOptions); + } else if (HttpProviderSpec.includesType(providerReader.type)) { + return buildHttpProvider(specWithoutInnerSearchers, providerReader, federationOptions); + } else if (LocalProviderSpec.includesType(providerReader.type)) { + return buildLocalProvider(specWithoutInnerSearchers, providerReader, federationOptions); + } else { + throw new RuntimeException("Unknown provider type '" + providerReader.type + "'"); + } + } + + private Provider buildLocalProvider(ChainSpecification specWithoutInnerSearchers, ProviderReader providerReader, FederationOptions federationOptions) { + try { + ensureEmpty(specWithoutInnerSearchers.componentId, providerReader.cacheWeight, providerReader.path, providerReader.nodes, + providerReader.readTimeout, providerReader.connectionTimeout, providerReader.connectionPoolTimeout, + providerReader.retries, providerReader.ycaApplicationId, providerReader.ycaTtl, + providerReader.ycaRetryWait, providerReader.ycaProxy); + + return new LocalProvider(specWithoutInnerSearchers, + federationOptions, + new LocalProviderSpec(providerReader.clusterName, providerReader.cacheSizeMB)); + } catch (Exception e) { + throw new RuntimeException("Failed creating local provider " + specWithoutInnerSearchers.componentId, e); + } + } + + private Provider buildHttpProvider(ChainSpecification specWithoutInnerSearchers, ProviderReader providerReader, FederationOptions federationOptions) { + ensureEmpty(specWithoutInnerSearchers.componentId, providerReader.clusterName); + + Provider httpProvider = buildEmptyHttpProvider(specWithoutInnerSearchers, providerReader, federationOptions); + + httpProvider.addInnerComponent(new HttpProviderSearcher( + new ChainedComponentModel( + HttpProviderSpec.toBundleInstantiationSpecification(HttpProviderSpec.Type.valueOf(providerReader.type)), + Dependencies.emptyDependencies()))); + + return httpProvider; + } + + + private Provider buildEmptyHttpProvider(ChainSpecification specWithoutInnerSearchers, ProviderReader providerReader, FederationOptions federationOptions) { + ensureEmpty(specWithoutInnerSearchers.componentId, providerReader.clusterName); + + return new HttpProvider(specWithoutInnerSearchers, + federationOptions, + new HttpProviderSpec( + providerReader.cacheWeight, + providerReader.path, + providerReader.nodes, + providerReader.ycaApplicationId, + providerReader.ycaTtl, + providerReader.ycaRetryWait, + providerReader.ycaProxy, + providerReader.cacheSizeMB, + connectionParameters(providerReader))); + } + + private HttpProviderSpec.ConnectionParameters connectionParameters(ProviderReader providerReader) { + return new HttpProviderSpec.ConnectionParameters( + providerReader.readTimeout, + providerReader.connectionTimeout, + providerReader.connectionPoolTimeout, + providerReader.retries); + } + + private void ensureEmpty(ComponentId componentId, Object... objects) { + for (Object object : objects) { + if (object != null) { + throw new RuntimeException("Invalid provider option in provider '" + componentId + "': value='" + object + "'"); + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainBuilder.java new file mode 100644 index 00000000000..d79ea1a536b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainBuilder.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains.search; + +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainBuilderBase; +import com.yahoo.vespa.model.container.search.searchchain.SearchChain; +import com.yahoo.vespa.model.container.search.searchchain.Searcher; +import org.w3c.dom.Element; + +import java.util.Arrays; +import java.util.Map; + +/** + * Builds a Search chain from xml. + * @author tonytv + */ +public class DomSearchChainBuilder extends DomChainBuilderBase<Searcher<?>, SearchChain> { + + public DomSearchChainBuilder(Map<String, ComponentsBuilder.ComponentType> outerSearcherTypeByComponentName) { + super(Arrays.asList(ComponentsBuilder.ComponentType.searcher, ComponentsBuilder.ComponentType.federation), + outerSearcherTypeByComponentName); + } + + protected SearchChain buildChain(AbstractConfigProducer ancestor, Element producerSpec, + ChainSpecification specWithoutInnerComponents) { + return new SearchChain(specWithoutInnerComponents); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilder.java new file mode 100644 index 00000000000..5cadc8cc271 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearchChainsBuilder.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains.search; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder.ComponentType; +import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainsBuilder; +import com.yahoo.vespa.model.container.search.searchchain.SearchChain; +import com.yahoo.vespa.model.container.search.searchchain.SearchChains; +import com.yahoo.vespa.model.container.search.searchchain.Searcher; +import org.w3c.dom.Element; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * Builds the search chains model from xml. + * + * @author tonytv + */ +public class DomSearchChainsBuilder extends DomChainsBuilder<Searcher<?>, SearchChain, SearchChains> { + + public DomSearchChainsBuilder(Element outerChainsElem, boolean supportSearchChainsDir) { + super(outerChainsElem, Arrays.asList(ComponentType.searcher, ComponentType.federation), + supportSearchChainsDir ? ApplicationPackage.SEARCHCHAINS_DIR: null); + } + + // For unit testing without outer chains + public DomSearchChainsBuilder() { + this(null, false); + } + + @Override + protected SearchChains newChainsInstance(AbstractConfigProducer parent) { + return new SearchChains(parent, "searchchains"); + } + + @Override + protected SearchChainsBuilder readChains(AbstractConfigProducer ancestor, List<Element> searchChainsElements, + Map<String, ComponentType> outerComponentTypeByComponentName) { + return new SearchChainsBuilder(ancestor, searchChainsElements, outerComponentTypeByComponentName); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearcherBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearcherBuilder.java new file mode 100644 index 00000000000..44f4e83b1c8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSearcherBuilder.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains.search; + +import com.yahoo.component.chain.model.ChainedComponentModel; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.ChainedComponentModelBuilder; +import com.yahoo.vespa.model.container.search.searchchain.Searcher; +import org.w3c.dom.Element; + +/** + * Builds a Searcher from XML. + * @author tonytv + */ +public class DomSearcherBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Searcher<?>> { + + protected Searcher<ChainedComponentModel> doBuild(AbstractConfigProducer ancestor, Element searcherElement) { + ChainedComponentModelBuilder modelBuilder = new ChainedComponentModelBuilder(searcherElement); + return new Searcher<>(modelBuilder.build()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSourceBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSourceBuilder.java new file mode 100644 index 00000000000..090ac6a6df3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/DomSourceBuilder.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains.search; + +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder; +import com.yahoo.vespa.model.container.search.searchchain.Source; +import org.w3c.dom.Element; + +import java.util.Map; + +/** + * Builds a source from xml. + * @author tonytv + */ +public class DomSourceBuilder extends DomGenericTargetBuilder<Source> { + DomSourceBuilder(Map<String, ComponentsBuilder.ComponentType> outerSearcherTypeByComponentName) { + super(outerSearcherTypeByComponentName); + } + + protected Source buildChain(AbstractConfigProducer ancestor, Element producerSpec, ChainSpecification specWithoutInnerComponents) { + Source.GroupOption groupOption = + XmlHelper.isReference(producerSpec) ? + Source.GroupOption.participant : + Source.GroupOption.leader; + + return new Source(specWithoutInnerComponents, readFederationOptions(producerSpec), groupOption); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/FederationOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/FederationOptionsBuilder.java new file mode 100644 index 00000000000..ba84e66cb97 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/FederationOptionsBuilder.java @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains.search; + +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import org.w3c.dom.Element; + + +/** + * Builds federation options from a federations options element + * @author tonytv + */ +public class FederationOptionsBuilder { + public static final String federationOptionsElement = "federationoptions"; + + private final FederationOptions federationOptions; + + FederationOptionsBuilder(Element spec) { + federationOptions = + new FederationOptions(). + setUseByDefault(readUseByDefault(spec)). + setOptional(readOptional(spec)). + setTimeoutInMilliseconds(readTimeout(spec)). + setRequestTimeoutInMilliseconds(readRequestTimeout(spec)); + } + + + private Integer readTimeout(Element spec) { + String timeout = spec.getAttribute("timeout"); + + return (timeout.isEmpty())? + null : + TimeParser.asMilliSeconds(timeout); + } + + private Integer readRequestTimeout(Element spec) { + String requestTimeout = spec.getAttribute("requestTimeout"); + + return (requestTimeout.isEmpty())? + null : + TimeParser.asMilliSeconds(requestTimeout); + } + + private Boolean readOptional(Element spec) { + String optional = spec.getAttribute("optional"); + return (optional.isEmpty()) ? + null : + Boolean.parseBoolean(optional); + } + + private Boolean readUseByDefault(Element spec) { + String useByDefault = spec.getAttribute("default"); + return (useByDefault.isEmpty()) ? + null : + Boolean.parseBoolean(useByDefault); + } + + FederationOptions build() { + return federationOptions; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/SearchChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/SearchChainsBuilder.java new file mode 100644 index 00000000000..9b4a8908afe --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/SearchChainsBuilder.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains.search; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.builder.xml.dom.chains.ChainsBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainBuilderBase; +import com.yahoo.vespa.model.container.search.searchchain.SearchChain; +import com.yahoo.vespa.model.container.search.searchchain.Searcher; +import org.w3c.dom.Element; + +import java.util.*; + +/** + * @author tonytv + * @author gjoranv + * Creates top level search chains(searchchain, provider) from xml. + */ +public class SearchChainsBuilder extends ChainsBuilder<Searcher<?>, SearchChain> { + + private static final Map<String, Class<? extends DomChainBuilderBase<? extends Searcher<?>, ? extends SearchChain>>> + chainType2builderClass = Collections.unmodifiableMap( + new LinkedHashMap<String, Class<? extends DomChainBuilderBase<? extends Searcher<?>, ? extends SearchChain>>>() {{ + put("chain", DomSearchChainBuilder.class); + put("searchchain", DomSearchChainBuilder.class); + put("provider", DomProviderBuilder.class); + }}); + + public SearchChainsBuilder(AbstractConfigProducer ancestor, List<Element> searchChainsElements, + Map<String, ComponentsBuilder.ComponentType> outerSearcherTypeByComponentName) { + super(ancestor, searchChainsElements, outerSearcherTypeByComponentName, chainType2builderClass); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/TimeParser.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/TimeParser.java new file mode 100644 index 00000000000..7aa84d3a14b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/chains/search/TimeParser.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.builder.xml.dom.chains.search; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Utility class for parsing timeout fields. + * + * @author tonytv + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class TimeParser { + private static final Pattern timeoutPattern = Pattern.compile("(\\d+(\\.\\d*)?)\\s*(m)?s"); + private static final double milliSecondsPerSecond = 1000.0d; + + public static Double seconds(String timeout) { + Matcher matcher = timeoutPattern.matcher(timeout); + if (!matcher.matches()) { + throw new RuntimeException("Timeout pattern not in sync with schema"); + } + + double value = Double.parseDouble(matcher.group(1)); + if (matcher.group(3) != null) { + value /= milliSecondsPerSecond; + } + return new Double(value); + } + + public static int asMilliSeconds(String timeout) { + Matcher matcher = timeoutPattern.matcher(timeout); + if (!matcher.matches()) { + throw new RuntimeException("Timeout pattern not in sync with schema"); + } + + double value = Double.parseDouble(matcher.group(1)); + if (matcher.group(3) == null) { + value *= milliSecondsPerSecond; + } + + return (int) value; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/package-info.java new file mode 100644 index 00000000000..e51d7e4acee --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.model.builder.xml.dom; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java new file mode 100644 index 00000000000..799d0b437ae --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/Clients.java @@ -0,0 +1,77 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.clients; + +import com.yahoo.vespa.config.content.LoadTypeConfig; +import com.yahoo.config.model.ConfigModel; +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.documentapi.messagebus.loadtypes.LoadType; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.search.ContainerHttpGateway; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; + +/** + * This is the clients plugin for the Vespa model. It is responsible for creating + * all clients services. + * + * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a> + */ +public class Clients extends ConfigModel { + + private static final long serialVersionUID = 1L; + private ContainerCluster containerHttpGateways = null; + private List<VespaSpoolerService> vespaSpoolers = new LinkedList<>(); + private LoadTypeSet loadTypes = new LoadTypeSet(); + private final AbstractConfigProducer parent; + + public Clients(ConfigModelContext modelContext) { + super(modelContext); + this.parent = modelContext.getParentProducer(); + } + + public AbstractConfigProducer getConfigProducer() { + return parent; + } + + @Override + public void prepare(ConfigModelRepo configModelRepo) { + if (containerHttpGateways != null) { + containerHttpGateways.prepare(); + } + } + + public void setContainerHttpGateways(ContainerCluster containerHttpGateways) { + this.containerHttpGateways = containerHttpGateways; + } + + public List<VespaSpoolerService> getVespaSpoolers() { + return vespaSpoolers; + } + + public LoadTypeSet getLoadTypes() { + return loadTypes; + } + + public void getConfig(LoadTypeConfig.Builder builder) { + for (LoadType t : loadTypes.getNameMap().values()) { + if (t != LoadType.DEFAULT) { + builder.type(getLoadTypeConfig(t)); + } + } + } + + private LoadTypeConfig.Type.Builder getLoadTypeConfig(LoadType loadType) { + LoadTypeConfig.Type.Builder builder = new LoadTypeConfig.Type.Builder(); + builder.name(loadType.getName()); + builder.id(loadType.getId()); + builder.priority(loadType.getPriority().toString()); + return builder; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java new file mode 100644 index 00000000000..55cfc8b2fba --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java @@ -0,0 +1,165 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.clients; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.Phase; +import com.yahoo.component.chain.dependencies.Dependencies; +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.component.chain.model.ChainedComponentModel; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.Handler; +import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; +import com.yahoo.vespa.model.container.search.ContainerSearch; +import com.yahoo.vespa.model.container.search.searchchain.SearchChain; +import com.yahoo.vespa.model.container.search.searchchain.SearchChains; +import com.yahoo.vespa.model.container.search.searchchain.Searcher; +import com.yahoo.vespaclient.config.FeederConfig; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Set; +import java.util.TreeSet; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.11 + */ +public class ContainerDocumentApi implements FeederConfig.Producer { + + public static final String vespaClientBundleSpecification = "vespaclient-container-plugin"; + private final Options options; + + public ContainerDocumentApi(ContainerCluster cluster, Options options) { + this.options = options; + setupLegacySearchers(cluster); + setupHandlers(cluster); + } + + private void setupHandlers(ContainerCluster cluster) { + cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandler", "feed")); + cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerRemove", "remove")); + cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation", "removelocation")); + cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerGet", "get")); + cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerVisit", "visit")); + cluster.addComponent(newVespaClientHandler("com.yahoo.document.restapi.resource.RestApi", "document/v1/*")); + cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerCompatibility", "document")); + cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerStatus", "feedstatus")); + cluster.addComponent(newVespaClientHandler("com.yahoo.vespa.http.server.FeedHandler", ContainerCluster.RESERVED_URI_PREFIX + "/feedapi")); + } + + private void setupLegacySearchers(ContainerCluster cluster) { + Set<ComponentSpecification> inherited = new TreeSet<>(); + + SearchChain vespaGetChain = new SearchChain(new ChainSpecification(new ComponentId("vespaget"), + new ChainSpecification.Inheritance(inherited, null), new ArrayList<Phase>(), new TreeSet<ComponentSpecification>())); + vespaGetChain.addInnerComponent(newVespaClientSearcher("com.yahoo.storage.searcher.GetSearcher")); + + SearchChain vespaVisitChain = new SearchChain(new ChainSpecification(new ComponentId("vespavisit"), + new ChainSpecification.Inheritance(inherited, null), new ArrayList<Phase>(), new TreeSet<ComponentSpecification>())); + vespaVisitChain.addInnerComponent(newVespaClientSearcher("com.yahoo.storage.searcher.VisitSearcher")); + + SearchChains chains; + if (cluster.getSearch() != null) { + chains = cluster.getSearchChains(); + } else { + chains = new SearchChains(cluster, "searchchain"); + } + chains.add(vespaGetChain); + chains.add(vespaVisitChain); + + if (cluster.getSearch() == null) { + ContainerSearch containerSearch = new ContainerSearch(cluster, chains, new ContainerSearch.Options()); + cluster.setSearch(containerSearch); + + final ProcessingHandler<SearchChains> searchHandler = new ProcessingHandler<>( + chains, "com.yahoo.search.handler.SearchHandler"); + searchHandler.addServerBindings("http://*/search/*", "https://*/search/*"); + cluster.addComponent(searchHandler); + } + } + + private Handler newVespaClientHandler(String componentId, String bindingSuffix) { + Handler<AbstractConfigProducer<?>> handler = new Handler<>(new ComponentModel( + BundleInstantiationSpecification.getFromStrings(componentId, null, vespaClientBundleSpecification), "")); + + for (String rootBinding : options.bindings) { + handler.addServerBindings(rootBinding + bindingSuffix, + rootBinding + bindingSuffix + '/'); + } + return handler; + } + + private Searcher newVespaClientSearcher(String componentSpec) { + return new Searcher<>(new ChainedComponentModel( + BundleInstantiationSpecification.getFromStrings(componentSpec, null, vespaClientBundleSpecification), + new Dependencies(null, null, null))); + } + + @Override + public void getConfig(FeederConfig.Builder builder) { + if (options.abortondocumenterror != null) + builder.abortondocumenterror(options.abortondocumenterror); + if (options.route!= null) + builder.route(options.route); + if (options.maxpendingdocs != null) + builder.maxpendingdocs(options.maxpendingdocs); + if (options.maxpendingbytes != null) + builder.maxpendingbytes(options.maxpendingbytes); + if (options.retryenabled != null) + builder.retryenabled(options.retryenabled); + if (options.retrydelay != null) + builder.retrydelay(options.retrydelay); + if (options.timeout != null) + builder.timeout(options.timeout); + if (options.tracelevel != null) + builder.tracelevel(options.tracelevel); + if (options.mbusport != null) + builder.mbusport(options.mbusport); + if (options.docprocChain != null) + builder.docprocchain(options.docprocChain); + } + + public static final class Options { + private final Collection<String> bindings; + private final Boolean abortondocumenterror; + private final String route; + private final Integer maxpendingdocs; + private final Integer maxpendingbytes; + private final Boolean retryenabled; + private final Double retrydelay; + private final Double timeout; + private final Integer tracelevel; + private final Integer mbusport; + private final String docprocChain; + + public Options(Collection<String> bindings, + Boolean abortondocumenterror, + String route, + Integer maxpendingdocs, + Integer maxpendingbytes, + Boolean retryenabled, + Double retrydelay, + Double timeout, + Integer tracelevel, + Integer mbusport, + String docprocChain) { + + this.bindings = Collections.unmodifiableCollection(bindings); + this.abortondocumenterror = abortondocumenterror; + this.route = route; + this.maxpendingdocs = maxpendingdocs; + this.maxpendingbytes = maxpendingbytes; + this.retryenabled = retryenabled; + this.retrydelay = retrydelay; + this.timeout = timeout; + this.tracelevel = tracelevel; + this.mbusport = mbusport; + this.docprocChain = docprocChain; + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/HttpGatewayOwner.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/HttpGatewayOwner.java new file mode 100644 index 00000000000..b5118e6c6c7 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/HttpGatewayOwner.java @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.clients; + +import com.yahoo.config.subscription.ConfigInstanceUtil; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespaclient.config.FeederConfig; + +public class HttpGatewayOwner extends AbstractConfigProducer implements FeederConfig.Producer { + private final FeederConfig.Builder feederConfig; + + public HttpGatewayOwner(AbstractConfigProducer parent, FeederConfig.Builder feederConfig) { + super(parent, "gateways"); + this.feederConfig = feederConfig; + } + + @Override + public void getConfig(FeederConfig.Builder builder) { + ConfigInstanceUtil.setValues(builder, feederConfig); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolMaster.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolMaster.java new file mode 100644 index 00000000000..1546b2d8433 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolMaster.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.clients; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.AbstractService; + +/** + * The spoolmaster program, which is used when multiple spooler instances are used to provide + * multi colo HTTP feeding. + * @author vegardh + * + */ +public class VespaSpoolMaster extends AbstractService { + + public VespaSpoolMaster(AbstractConfigProducer parent, int index) { + super(parent, "spoolmaster."+index); + } + + @Override + public int getPortCount() { + // TODO Auto-generated method stub + return 0; + } + + @Override + public String getStartupCommand() { + return "exec spoolmaster"; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpooler.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpooler.java new file mode 100644 index 00000000000..7a4ec77b2da --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpooler.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.clients; + +import com.yahoo.vespa.config.content.spooler.SpoolerConfig; +import com.yahoo.config.subscription.ConfigInstanceUtil; +import com.yahoo.vespaclient.config.FeederConfig; + +/** + * Holds configuration for VespaSpoolers. Actual services use VespaSpoolerService, + * while virtual services can be generated for external spoolers (VespaSpoolerProducer). + * + * @author <a href="mailto:thomasg@yahoo-inc.com">Gunnar Gauslaa Bergem</a> + * @author Vidar Larsen + */ +public class VespaSpooler { + private final SpoolerConfig.Builder spoolConfig; + private final FeederConfig.Builder feederConfig; + + public VespaSpooler(FeederConfig.Builder feederConfig, SpoolerConfig.Builder spoolConfig) { + this.feederConfig = feederConfig; + this.spoolConfig = spoolConfig; + } + + public void getConfig(SpoolerConfig.Builder builder) { + ConfigInstanceUtil.setValues(builder, spoolConfig); + } + + public void getConfig(FeederConfig.Builder builder) { + ConfigInstanceUtil.setValues(builder, feederConfig); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerProducer.java new file mode 100644 index 00000000000..671603553ec --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerProducer.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.clients; + +import com.yahoo.vespa.config.content.spooler.SpoolerConfig; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespaclient.config.FeederConfig; + +/** + * This model represents a config producer for spooler used for feeding documents to Vespa. + * + * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a> + * @author Vidar Larsen + */ +public class VespaSpoolerProducer extends AbstractConfigProducer implements SpoolerConfig.Producer, FeederConfig.Producer { + private static final long serialVersionUID = 1L; + private VespaSpooler spoolerConfig; + + public VespaSpoolerProducer(AbstractConfigProducer parent, String configId, VespaSpooler spooler) { + super(parent, configId); + spoolerConfig = spooler; + } + + @Override + public void getConfig(SpoolerConfig.Builder builder) { + spoolerConfig.getConfig(builder); + } + + @Override + public void getConfig(FeederConfig.Builder builder) { + spoolerConfig.getConfig(builder); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerService.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerService.java new file mode 100644 index 00000000000..8e62b797441 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/VespaSpoolerService.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.clients; + +import com.yahoo.vespa.config.content.spooler.SpoolerConfig; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespaclient.config.FeederConfig; + +/** + * This model represents a spooler used for feeding documents to Vespa. + * + * @author <a href="mailto:gunnarga@yahoo-inc.com">Gunnar Gauslaa Bergem</a> + * @author Vidar Larsen + */ +public class VespaSpoolerService extends AbstractService implements SpoolerConfig.Producer, FeederConfig.Producer { + private static final long serialVersionUID = 1L; + private VespaSpooler spooler; + + public VespaSpoolerService(AbstractConfigProducer parent, int index, VespaSpooler spooler) { + super(parent, "spooler." + index); + this.spooler = spooler; + monitorService("spooler"); + } + + public int getPortCount() { + return 0; + } + + public String getStartupCommand() { + return "exec vespaspooler "+getJvmArgs(); + } + + @Override + public void getConfig(SpoolerConfig.Builder builder) { + spooler.getConfig(builder); + } + + @Override + public void getConfig(FeederConfig.Builder builder) { + spooler.getConfig(builder); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java new file mode 100644 index 00000000000..a13c7c9cec4 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -0,0 +1,408 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.container.ComponentsConfig; +import com.yahoo.container.QrConfig; +import com.yahoo.container.core.ContainerHttpConfig; +import com.yahoo.container.jdisc.ContainerMbusConfig; +import com.yahoo.container.jdisc.JdiscBindingsConfig; +import com.yahoo.container.jdisc.config.PortOverridesConfig; +import com.yahoo.search.config.QrStartConfig; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.application.validation.RestartConfigs; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.ComponentGroup; +import com.yahoo.vespa.model.container.component.ComponentsConfigGenerator; +import com.yahoo.vespa.model.container.component.DiscBindingsConfigGenerator; +import com.yahoo.vespa.model.container.component.Handler; +import com.yahoo.vespa.model.container.component.SimpleComponent; +import com.yahoo.vespa.model.container.http.ConnectorFactory; +import com.yahoo.vespa.model.container.http.Http; +import com.yahoo.vespa.model.container.http.JettyHttpServer; +import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + +import static com.yahoo.container.QrConfig.Filedistributor; +import static com.yahoo.container.QrConfig.Rpc; + + +/** + * @author gjoranv + * @author einarmr + * @author tonytv + */ +//qr is restart because it is handled by ConfiguredApplication.start +@RestartConfigs({QrStartConfig.class, QrConfig.class}) +public class Container extends AbstractService implements + QrConfig.Producer, + ComponentsConfig.Producer, + JdiscBindingsConfig.Producer, + ContainerHttpConfig.Producer, + PortOverridesConfig.Producer, + ContainerMbusConfig.Producer +{ + public static final class PortOverride { + public final ComponentSpecification serverId; + public final int port; + + public PortOverride(ComponentSpecification serverId, int port) { + this.serverId = serverId; + this.port = port; + } + } + + public static final int BASEPORT = Defaults.getDefaults().vespaWebServicePort(); + public static final String SINGLENODE_CONTAINER_SERVICESPEC = "default_singlenode_container"; + + private final AbstractConfigProducer parent; + private final String name; + private String clusterName = null; + private boolean rpcServerEnabled = true; + + // TODO: move these up to cluster + private boolean httpServerEnabled = true; + private boolean messageBusEnabled = true; + + private final boolean retired; + + private final ComponentGroup<Handler<?>> handlers = new ComponentGroup<>(this, "handler"); + private final ComponentGroup<Component<?, ?>> components = new ComponentGroup(this, "components"); + + private final JettyHttpServer defaultHttpServer = new JettyHttpServer(new ComponentId("DefaultHttpServer")); + + private final List<PortOverride> portOverrides; + + private final int numHttpServerPorts; + private final int numRpcServerPorts = 2; + private static String defaultHostedJVMArgs = "-XX:+UseOSErrorReporting -XX:+SuppressFatalErrorMessage"; + + public Container(AbstractConfigProducer parent, String name) { + this(parent, name, Collections.<PortOverride>emptyList()); + } + public Container(AbstractConfigProducer parent, String name, boolean retired) { + this(parent, name, retired, Collections.<PortOverride>emptyList()); + } + public Container(AbstractConfigProducer parent, String name, List<PortOverride> portOverrides) { + this(parent, name, false, portOverrides); + } + public Container(AbstractConfigProducer parent, String name, boolean retired, List<PortOverride> portOverrides) { + super(parent, name); + this.name = name; + this.parent = parent; + this.portOverrides = Collections.unmodifiableList(new ArrayList<>(portOverrides)); + this.retired = retired; + + if (getHttp() == null) { + numHttpServerPorts = 2; + addChild(defaultHttpServer); + } else if (getHttp().getHttpServer() == null) { + numHttpServerPorts = 0; + } else { + numHttpServerPorts = getHttp().getHttpServer().getConnectorFactories().size(); + } + addBuiltinHandlers(); + + addChild(new SimpleComponent("com.yahoo.container.jdisc.ConfiguredApplication$ApplicationContext")); + addChild(new SimpleComponent("com.yahoo.container.jdisc.ContainerPortsOverride")); + } + + /** True if this container is retired (slated for removal) */ + public boolean isRetired() { return retired; } + + public ComponentGroup<Handler<?>> getHandlers() { + return handlers; + } + + public ComponentGroup getComponents() { + return components; + } + + public void addComponent(Component c) { + components.addComponent(c); + } + + public void addHandler(Handler h) { + handlers.addComponent(h); + } + + public Http getHttp() { + return (parent instanceof ContainerCluster) ? + ((ContainerCluster) parent).getHttp(): + null; + } + + public JettyHttpServer getDefaultHttpServer() { + return defaultHttpServer; + } + + // We cannot set bindings yet, as baseport is not initialized + public void addBuiltinHandlers() { + } + + @Override + public void initService() { + // XXX: Must be called first, to set the baseport + super.initService(); + + if (getHttp() == null) { + initDefaultJettyConnector(); + } else { + reserveHttpPortsPrepended(); + } + + tagServers(); + monitorService(); + } + + private void tagServers() { + if (numHttpServerPorts > 0) { + portsMeta.on(0).tag("http").tag("query").tag("external").tag("state"); + } + + for (int i = 1; i < numHttpServerPorts; i++) + portsMeta.on(i).tag("http").tag("external"); + + if (rpcServerEnabled) { + portsMeta.on(numHttpServerPorts + 0).tag("rpc").tag("messaging"); + portsMeta.on(numHttpServerPorts + 1).tag("rpc").tag("admin"); + } + } + + private void reserveHttpPortsPrepended() { + if (getHttp().getHttpServer() != null) { + for (ConnectorFactory connectorFactory : getHttp().getHttpServer().getConnectorFactories()) { + reservePortPrepended(getPort(connectorFactory, portOverrides)); + } + } + } + + private int getPort(ConnectorFactory connectorFactory, List<PortOverride> portOverrides) { + ComponentId id = ComponentId.fromString(connectorFactory.getName()); + for (PortOverride override : portOverrides) { + if (override.serverId.matches(id)) { + return override.port; + } + } + return connectorFactory.getListenPort(); + } + + private void initDefaultJettyConnector() { + defaultHttpServer.addConnector(new ConnectorFactory("SearchServer", getSearchPort(), null)); + } + + private boolean hasDocproc() { + return (parent instanceof ContainerCluster) && (((ContainerCluster)parent).getDocproc() != null); + } + + // TODO: hack to retain old service names, e.g. in ymon config, vespa.log etc. + @Override + public String getServiceType() { + if (parent instanceof ContainerCluster) { + ContainerCluster cluster = (ContainerCluster)parent; + if (cluster.getSearch() != null && cluster.getDocproc() == null && cluster.getDocumentApi() == null) { + return "qrserver"; + } + if (cluster.getSearch() == null && cluster.getDocproc() != null) { + return "docprocservice"; + } + } + return super.getServiceType(); + } + + public void setClusterName(String name) { + this.clusterName = name; + } + + @Override + public int getWantedPort() { + return getHttp() == null ? + BASEPORT: + 0; + } + + /** + * First Qrserver or container must run on ports familiar to the user. + */ + @Override + public boolean requiresWantedPort() { + return getHttp() == null; + + } + + public boolean requiresConsecutivePorts() { + return false; + } + + /** + * @return the number of ports needed by the Container - those reserved manually(reservePortPrepended) + */ + public int getPortCount() { + int httpPorts = (getHttp() != null) ? 0 : numHttpServerPorts + 2; // TODO remove +2, only here to keep irrelevant unit tests from failing. + int rpcPorts = (isRpcServerEnabled()) ? numRpcServerPorts : 0; + return httpPorts + rpcPorts; + } + + /** + * @return the actual search port + * TODO: Remove. Use {@link #getPortsMeta()} and check tags in conjunction with {@link #getRelativePort(int)}. + */ + public int getSearchPort(){ + if (getHttp() != null) + throw new AssertionError("getSearchPort must not be used when http section is present."); + + return getRelativePort(0); + } + + private int getRpcPort() { + return isRpcServerEnabled() ? + getRelativePort(numHttpServerPorts + 1) : + 0; + } + + private int getMessagingPort() { + return getRelativePort(numHttpServerPorts); + } + + @Override + public int getHealthPort() { + final Http http = getHttp(); + if (http != null) { + // TODO: allow the user to specify health port manually + if (http.getHttpServer() == null) { + return -1; + } else { + return getRelativePort(0); + } + } else { + return httpServerEnabled ? getSearchPort() : -1; + } + } + + public String getStartupCommand() { + return "PRELOAD=" + getPreLoad() + " exec vespa-start-container-daemon " + getJvmArgs() + " "; + } + + public boolean isRpcServerEnabled() { + return rpcServerEnabled; + } + + @Override + public void getConfig(PortOverridesConfig.Builder builder) { + for (PortOverride portOverride: portOverrides) { + builder.server(new PortOverridesConfig.Server.Builder(). + id(portOverride.serverId.stringValue()). + port(portOverride.port)); + } + } + + @Override + public void getConfig(QrConfig.Builder builder) { + builder. + rpc(new Rpc.Builder() + .enabled(isRpcServerEnabled()) + .port(getRpcPort()) + .slobrokId(serviceSlobrokId())). + filedistributor(filedistributorConfig()); + if (clusterName != null) { + builder.discriminator(clusterName+"."+name); + } else { + builder.discriminator(name); + } + } + + @Override + public String getJvmArgs() { + String jvmArgs = super.getJvmArgs(); + return isHostedVespa() && hasDocproc() + ? ("".equals(jvmArgs) ? defaultHostedJVMArgs : defaultHostedJVMArgs + " " + jvmArgs) + : jvmArgs; + } + private String serviceSlobrokId() { + return "vespa/service/" + getConfigId(); + } + + private Filedistributor.Builder filedistributorConfig() { + Filedistributor.Builder builder = new Filedistributor.Builder(); + + FileDistributionConfigProducer fileDistribution = getRoot().getFileDistributionConfigProducer(); + if (fileDistribution != null) + builder.configid(fileDistribution.getFileDistributionServiceConfigId(getHost())); + + return builder; + } + + @Override + public void getConfig(ComponentsConfig.Builder builder) { + builder.components.addAll( + ComponentsConfigGenerator.generate(allEnabledComponents())); + } + + private Collection<Component<?, ?>> allEnabledComponents() { + Collection<Component<?, ?>> allComponents = new ArrayList<>(); + addAllEnabledComponents(allComponents, this); + return Collections.unmodifiableCollection(allComponents); + } + + private void addAllEnabledComponents(Collection<Component<?, ?>> allComponents, AbstractConfigProducer<?> current) { + for (AbstractConfigProducer<?> child: current.getChildren().values()) { + if (!httpServerEnabled && isHttpServer(child)) + continue; + + if (child instanceof Component) { + allComponents.add((Component<?, ?>) child); + } + + addAllEnabledComponents(allComponents, child); + } + } + + private boolean isHttpServer(AbstractConfigProducer<?> component) { + return component instanceof JettyHttpServer; + } + + @Override + public final void getConfig(JdiscBindingsConfig.Builder builder) { + builder.handlers(DiscBindingsConfigGenerator.generate(handlers.getComponents())); + } + + @Override + public void getConfig(ContainerHttpConfig.Builder builder) { + builder + .enabled(httpServerEnabled) + .port(new ContainerHttpConfig.Port.Builder() + .search(getSearchPort())); + } + + @Override + public void getConfig(ContainerMbusConfig.Builder builder) { + builder.enabled(messageBusEnabled). + port(getMessagingPort()); + } + + @Override + public HashMap<String,String> getDefaultMetricDimensions(){ + HashMap<String, String> dimensions = new HashMap<>(); + if (clusterName != null) { + dimensions.put("clustername", clusterName); + } + return dimensions; + } + + public void setRpcServerEnabled(boolean rpcServerEnabled) { + this.rpcServerEnabled = rpcServerEnabled; + } + + public void setHttpServerEnabled(boolean httpServerEnabled) { + this.httpServerEnabled = httpServerEnabled; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java new file mode 100755 index 00000000000..ae158567587 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -0,0 +1,816 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container; + +import com.yahoo.cloud.config.ClusterInfoConfig; +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.cloud.config.RoutingProviderConfig; +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.docproc.DocprocConfig; +import com.yahoo.config.docproc.SchemamappingConfig; +import com.yahoo.config.application.api.ApplicationMetaData; +import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.model.ApplicationConfigProducerRoot; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.model.producer.AbstractConfigProducerRoot; +import com.yahoo.config.provision.Rotation; +import com.yahoo.config.provision.Zone; +import com.yahoo.container.BundlesConfig; +import com.yahoo.container.ComponentsConfig; +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.container.core.ApplicationMetadataConfig; +import com.yahoo.container.core.document.ContainerDocumentConfig; +import com.yahoo.container.handler.ThreadPoolProvider; +import com.yahoo.container.jdisc.ContainerMbusConfig; +import com.yahoo.container.jdisc.JdiscBindingsConfig; +import com.yahoo.container.jdisc.config.HealthMonitorConfig; +import com.yahoo.container.jdisc.config.MetricDefaultsConfig; +import com.yahoo.container.jdisc.messagebus.MbusServerProvider; +import com.yahoo.container.jdisc.state.StateHandler; +import com.yahoo.container.logging.AccessLog; +import com.yahoo.container.usability.BindingsOverviewHandler; +import com.yahoo.container.xml.providers.DatatypeFactoryProvider; +import com.yahoo.container.xml.providers.DocumentBuilderFactoryProvider; +import com.yahoo.container.xml.providers.JAXBContextFactoryProvider; +import com.yahoo.container.xml.providers.SAXParserFactoryProvider; +import com.yahoo.container.xml.providers.SchemaFactoryProvider; +import com.yahoo.container.xml.providers.TransformerFactoryProvider; +import com.yahoo.container.xml.providers.XMLEventFactoryProvider; +import com.yahoo.container.xml.providers.XMLInputFactoryProvider; +import com.yahoo.container.xml.providers.XMLOutputFactoryProvider; +import com.yahoo.container.xml.providers.XPathFactoryProvider; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.jdisc.http.ServletPathsConfig; +import com.yahoo.metrics.simple.runtime.MetricProperties; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.prelude.semantics.SemanticRulesConfig; +import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.search.config.QrStartConfig; +import com.yahoo.search.pagetemplates.PageTemplatesConfig; +import com.yahoo.search.query.profile.config.QueryProfilesConfig; +import com.yahoo.vespa.configdefinition.IlscriptsConfig; +import com.yahoo.vespa.model.PortsMeta; +import com.yahoo.vespa.model.Service; +import com.yahoo.vespa.model.admin.MonitoringSystem; +import com.yahoo.vespa.model.clients.ContainerDocumentApi; +import com.yahoo.vespa.model.container.component.AccessLogComponent; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.ComponentGroup; +import com.yahoo.vespa.model.container.component.ComponentsConfigGenerator; +import com.yahoo.vespa.model.container.component.ConfigProducerGroup; +import com.yahoo.vespa.model.container.component.DiscBindingsConfigGenerator; +import com.yahoo.vespa.model.container.component.Handler; +import com.yahoo.vespa.model.container.component.SimpleComponent; +import com.yahoo.vespa.model.container.component.Servlet; +import com.yahoo.vespa.model.container.component.StatisticsComponent; +import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; +import com.yahoo.vespa.model.container.docproc.ContainerDocproc; +import com.yahoo.vespa.model.container.docproc.DocprocChains; +import com.yahoo.vespa.model.container.http.Http; +import com.yahoo.vespa.model.container.jersey.Jersey2Servlet; +import com.yahoo.vespa.model.container.jersey.JerseyHandler; +import com.yahoo.vespa.model.container.jersey.RestApi; +import com.yahoo.vespa.model.container.processing.ProcessingChains; +import com.yahoo.vespa.model.container.search.ContainerSearch; +import com.yahoo.vespa.model.container.search.searchchain.SearchChains; +import com.yahoo.vespa.model.content.Content; +import com.yahoo.vespa.model.search.AbstractSearchCluster; +import com.yahoo.vespa.model.utils.FileSender; +import com.yahoo.vespaclient.config.FeederConfig; + +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; + +import java.io.Reader; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.yahoo.container.core.BundleLoaderProperties.DISK_BUNDLE_PREFIX; + +/** + * @author gjoranv + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author tonytv + */ +public final class ContainerCluster + extends AbstractConfigProducer<AbstractConfigProducer<?>> + implements + ComponentsConfig.Producer, + JdiscBindingsConfig.Producer, + DocumentmanagerConfig.Producer, + ContainerMbusConfig.Producer, + ContainerDocumentConfig.Producer, + HealthMonitorConfig.Producer, + ApplicationMetadataConfig.Producer, + BundlesConfig.Producer, + FeederConfig.Producer, + IndexInfoConfig.Producer, + IlscriptsConfig.Producer, + SchemamappingConfig.Producer, + QrSearchersConfig.Producer, + QrStartConfig.Producer, + QueryProfilesConfig.Producer, + PageTemplatesConfig.Producer, + SemanticRulesConfig.Producer, + DocprocConfig.Producer, + MetricDefaultsConfig.Producer, + ClusterInfoConfig.Producer, + ServletPathsConfig.Producer, + RoutingProviderConfig.Producer, + ConfigserverConfig.Producer { + /** + * URI prefix used for internal, usually programmatic, APIs. URIs using this + * prefix should never considered available for direct use by customers, and + * normal compatibility concerns only applies to libraries using the URIs in + * question, not contents served from the URIs themselves. + */ + public static final String RESERVED_URI_PREFIX = "reserved-for-internal-use"; + + public static final String ROOT_HANDLER_BINDING = "*://*/"; + + private String name; + + private List<Container> containers = new ArrayList<>(); + + private Http http; + private ProcessingChains processingChains; + private ContainerSearch containerSearch; + private ContainerDocproc containerDocproc; + private ContainerDocumentApi containerDocumentApi; + + private MbusParams mbusParams; + + private final Set<FileReference> applicationBundles = new LinkedHashSet<>(); + private final Set<Path> platformBundles = new LinkedHashSet<>(); + + private final List<String> serviceAliases = new ArrayList<>(); + private final List<String> endpointAliases = new ArrayList<>(); + protected final ComponentGroup<Component<?, ?>> componentGroup; + private final ConfigProducerGroup<RestApi> restApiGroup; + private final ConfigProducerGroup<Servlet> servletGroup; + + private Map<String, String> concreteDocumentTypes = new LinkedHashMap<>(); + private MetricDefaultsConfig.Factory.Enum defaultMetricConsumerFactory; + + private ApplicationMetaData applicationMetaData = null; + + /** The zone this is deployed in, or the default zone if not on hosted Vespa */ + private Zone zone; + + public ContainerCluster(AbstractConfigProducer<?> parent, String subId, String name) { + super(parent, subId); + this.name = name; + this.zone = getRoot() != null ? getRoot().getDeployState().zone() : Zone.defaultZone(); + componentGroup = new ComponentGroup<>(this, "component"); + restApiGroup = new ConfigProducerGroup<>(this, "rest-api"); + servletGroup = new ConfigProducerGroup<>(this, "servlet"); + + addComponent(new StatisticsComponent()); + addSimpleComponent(AccessLog.class); + // TODO better modelling + addSimpleComponent(ThreadPoolProvider.class); + addSimpleComponent("com.yahoo.jdisc.http.filter.SecurityFilterInvoker"); + addSimpleComponent("com.yahoo.language.provider.SimpleLinguisticsProvider"); + addSimpleComponent("com.yahoo.container.jdisc.SslKeyStoreFactoryProvider"); + addSimpleComponent("com.yahoo.container.jdisc.SecretStoreProvider"); + addSimpleComponent("com.yahoo.container.jdisc.CertificateStoreProvider"); + addSimpleComponent("com.yahoo.container.jdisc.metric.MetricConsumerProviderProvider"); + addSimpleComponent("com.yahoo.container.jdisc.metric.MetricProvider"); + addSimpleComponent("com.yahoo.container.jdisc.metric.MetricUpdater"); + addSimpleComponent(com.yahoo.metrics.simple.MetricManager.class.getName(), null, MetricProperties.BUNDLE_SYMBOLIC_NAME); + addSimpleComponent(com.yahoo.metrics.simple.jdisc.JdiscMetricsFactory.class.getName(), null, MetricProperties.BUNDLE_SYMBOLIC_NAME); + addSimpleComponent("com.yahoo.container.jdisc.state.StateMonitor"); + addSimpleComponent("com.yahoo.container.jdisc.ContainerThreadFactory"); + addSimpleComponent("com.yahoo.container.protect.FreezeDetector"); + addSimpleComponent("com.yahoo.container.core.slobrok.SlobrokConfigurator"); + addSimpleComponent("com.yahoo.container.handler.VipStatus"); + addJaxProviders(); + } + + public void setZone(Zone zone) { + this.zone = zone; + } + + public void addMetricStateHandler() { + Handler<AbstractConfigProducer<?>> stateHandler = new Handler<>( + new ComponentModel("com.yahoo.container.jdisc.state.StateHandler", null, null, null)); + stateHandler.addServerBindings("http://*" + StateHandler.STATE_API_ROOT, + "https://*" + StateHandler.STATE_API_ROOT, + "http://*" + StateHandler.STATE_API_ROOT + "/*", + "https://*" + StateHandler.STATE_API_ROOT + "/*"); + addComponent(stateHandler); + } + + public void addDefaultRootHandler() { + if (hasHandlerWithBinding(ROOT_HANDLER_BINDING)) + return; + + Handler<AbstractConfigProducer<?>> handler = new Handler<>( + new ComponentModel(BundleInstantiationSpecification.getFromStrings( + BindingsOverviewHandler.class.getName(), null, null), null)); // null bundle, as the handler is in container-disc + handler.addServerBindings(ROOT_HANDLER_BINDING); + addComponent(handler); + } + + private boolean hasHandlerWithBinding(String binding) { + Collection<Handler<?>> handlers = getHandlers(); + for (Handler handler : handlers) { + if (handler.getServerBindings().contains(binding)) + return true; + } + return false; + } + + public void addApplicationStatusHandler() { + Handler<AbstractConfigProducer<?>> statusHandler = new Handler<>( + new ComponentModel(BundleInstantiationSpecification.getInternalHandlerSpecificationFromStrings( + "com.yahoo.container.handler.observability.ApplicationStatusHandler", null), null)); + statusHandler.addServerBindings("http://*/ApplicationStatus", + "https://*/ApplicationStatus"); + addComponent(statusHandler); + } + + public void addVipHandler() { + Handler<?> vipHandler = Handler.fromClassName("com.yahoo.container.handler.VipStatusHandler"); + vipHandler.addServerBindings("http://*/status.html", "https://*/status.html"); + addComponent(vipHandler); + } + + public void addStatisticsHandler() { + Handler<?> statsHandler = Handler.fromClassName("com.yahoo.container.config.StatisticsRequestHandler"); + statsHandler.addServerBindings("http://*/statistics/*", "https://*/statistics/*"); + addComponent(statsHandler); + } + + public void addJaxProviders() { + addSimpleComponent(DatatypeFactoryProvider.class); + addSimpleComponent(DocumentBuilderFactoryProvider.class); + addSimpleComponent(JAXBContextFactoryProvider.class); + addSimpleComponent(SAXParserFactoryProvider.class); + addSimpleComponent(SchemaFactoryProvider.class); + addSimpleComponent(TransformerFactoryProvider.class); + addSimpleComponent(XMLEventFactoryProvider.class); + addSimpleComponent(XMLInputFactoryProvider.class); + addSimpleComponent(XMLOutputFactoryProvider.class); + addSimpleComponent(XPathFactoryProvider.class); + } + + public final void addComponent(Component<?, ?> component) { + componentGroup.addComponent(component); + } + + public final void addComponents(Collection<Component<?, ?>> components) { + for (Component<?, ?> component : components) { + addComponent(component); + } + } + + public final void addSimpleComponent(String idSpec, String classSpec, String bundleSpec) { + addComponent(new SimpleComponent(new ComponentModel(idSpec, classSpec, bundleSpec))); + } + + /** + * Removes a component by id + * + * @return the removed component, or null if it was not present + */ + public Component removeComponent(ComponentId componentId) { + return componentGroup.removeComponent(componentId); + } + + public void addSimpleComponent(Class<?> clazz) { + addSimpleComponent(clazz.getName()); + } + + private void addSimpleComponent(String className) { + addComponent(new SimpleComponent(className)); + } + + public void prepare() { + addAndSendApplicationBundles(); + sendUserConfiguredFiles(); + setApplicationMetaData(); + for (RestApi restApi : restApiGroup.getComponents()) + restApi.prepare(); + } + + private void setApplicationMetaData() { + applicationMetaData = getRoot().getDeployState().getApplicationPackage().getMetaData(); + } + + public void addMbusServer(ComponentId chainId) { + ComponentId serviceId = chainId.nestInNamespace(ComponentId.fromString("MbusServer")); + + addComponent( + new Component<>(new ComponentModel(new BundleInstantiationSpecification( + serviceId, + ComponentSpecification.fromString(MbusServerProvider.class.getName()), + null)))); + } + + private void addAndSendApplicationBundles() { + for (ComponentInfo component : getRoot().getDeployState().getApplicationPackage().getComponentsInfo(getRoot().getDeployState().getProperties().vespaVersion())) { + FileReference reference = FileSender.sendFileToServices(component.getPathRelativeToAppDir(), containers); + applicationBundles.add(reference); + } + } + + private void sendUserConfiguredFiles() { + // Files referenced from user configs to all components. + for (Component<?, ?> component : getAllComponents()) { + FileSender.sendUserConfiguredFiles(component, containers, deployLogger()); + } + } + + public String getName() { + return name; + } + + public List<Container> getContainers() { + return Collections.unmodifiableList(containers); + } + + public void addContainers(Collection<Container> containers) { + int index = this.containers.size(); + for (Container container : containers) { + container.setClusterName(name); + container.setProp("clustername", name) + .setProp("index", index++); + setRotations(container, getRotations(), getGlobalServiceId(), name); + container.setProp("activeRotation", Boolean.toString(getActiveRotation())); + } + this.containers.addAll(containers); + } + + private Optional<String> getGlobalServiceId() { + Optional<String> globalServiceId = Optional.empty(); + Optional<DeploymentSpec> deploymentSpec = getDeploymentSpec(); + if (deploymentSpec.isPresent()) { + globalServiceId = deploymentSpec.get().globalServiceId(); + } + return globalServiceId; + } + + private Set<Rotation> getRotations() { + return Optional.ofNullable(getRoot()) + .map(root -> root.getDeployState().getRotations()) + .orElse(Collections.emptySet()); + } + + private boolean getActiveRotation() { + return Optional.ofNullable(getRoot()) + .map(root -> root.getDeployState().getProperties().zone()) + .map(this::zoneHasActiveRotation) + .orElse(false); + } + + private boolean zoneHasActiveRotation(Zone zone) { + return getDeploymentSpec() + .flatMap(spec -> spec.zones().stream() + .filter(dz -> dz.matches(zone.environment(), Optional.of(zone.region()))) + .findFirst()) + .map(DeploymentSpec.DeclaredZone::active) + .orElse(false); + } + + private Optional<DeploymentSpec> getDeploymentSpec() { + Optional<DeploymentSpec> deploymentSpec = Optional.empty(); + AbstractConfigProducerRoot root = getRoot(); + if (root != null) { + final Optional<Reader> deployment = root.getDeployState().getApplicationPackage().getDeployment(); + if (deployment.isPresent()) { + deploymentSpec = Optional.of(DeploymentSpec.fromXml(deployment.get())); + } + } + return deploymentSpec; + } + + private void setRotations(Container container, Set<Rotation> rotations, Optional<String> globalServiceId, String containerClusterName) { + if (!rotations.isEmpty() && globalServiceId.isPresent()) { + if (containerClusterName.equals(globalServiceId.get())) { + container.setProp("rotations", rotations.stream().map(Rotation::getId).collect(Collectors.joining(","))); + } + } + } + + public void addContainer(Container container) { + addContainers(Collections.singletonList(container)); + } + + public void setProcessingChains(ProcessingChains processingChains, String... serverBindings) { + if (this.processingChains != null) + throw new IllegalStateException("ProcessingChains should only be set once."); + + this.processingChains = processingChains; + + // Cannot use the class object for ProcessingHandler, because its superclass is not accessible + ProcessingHandler<?> processingHandler = new ProcessingHandler<>( + processingChains, + "com.yahoo.processing.handler.ProcessingHandler"); + + for (String binding: serverBindings) + processingHandler.addServerBindings(binding); + + addComponent(processingHandler); + } + + public ProcessingChains getProcessingChains() { + return processingChains; + } + + @NonNull + public SearchChains getSearchChains() { + if (containerSearch == null) + throw new IllegalStateException("Null search components!"); + return containerSearch.getChains(); + } + + @Nullable + public ContainerSearch getSearch() { + return containerSearch; + } + + public void setSearch(ContainerSearch containerSearch) { + this.containerSearch = containerSearch; + } + + public void setHttp(Http http) { + this.http = http; + addChild(http); + } + + @Nullable + public Http getHttp() { + return http; + } + + public final void addRestApi(@NonNull RestApi restApi) { + restApiGroup.addComponent(ComponentId.fromString(restApi.getBindingPath()), restApi); + } + + public Map<ComponentId, RestApi> getRestApiMap() { + return restApiGroup.getComponentMap(); + } + + public Map<ComponentId, Servlet> getServletMap() { + return servletGroup.getComponentMap(); + } + + public final void addServlet(@NonNull Servlet servlet) { + servletGroup.addComponent(servlet.getGlobalComponentId(), servlet); + } + + @Nullable + public ContainerDocproc getDocproc() { + return containerDocproc; + } + + public void setDocproc(ContainerDocproc containerDocproc) { + this.containerDocproc = containerDocproc; + } + + @Nullable + public ContainerDocumentApi getDocumentApi() { + return containerDocumentApi; + } + + public void setDocumentApi(ContainerDocumentApi containerDocumentApi) { + this.containerDocumentApi = containerDocumentApi; + } + + @NonNull + public DocprocChains getDocprocChains() { + if (containerDocproc == null) + throw new IllegalStateException("Null docproc components!"); + return containerDocproc.getChains(); + } + + @SuppressWarnings("unchecked") + public Collection<Handler<?>> getHandlers() { + return (Collection<Handler<?>>)(Collection)componentGroup.getComponents(Handler.class); + } + + public Map<ComponentId, Component<?, ?>> getComponentsMap() { + return componentGroup.getComponentMap(); + } + + /** Returns all components in this cluster (generic, handlers, chained) */ + public Collection<Component<?, ?>> getAllComponents() { + List<Component<?, ?>> allComponents = new ArrayList<>(); + recursivelyFindAllComponents(allComponents, this); + // We need consistent ordering + Collections.sort(allComponents); + return Collections.unmodifiableCollection(allComponents); + } + + private void recursivelyFindAllComponents(Collection<Component<?, ?>> allComponents, AbstractConfigProducer<?> current) { + for (AbstractConfigProducer<?> child: current.getChildren().values()) { + if (child instanceof Component) + allComponents.add((Component<?, ?>) child); + + if (!(child instanceof Container)) + recursivelyFindAllComponents(allComponents, child); + } + } + + @Override + public final void getConfig(ComponentsConfig.Builder builder) { + builder.components.addAll(ComponentsConfigGenerator.generate(getAllComponents())); + builder.components(new ComponentsConfig.Components.Builder().id("com.yahoo.container.core.config.HandlersConfigurerDi$RegistriesHack")); + } + + @Override + public final void getConfig(JdiscBindingsConfig.Builder builder) { + builder.handlers.putAll(DiscBindingsConfigGenerator.generate(getHandlers())); + + allJersey1Handlers().forEach(handler -> + builder.handlers.putAll(DiscBindingsConfigGenerator.generate(handler)) + ); + } + + private Stream<JerseyHandler> allJersey1Handlers() { + return restApiGroup.getComponents().stream().flatMap(streamOf(RestApi::getJersey1Handler)); + } + + @Override + public void getConfig(ServletPathsConfig.Builder builder) { + allServlets().forEach(servlet -> + builder.servlets(servlet.getComponentId().stringValue(), + servlet.toConfigBuilder()) + ); + } + + private Stream<Servlet> allServlets() { + return Stream.concat(allJersey2Servlets(), + servletGroup.getComponents().stream()); + } + + private Stream<Jersey2Servlet> allJersey2Servlets() { + return restApiGroup.getComponents().stream().flatMap(streamOf(RestApi::getJersey2Servlet)); + } + + private <T, R> Function<T, Stream<R>> streamOf(Function<T, Optional<R>> f) { + return t -> + f.apply(t). + <Stream<R>>map(Stream::of). + orElse(Stream.empty()); + } + + @Override + public void getConfig(DocumentmanagerConfig.Builder builder) { + if (containerDocproc != null && containerDocproc.isCompressDocuments()) + builder.enablecompression(true); + } + + @Override + public void getConfig(ContainerDocumentConfig.Builder builder) { + for (Map.Entry<String, String> e : concreteDocumentTypes.entrySet()) { + ContainerDocumentConfig.Doctype.Builder dtb = new ContainerDocumentConfig.Doctype.Builder(); + dtb.type(e.getKey()); + dtb.factorycomponent(e.getValue()); + builder.doctype(dtb); + } + } + + @Override + public void getConfig(HealthMonitorConfig.Builder builder) { + MonitoringSystem monitoringSystem = getMonitoringService(); + if (monitoringSystem != null) { + builder.snapshot_interval(monitoringSystem.getIntervalSeconds()); + } + } + + @Override + public void getConfig(ApplicationMetadataConfig.Builder builder) { + if (applicationMetaData != null) { + builder.name(applicationMetaData.getApplicationName()). + user(applicationMetaData.getDeployedByUser()). + path(applicationMetaData.getDeployPath()). + timestamp(applicationMetaData.getDeployTimestamp()). + checksum(applicationMetaData.getCheckSum()). + generation(applicationMetaData.getGeneration()); + } + } + + /** + * Adds a bundle present at a known location at the target container nodes. + * + * @param bundlePath usually an absolute path, e.g. '$VESPA_HOME/lib/jars/foo.jar' + */ + public final void addPlatformBundle(Path bundlePath) { + platformBundles.add(bundlePath); + } + + @Override + public void getConfig(BundlesConfig.Builder builder) { + Stream.concat(applicationBundles.stream().map(FileReference::value), + platformBundles.stream().map(ContainerCluster::toFileReferenceString)) + .forEach(builder::bundle); + } + + private static String toFileReferenceString(Path path) { + return DISK_BUNDLE_PREFIX + path.toString(); + } + + @Override + public void getConfig(QrSearchersConfig.Builder builder) { + if (containerSearch!=null) containerSearch.getConfig(builder); + } + + @Override + public void getConfig(QrStartConfig.Builder builder) { + if (containerSearch!=null) containerSearch.getConfig(builder); + } + + @Override + public void getConfig(DocprocConfig.Builder builder) { + if (containerDocproc != null) { + containerDocproc.getConfig(builder); + } + } + + @Override + public void getConfig(PageTemplatesConfig.Builder builder) { + if (containerSearch != null) { + containerSearch.getConfig(builder); + } + } + + @Override + public void getConfig(SemanticRulesConfig.Builder builder) { + if (containerSearch != null) { + containerSearch.getConfig(builder); + } + } + + @Override + public void getConfig(QueryProfilesConfig.Builder builder) { + if (containerSearch != null) { + containerSearch.getConfig(builder); + } + } + + @Override + public void getConfig(SchemamappingConfig.Builder builder) { + if (containerDocproc!=null) containerDocproc.getConfig(builder); + } + + @Override + public void getConfig(IndexInfoConfig.Builder builder) { + if (containerSearch!=null) containerSearch.getConfig(builder); + } + + @Override + public void getConfig(FeederConfig.Builder builder) { + if (containerDocumentApi != null) { + containerDocumentApi.getConfig(builder); + } + } + + @Override + public void getConfig(ContainerMbusConfig.Builder builder) { + if (mbusParams != null) { + if (mbusParams.maxConcurrentFactor != null) + builder.maxConcurrentFactor(mbusParams.maxConcurrentFactor); + if (mbusParams.documentExpansionFactor != null) + builder.documentExpansionFactor(mbusParams.documentExpansionFactor); + if (mbusParams.containerCoreMemory != null) + builder.containerCoreMemory(mbusParams.containerCoreMemory); + } + if (containerDocproc != null) + containerDocproc.getConfig(builder); + } + + public void setMbusParams(MbusParams mbusParams) { + this.mbusParams = mbusParams; + } + + public void initialize(Map<String, AbstractSearchCluster> clusterMap) { + if (containerSearch != null) + containerSearch.connectSearchClusters(clusterMap); + } + + public void addDefaultSearchAccessLog() { + addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.queryAccessLog, getName())); + } + + @Override + public void getConfig(IlscriptsConfig.Builder builder) { + List<AbstractSearchCluster> searchClusters = new ArrayList<>(); + searchClusters.addAll(Content.getSearchClusters(getRoot().configModelRepo())); + for (AbstractSearchCluster searchCluster : searchClusters) { + searchCluster.getConfig(builder); + } + } + + @Override + public void getConfig(MetricDefaultsConfig.Builder builder) { + if (defaultMetricConsumerFactory != null) { + builder.factory(defaultMetricConsumerFactory); + } + } + + @Override + public void getConfig(ClusterInfoConfig.Builder builder) { + builder.clusterId(name); + builder.nodeCount(containers.size()); + + for (Service service : getDescendantServices()) { + builder.services.add(new ClusterInfoConfig.Services.Builder() + .index(Integer.parseInt(service.getServicePropertyString("index", "99999"))) + .hostname(service.getHostName()) + .ports(getPorts(service))); + } + } + + /** + * Returns a config server config containing the right zone settings (and defaults for the rest). + * This is useful to allow applications to find out in which zone they are runnung by having the Zone + * object (which is constructed from this config) injected. + */ + @Override + public void getConfig(ConfigserverConfig.Builder builder) { + builder.environment(zone.environment().value()); + builder.region(zone.region().value()); + } + + private List<ClusterInfoConfig.Services.Ports.Builder> getPorts(Service service) { + List<ClusterInfoConfig.Services.Ports.Builder> builders = new ArrayList<>(); + PortsMeta portsMeta = service.getPortsMeta(); + for (int i = 0; i < portsMeta.getNumPorts(); i++) { + builders.add(new ClusterInfoConfig.Services.Ports.Builder() + .number(service.getRelativePort(i)) + .tags(ApplicationConfigProducerRoot.getPortTags(portsMeta, i)) + ); + } + return builders; + } + + public void setDefaultMetricConsumerFactory(MetricDefaultsConfig.Factory.Enum defaultMetricConsumerFactory) { + Objects.requireNonNull(defaultMetricConsumerFactory, "defaultMetricConsumerFactory"); + this.defaultMetricConsumerFactory = defaultMetricConsumerFactory; + } + + @Override + public void getConfig(RoutingProviderConfig.Builder builder) { + builder.enabled(isHostedVespa()); + } + + public static class MbusParams { + //the amount of the maxpendingbytes to process concurrently, typically 0.2 (20%) + public final Double maxConcurrentFactor; + + //the amount that documents expand temporarily when processing them + public final Double documentExpansionFactor; + + //the space to reserve for container, docproc stuff (memory that cannot be used for processing documents), in MB + public final Integer containerCoreMemory; + + public MbusParams(Double maxConcurrentFactor, Double documentExpansionFactor, Integer containerCoreMemory) { + this.maxConcurrentFactor = maxConcurrentFactor; + this.documentExpansionFactor = documentExpansionFactor; + this.containerCoreMemory = containerCoreMemory; + } + } + + public Map<String, String> concreteDocumentTypes() { + return concreteDocumentTypes; + } + + /** + * The configured service aliases for the service in this cluster + * @return alias list + */ + public List<String> serviceAliases() { + return serviceAliases; + } + + /** + * The configured endpoint aliases (fqdn) for the service in this cluster + * @return alias list + */ + public List<String> endpointAliases() { + return endpointAliases; + } + + @Override + public String toString() { + return "container cluster '" + getName() + "'"; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModel.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModel.java new file mode 100644 index 00000000000..29846ded00f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModel.java @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container; + +import com.yahoo.config.model.ConfigModel; +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.vespa.model.content.Content; +import com.yahoo.vespa.model.search.AbstractSearchCluster; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + + +/** + * A model of a container cluster. + * + * @author tonytv + */ +public class ContainerModel extends ConfigModel { + + // TODO: Move to referer + public static final String DOCPROC_RESERVED_NAME = "docproc"; + + private ContainerCluster containerCluster; + + public ContainerModel(ConfigModelContext context) { + super(context); + } + + public void setCluster(ContainerCluster containerCluster) { this.containerCluster = containerCluster; } + + public ContainerCluster getCluster() { return containerCluster; } + + @Override + public void prepare(ConfigModelRepo plugins) { + assert (getCluster() != null) : "Null container cluster!"; + getCluster().prepare(); + } + + @Override + public void initialize(ConfigModelRepo configModelRepo) { + List<AbstractSearchCluster> searchClusters = Content.getSearchClusters(configModelRepo); + + Map<String, AbstractSearchCluster> searchClustersByName = new TreeMap<>(); + for (AbstractSearchCluster c : searchClusters) { + searchClustersByName.put(c.getClusterName(), c); + } + + getCluster().initialize(searchClustersByName); + } + + public static Collection<ContainerCluster> containerClusters(ConfigModelRepo models) { + List<ContainerCluster> containerClusters = new ArrayList<>(); + + for (ContainerModel model: models.getModels(ContainerModel.class)) + containerClusters.add(model.getCluster()); + + return containerClusters; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java new file mode 100644 index 00000000000..75ca2c1e958 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/AccessLogComponent.java @@ -0,0 +1,99 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component; + +import com.yahoo.container.core.AccessLogConfig; +import com.yahoo.container.logging.VespaAccessLog; +import com.yahoo.container.logging.YApacheAccessLog; +import com.yahoo.osgi.provider.model.ComponentModel; +import edu.umd.cs.findbugs.annotations.Nullable; + +import static com.yahoo.container.core.AccessLogConfig.FileHandler.RotateScheme; + +/** + * @author tonytv + * @author gjoranv + * @since 5.1.4 + */ +public final class AccessLogComponent extends SimpleComponent implements AccessLogConfig.Producer { + + + public enum AccessLogType { queryAccessLog, yApacheAccessLog } + + private final String fileNamePattern; + private final String rotationInterval; + private final RotateScheme.Enum rotationScheme; + private final String symlinkName; + + public AccessLogComponent(AccessLogType logType, String clusterName) { + this(logType, + String.format("logs/vespa/qrs/%s.%s.%s", capitalize(logType.name()), clusterName, "%Y%m%d%H%M%S"), + null, null, + capitalize(logType.name()) + "." + clusterName); + } + + private static String capitalize(String name) { + return name.substring(0, 1).toUpperCase() + name.substring(1); + } + + public AccessLogComponent(AccessLogType logType, + String fileNamePattern, + String rotationInterval, + RotateScheme.Enum rotationScheme, String symlinkName) { + super(new ComponentModel(accessLogClass(logType), null, "container-core", null)); + this.fileNamePattern = fileNamePattern; + this.rotationInterval = rotationInterval; + this.rotationScheme = rotationScheme; + this.symlinkName = symlinkName; + + if (fileNamePattern == null) + throw new RuntimeException("File name pattern required when configuring access log."); + } + + private static String accessLogClass(AccessLogType logType) { + switch (logType) { + case yApacheAccessLog: + return YApacheAccessLog.class.getName(); + case queryAccessLog: + return VespaAccessLog.class.getName(); + default: + throw new AssertionError(); + } + } + + @Override + public void getConfig(AccessLogConfig.Builder builder) { + builder.fileHandler(fileHandlerConfig()); + } + + private AccessLogConfig.FileHandler.Builder fileHandlerConfig() { + AccessLogConfig.FileHandler.Builder builder = new AccessLogConfig.FileHandler.Builder(); + if (fileNamePattern != null) + builder.pattern(fileNamePattern); + if (rotationInterval != null) + builder.rotation(rotationInterval); + if (rotationScheme != null) + builder.rotateScheme(rotationScheme); + if (symlinkName != null) + builder.symlink(symlinkName); + + return builder; + } + + public String getFileNamePattern() { + return fileNamePattern; + } + + public static final RotateScheme.Enum rotateScheme(@Nullable String name) { + if (name == null) + return null; + + switch (name) { + case "date": + return RotateScheme.Enum.DATE; + case "sequence": + return RotateScheme.Enum.SEQUENCE; + default: + throw new IllegalArgumentException("Invalid rotation scheme " + name); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Component.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Component.java new file mode 100644 index 00000000000..8f83978ce02 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Component.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component; + +import com.yahoo.collections.Pair; +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.osgi.provider.model.ComponentModel; + +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * @author gjoranv + * @author tonytv + */ +public class Component<CHILD extends AbstractConfigProducer<?>, MODEL extends ComponentModel> + extends AbstractConfigProducer<CHILD> implements Comparable<Component<?, ?>> { + + public final MODEL model; + final Set<Pair<String, Component>> injectedComponents = new LinkedHashSet<>(); + + public Component(MODEL model) { + super(model.getComponentId().stringValue()); + this.model = model; + } + + public ComponentId getGlobalComponentId() { + return model.getComponentId(); + } + + public ComponentId getComponentId() { + return model.getComponentId(); + } + + public ComponentSpecification getClassId() { + return model.getClassId(); + } + + public void inject(Component component) { + injectForName("", component); + } + + public void injectForName(String name, Component component) { + injectedComponents.add(new Pair<>(name, component)); + } + + public void addComponent(CHILD child) { + addChild(child); + } + + /** For testing only */ + public Set<String> getInjectedComponentIds() { + Set<String> injectedIds = new HashSet<>(); + for (Pair<String, Component> injected : injectedComponents) { + injectedIds.add(injected.getSecond().getSubId()); + } + return injectedIds; + } + + @Override + public int compareTo(Component<?, ?> other) { + return getComponentId().compareTo(other.getComponentId()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ComponentGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ComponentGroup.java new file mode 100644 index 00000000000..dda21bd80da --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ComponentGroup.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component; + +import com.yahoo.config.model.producer.AbstractConfigProducer; + +/** + * @author tonytv + */ +public class ComponentGroup <CHILD extends Component<?, ?>> extends ConfigProducerGroup<CHILD> { + + public ComponentGroup(AbstractConfigProducer parent, String subId) { + super(parent, subId); + } + + public void addComponent(CHILD producer) { + super.addComponent(producer.getComponentId(), producer); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ComponentsConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ComponentsConfigGenerator.java new file mode 100644 index 00000000000..a800daedd38 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ComponentsConfigGenerator.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component; + +import com.yahoo.collections.Pair; +import com.yahoo.container.bundle.BundleInstantiationSpecification; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import static com.yahoo.container.ComponentsConfig.Components; + +/** + * @author gjoranv + */ +public class ComponentsConfigGenerator { + + public static List<Components.Builder> generate(Collection<? extends Component<?, ?>> components) { + List<Components.Builder> result = new ArrayList<>(); + + for (Component component : components) { + result.add(componentsConfig(component)); + } + return result; + } + + public static Components.Builder componentsConfig(Component<?, ?> component) { + Components.Builder builder = new Components.Builder(); + builder.id(component.getGlobalComponentId().stringValue()); + builder.configId(component.getConfigId()); + + + bundleInstantiationSpecification(builder, component.model.bundleInstantiationSpec); + builder.inject.addAll(componentsToInject(component.injectedComponents)); + + return builder; + } + + private static void bundleInstantiationSpecification(Components.Builder config, BundleInstantiationSpecification spec) { + config.classId(spec.classId.stringValue()); + config.bundle(spec.bundle.stringValue()); + } + + + private static List<Components.Inject.Builder> componentsToInject( + Collection<Pair<String, Component>> injectedComponents) { + + List<Components.Inject.Builder> result = new ArrayList<>(); + + for (Pair<String, Component> injected : injectedComponents) { + Components.Inject.Builder builder = new Components.Inject.Builder(); + + builder.id(injected.getSecond().getGlobalComponentId().stringValue()); + builder.name(injected.getFirst()); + result.add(builder); + } + return result; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java new file mode 100644 index 00000000000..d3311806d69 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConfigProducerGroup.java @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component; + +import com.yahoo.component.ComponentId; +import com.yahoo.config.model.producer.AbstractConfigProducer; + +import java.util.*; + +/** + * A group of config producers that have a component id. + * + * @author tonytv + */ +public class ConfigProducerGroup<CHILD extends AbstractConfigProducer<?>> extends AbstractConfigProducer<CHILD> { + + private final Map<ComponentId, CHILD> producerById = new LinkedHashMap<>(); + + public ConfigProducerGroup(AbstractConfigProducer parent, String subId) { + super(parent, subId); + } + + public void addComponent(ComponentId id, CHILD producer) { + boolean wasAdded = producerById.put(id, producer) == null; + if (!wasAdded) { + throw new IllegalArgumentException("Two entities have the same component id '" + + id + "' in the same scope."); + } + addChild(producer); + } + + /** + * Removes a component by id + * + * @return the removed component, or null if it was not present + */ + public CHILD removeComponent(ComponentId componentId) { + CHILD component = producerById.remove(componentId); + if (component == null) return null; + removeChild(component); + return component; + } + + public Collection<CHILD> getComponents() { + return Collections.unmodifiableCollection(getChildren().values()); + } + + public <T extends CHILD> Collection<T> getComponents(Class<T> componentClass) { + Collection<T> result = new ArrayList<>(); + + for (CHILD child: getChildren().values()) { + if (componentClass.isInstance(child)) { + result.add(componentClass.cast(child)); + } + } + return Collections.unmodifiableCollection(result); + } + + /** + * @return A map of all components in this group, with (local) component ID as key. + */ + public Map<ComponentId, CHILD> getComponentMap() { + return Collections.unmodifiableMap(producerById); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ContainerSubsystem.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ContainerSubsystem.java new file mode 100644 index 00000000000..9880aa49a95 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ContainerSubsystem.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.chain.Chains; +import edu.umd.cs.findbugs.annotations.NonNull; + +/** + * Holder for components and options related to either processing/search/docproc + * for a container cluster. + * + * @author gjoranv + * @since 5.1.9 + */ +public abstract class ContainerSubsystem<CHAINS extends Chains<?>> { + + private final CHAINS chains; + + public ContainerSubsystem(CHAINS chains) { + this.chains = chains; + } + + @NonNull + public CHAINS getChains() { + if (chains == null) + throw new IllegalStateException("Null chains for " + this); + return chains; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java new file mode 100644 index 00000000000..6c03c64ce02 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/DiscBindingsConfigGenerator.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component; + +import java.util.*; + +import static com.yahoo.container.jdisc.JdiscBindingsConfig.Handlers; + +/** + * @author gjoranv + * @since 5.1.8 + */ +public class DiscBindingsConfigGenerator { + + public static Map<String, Handlers.Builder> generate(Collection<? extends Handler<?>> handlers) { + Map<String, Handlers.Builder> handlerBuilders = new LinkedHashMap<>(); + + for (Handler<?> handler : handlers) { + handlerBuilders.putAll(generate(handler)); + } + return handlerBuilders; + } + + public static <T extends Handler<?>> Map<String, Handlers.Builder> generate(T handler) { + if (handler.getServerBindings().isEmpty() && handler.getClientBindings().isEmpty()) + return Collections.emptyMap(); + + return Collections.singletonMap(handler.model.getComponentId().stringValue(), + new Handlers.Builder() + .serverBindings(handler.getServerBindings()) + .clientBindings(handler.getClientBindings())); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java new file mode 100644 index 00000000000..758806395a6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/FileStatusHandlerComponent.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component; + +import com.yahoo.container.core.VipStatusConfig; +import com.yahoo.osgi.provider.model.ComponentModel; + +/** + * Sets up VipStatusHandler that answers OK when a certain file is present. + * @author tonytv + */ +public class FileStatusHandlerComponent extends Handler implements VipStatusConfig.Producer { + private final String fileName; + + public FileStatusHandlerComponent(String id, String fileName, String... bindings) { + super(new ComponentModel(id, "com.yahoo.container.handler.VipStatusHandler", null, null)); + + this.fileName = fileName; + addServerBindings(bindings); + } + + @Override + public void getConfig(VipStatusConfig.Builder builder) { + builder.accessdisk(true). + statusfile(fileName); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java new file mode 100644 index 00000000000..0a9657a2d9a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Handler.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component; + +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.config.model.producer.AbstractConfigProducer; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * <p> + * Models a jdisc RequestHandler (including ClientProvider). + * RequestHandlers always have at least one server binding, + * while ClientProviders have at least one client binding. + * </p> + * <p> + * Note that this is also used to model vespa handlers (which do not have any bindings) + * </p> + * + * @author gjoranv + * @since 5.1.6 + */ +public class Handler<CHILD extends AbstractConfigProducer<?>> extends Component<CHILD, ComponentModel> { + + private List<String> serverBindings = new ArrayList<>(); + private List<String> clientBindings = new ArrayList<>(); + + public Handler(ComponentModel model) { + super(model); + } + + public static Handler<AbstractConfigProducer<?>> fromClassName(String className) { + return new Handler<>(new ComponentModel(className, null, null, null)); + } + + public static Handler<AbstractConfigProducer<?>> getVespaHandlerFromClassName(String className) { + return new Handler<>(new ComponentModel(BundleInstantiationSpecification.getInternalHandlerSpecificationFromStrings(className, null), null)); + } + + public void addServerBindings(String... bindings) { + serverBindings.addAll(Arrays.asList(bindings)); + } + + public void addClientBindings(String... bindings) { + clientBindings.addAll(Arrays.asList(bindings)); + } + + public final List<String> getServerBindings() { + return Collections.unmodifiableList(serverBindings); + } + + public final List<String> getClientBindings() { + return Collections.unmodifiableList(clientBindings); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/HttpFilter.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/HttpFilter.java new file mode 100644 index 00000000000..b919d8bd70e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/HttpFilter.java @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.container.FilterConfigProvider; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.osgi.provider.model.ComponentModel; + +/** + * This is only for the legacy yca filter setup, outside http. + * + * TODO: Remove when 'filter' directly under 'jdisc' can be removed from services.xml + * + * @author tonytv + */ +public class HttpFilter extends SimpleComponent { + private static final ComponentSpecification filterConfigProviderClass = + ComponentSpecification.fromString(FilterConfigProvider.class.getName()); + + public final SimpleComponent filterConfigProvider; + + public HttpFilter(BundleInstantiationSpecification spec) { + super(new ComponentModel(spec)); + + filterConfigProvider = new SimpleComponent(new ComponentModel( + new BundleInstantiationSpecification(configProviderId(spec.id), filterConfigProviderClass, null))); + + addChild(filterConfigProvider); + } + + // public for testing + public static ComponentId configProviderId(ComponentId filterId) { + return ComponentId.fromString("filterConfig").nestInNamespace(filterId); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Servlet.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Servlet.java new file mode 100644 index 00000000000..68ba3436209 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Servlet.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.jdisc.http.ServletPathsConfig; +import com.yahoo.osgi.provider.model.ComponentModel; + +/** + * @author stiankri + */ +public class Servlet extends SimpleComponent { + private final String bindingPath; + + public Servlet(ComponentModel componentModel, String bindingPath) { + super(componentModel); + this.bindingPath = bindingPath; + } + + public ServletPathsConfig.Servlets.Builder toConfigBuilder() { + return new ServletPathsConfig.Servlets.Builder() + .path(bindingPath); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ServletProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ServletProvider.java new file mode 100644 index 00000000000..7c6bc169878 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ServletProvider.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.container.servlet.ServletConfigConfig; +import com.yahoo.osgi.provider.model.ComponentModel; + +import java.util.Map; + +/** + * @author stiankri + */ +public class ServletProvider extends Servlet implements ServletConfigConfig.Producer { + public static final String BUNDLE = "container-core"; + public static final String CLASS = "com.yahoo.container.servlet.ServletProvider"; + + private static final ComponentId SERVLET_PROVIDER_NAMESPACE = ComponentId.fromString("servlet-provider"); + private final Map<String, String> servletConfig; + + public ServletProvider(SimpleComponent servletToProvide, String bindingPath, Map<String, String> servletConfig) { + super(new ComponentModel( + new BundleInstantiationSpecification(servletToProvide.getComponentId().nestInNamespace(SERVLET_PROVIDER_NAMESPACE), + ComponentSpecification.fromString(CLASS), + ComponentSpecification.fromString(BUNDLE))), + bindingPath); + + inject(servletToProvide); + addChild(servletToProvide); + this.servletConfig = servletConfig; + } + + @Override + public void getConfig(ServletConfigConfig.Builder builder) { + builder.map(servletConfig); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java new file mode 100644 index 00000000000..a0835f3ce88 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SimpleComponent.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component; + +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.config.model.producer.AbstractConfigProducer; + +/** + * A component that only needs a simple ComponentModel. + * + * @author gjoranv + * @since 5.1.9 + */ +public class SimpleComponent extends Component<AbstractConfigProducer<?>, ComponentModel> { + + public SimpleComponent(ComponentModel model) { + super(model); + } + + // @Convenience // For a component that uses the class name as id. + public SimpleComponent(String className) { + this(new ComponentModel(BundleInstantiationSpecification.getFromStrings(className, null, null))); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/StatisticsComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/StatisticsComponent.java new file mode 100644 index 00000000000..b459acd63e9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/StatisticsComponent.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component; + +import com.yahoo.vespa.model.admin.MonitoringSystem; +import com.yahoo.container.StatisticsConfig; + +/** + * @author tonytv + */ +public class StatisticsComponent extends SimpleComponent implements StatisticsConfig.Producer { + + public StatisticsComponent() { + super("com.yahoo.statistics.StatisticsImpl"); + } + + @Override + public void getConfig(StatisticsConfig.Builder builder) { + MonitoringSystem monitoringSystem = getMonitoringService(); + if (monitoringSystem != null) { + builder. + collectionintervalsec(monitoringSystem.getIntervalSeconds().doubleValue()). + loggingintervalsec(monitoringSystem.getIntervalSeconds().doubleValue()); + } + builder.values(new StatisticsConfig.Values.Builder(). + name("query_latency"). + operations(new StatisticsConfig.Values.Operations.Builder(). + name(StatisticsConfig.Values.Operations.Name.REGULAR). + arguments(new StatisticsConfig.Values.Operations.Arguments.Builder(). + key("limits"). + value("25,50,100,500")))); + + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/Chain.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/Chain.java new file mode 100644 index 00000000000..558fe368786 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/Chain.java @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component.chain; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.container.component.ComponentGroup; + +import java.util.ArrayList; +import java.util.Collection; + +import static com.yahoo.container.core.ChainsConfig.Chains.Type; + +/** + * Represents a component chain in the vespa model. + * The inner components are represented as children. + * + * @author tonytv + * @author gjoranv + */ +public class Chain<T extends ChainedComponent<?>> extends AbstractConfigProducer<AbstractConfigProducer<?>> { + + private final ComponentId componentId; + private final ChainSpecification specWithoutInnerComponents; + private final ComponentGroup<T> innerComponentsGroup; + private static final Type.Enum TYPE = Type.SEARCH; + + + public Chain(ChainSpecification specWithoutInnerComponents) { + super(specWithoutInnerComponents.componentId.stringValue()); + + this.componentId = specWithoutInnerComponents.componentId; + this.specWithoutInnerComponents = specWithoutInnerComponents; + assertNoInnerComponents(specWithoutInnerComponents); + + innerComponentsGroup = new ComponentGroup<>(this, "component"); + } + + private void assertNoInnerComponents(ChainSpecification specWithoutInnerComponents) { + for (ComponentSpecification component : specWithoutInnerComponents.componentReferences) { + assert (component.getNamespace() == null); + } + } + + public void addInnerComponent(T component) { + innerComponentsGroup.addComponent(component); + } + + public ChainSpecification getChainSpecification() { + Collection<ComponentSpecification> innerComponentSpecifications = new ArrayList<>(); + + for (ChainedComponent innerComponent : getInnerComponents()) { + innerComponentSpecifications.add(innerComponent.getGlobalComponentId().toSpecification()); + } + + return specWithoutInnerComponents. + addComponents(innerComponentSpecifications). + setComponentId(getGlobalComponentId()); + } + + public Collection<T> getInnerComponents() { + return innerComponentsGroup.getComponents(); + } + + public ComponentId getGlobalComponentId() { + return componentId; + } + + public final ComponentId getId() { return getGlobalComponentId(); } + + public final ComponentId getComponentId() { + return componentId; + } + + // TODO: remove when DocumentProcessingHandler takes its own version of the chains config as ctor arg + public Type.Enum getType() { + return TYPE; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainedComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainedComponent.java new file mode 100644 index 00000000000..90579e8b77c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainedComponent.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component.chain; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.model.ChainedComponentModel; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.container.component.Component; + + +/** + * @author tonytv + * @author gjoranv + * + * Base class for all ChainedComponent config producers. + */ +public class ChainedComponent<T extends ChainedComponentModel> extends Component<AbstractConfigProducer<?>, T> { + + public ChainedComponent(T model) { + super(model); + } + + public void initialize() {} + + @Override + public ComponentId getGlobalComponentId() { + return model.getComponentId().nestInNamespace(namespace()); + } + + private ComponentId namespace() { + AbstractConfigProducer owner = getParent().getParent(); + return (owner instanceof Chain) ? + ((Chain) owner).getGlobalComponentId() : + null; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainedComponentConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainedComponentConfigGenerator.java new file mode 100644 index 00000000000..34aee0e217e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainedComponentConfigGenerator.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component.chain; + +import com.yahoo.component.chain.dependencies.Dependencies; +import com.yahoo.component.chain.model.ChainedComponentModel; +import com.yahoo.container.core.ChainsConfig; + +import java.util.Set; + +import static com.yahoo.container.core.ChainsConfig.Components; + +/** + * @author tonytv + * @author gjoranv + * + * Generates config for all the chained components. + */ +class ChainedComponentConfigGenerator { + + public static void generate(ChainsConfig.Builder builder, Set<? extends ChainedComponent> components) { + for (ChainedComponent<ChainedComponentModel> component : components) { + builder.components(getComponent(component)); + } + } + + private static Components.Builder getComponent(ChainedComponent<ChainedComponentModel> component) { + return new Components.Builder() + .id(component.getGlobalComponentId().stringValue()) + .dependencies(getDependencies(component)); + } + + private static Components.Dependencies.Builder getDependencies(ChainedComponent<ChainedComponentModel> component) { + Dependencies dependencies = component.model.dependencies; + return new Components.Dependencies.Builder() + .provides(dependencies.provides()) + .before(dependencies.before()) + .after(dependencies.after()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/Chains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/Chains.java new file mode 100644 index 00000000000..bf8e611bfe8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/Chains.java @@ -0,0 +1,107 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component.chain; + +import com.yahoo.component.chain.model.ChainsModel; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.container.core.ChainsConfig; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.container.component.ComponentGroup; +import com.yahoo.vespa.model.container.component.ConfigProducerGroup; + +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * Root config producer the whole chains model(contains chains and components). + * @author tonytv + * @author gjoranv + */ +public class Chains<CHAIN extends Chain<?>> + extends AbstractConfigProducer<AbstractConfigProducer<?>> + implements ChainsConfig.Producer { + + private final ComponentGroup<ChainedComponent<?>> componentGroup; + private final ConfigProducerGroup<CHAIN> chainGroup; + + public Chains(AbstractConfigProducer parent, String subId) { + super(parent, subId); + componentGroup = new ComponentGroup<>(this, "component"); + chainGroup = new ConfigProducerGroup<>(this, "chain"); + } + + public void initializeComponents() { + for (ChainedComponent component : allComponents()) { + component.initialize(); + } + } + + public void validate() throws Exception { + ChainsModel chainsModel = new ChainsModel(); + + for (CHAIN chain : allChains().allComponents()) { + chainsModel.register(chain.getChainSpecification()); + } + for (ChainedComponent<?> component : allComponents()) { + chainsModel.register(component.getGlobalComponentId(), component.model); + } + chainsModel.validate(); + + super.validate(); + } + + public Set<ChainedComponent<?>> allComponents() { + Set<ChainedComponent<?>> result = new LinkedHashSet<>(); + result.addAll(componentGroup.getComponents()); + + for (CHAIN chain : allChains().allComponents()) { + result.addAll(chain.getInnerComponents()); + } + return result; + } + + public ComponentRegistry<ChainedComponent<?>> componentsRegistry() { + ComponentRegistry<ChainedComponent<?>> result = new ComponentRegistry<>(); + + for (ChainedComponent<?> component: componentGroup.getComponents()) + result.register(component.getGlobalComponentId(), component); + + for (CHAIN chain : allChains().allComponents()) { + for (ChainedComponent<?> component: chain.getInnerComponents()) { + result.register(component.getGlobalComponentId(), component); + } + } + return result; + } + + public ComponentRegistry<CHAIN> allChains() { + ComponentRegistry<CHAIN> allChains = new ComponentRegistry<>(); + for (CHAIN chain : chainGroup.getComponents()) { + allChains.register(chain.getId(), chain); + } + allChains.freeze(); + return allChains; + } + + public void add(CHAIN chain) { + chainGroup.addComponent(chain.getId(), chain); + } + + public void add(ChainedComponent outerComponent) { + componentGroup.addComponent(outerComponent); + } + + @Override + public void getConfig(ChainsConfig.Builder builder) { + ChainsConfigGenerator.generate(builder, allChains().allComponents()); + ChainedComponentConfigGenerator.generate(builder, allComponents()); + } + + public ConfigProducerGroup<ChainedComponent<?>> getComponentGroup() { + return componentGroup; + } + + protected ConfigProducerGroup<CHAIN> getChainGroup() { + return chainGroup; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainsConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainsConfigGenerator.java new file mode 100644 index 00000000000..da3f8846974 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ChainsConfigGenerator.java @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component.chain; + +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.Phase; +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.container.core.ChainsConfig; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import static com.yahoo.container.core.ChainsConfig.Chains.*; + +/** + * @author tonytv + * @author gjoranv + * + * Generates config for a all the chains. + */ +class ChainsConfigGenerator<T extends Chain> { + + public static <T extends Chain> void generate(ChainsConfig.Builder builder, Collection<T> chains) { + for (T chain : chains) { + builder.chains(getChain(chain)); + } + } + + private static <T extends Chain> ChainsConfig.Chains.Builder getChain(T chain) { + ChainSpecification specification = chain.getChainSpecification(); + + return new ChainsConfig.Chains.Builder() + .type(chain.getType()) + .id(specification.componentId.stringValue()) + .components(getComponents(specification.componentReferences)) + .inherits(getComponents(specification.inheritance.chainSpecifications)) + .excludes(getComponents(specification.inheritance.excludedComponents)) + .phases(getPhases(specification.phases())); + } + + private static List<String> getComponents(Collection<ComponentSpecification> componentSpecs) { + List<String> components = new ArrayList<>(); + for (ComponentSpecification spec : componentSpecs) + components.add(spec.stringValue()); + return components; + } + + private static List<Phases.Builder> getPhases(Collection<Phase> phases) { + List<Phases.Builder> builders = new ArrayList<>(); + for (Phase phase : phases) { + builders.add( + new Phases.Builder() + .id(phase.getName()) + .before(phase.before()) + .after(phase.after())); + } + return builders; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ProcessingHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ProcessingHandler.java new file mode 100644 index 00000000000..ecd1e45c7fa --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/ProcessingHandler.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.component.chain; + +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.container.core.ChainsConfig; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.container.component.Handler; + + +/** + * Represents a handler for processing chains. + * + * @author gjoranv + * @since 5.1.7 + */ +public class ProcessingHandler<CHAINS extends Chains<?>> + extends Handler<AbstractConfigProducer<?>> + implements ChainsConfig.Producer { + + protected final CHAINS chains; + + public ProcessingHandler(CHAINS chains, String handlerClass) { + super(new ComponentModel(BundleInstantiationSpecification.getInternalProcessingSpecificationFromStrings(handlerClass, null), null)); + this.chains = chains; + + } + + @Override + public void getConfig(ChainsConfig.Builder builder) { + chains.getConfig(builder); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/package-info.java new file mode 100644 index 00000000000..c35c50bfde9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/chain/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.model.container.component.chain; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/package-info.java new file mode 100644 index 00000000000..4471383c95e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.model.container.component; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java new file mode 100644 index 00000000000..c52a7375528 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java @@ -0,0 +1,181 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.configserver; + +import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.container.StatisticsConfig; +import com.yahoo.container.jdisc.config.HealthMonitorConfig; +import com.yahoo.jdisc.metrics.yamasconsumer.cloud.ScoreBoardConfig; +import com.yahoo.net.HostName; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; +import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions.ConfigServer; + +import java.util.Optional; + +/** + * Represents a config server cluster. + * + * @author lulf + * @since 5.15 + */ +public class ConfigserverCluster extends AbstractConfigProducer + implements + ZookeeperServerConfig.Producer, + ConfigserverConfig.Producer, + ScoreBoardConfig.Producer, + StatisticsConfig.Producer, + HealthMonitorConfig.Producer { + private final CloudConfigOptions options; + private ContainerCluster containerCluster; + + public ConfigserverCluster(AbstractConfigProducer parent, String subId, CloudConfigOptions options) { + super(parent, subId); + this.options = options; + } + + public void setContainerCluster(ContainerCluster containerCluster) { + this.containerCluster = containerCluster; + + // If we are in a config server cluster the correct zone is propagated through cloud config options, + // not through config to deployment options (see StandaloneContainerApplication.scala), + // so we need to propagate the zone options into the container from here + Environment environment = options.environment().isPresent() ? Environment.from(options.environment().get()) : Environment.defaultEnvironment(); + RegionName region = options.region().isPresent() ? RegionName.from(options.region().get()) : RegionName.defaultName(); + containerCluster.setZone(new Zone(environment, region)); + } + + @Override + public void getConfig(ZookeeperServerConfig.Builder builder) { + String myhostname = HostName.getLocalhost(); + int myid = 0; + int i = 0; + for (ConfigServer server : getConfigServers()) { + if (server.hostName.equals(myhostname)) { + myid = i; + } + builder.server(getZkServer(server, i)); + i++; + } + builder.myid(myid); + if (options.zookeeperClientPort().isPresent()) { + builder.clientPort(options.zookeeperClientPort().get()); + } + } + + @Override + public void getConfig(ConfigserverConfig.Builder builder) { + for (String pluginDir : getConfigModelPluginDirs()) { + builder.configModelPluginDir(pluginDir); + } + if (options.sessionLifeTimeSecs().isPresent()) { + builder.sessionLifetime(options.sessionLifeTimeSecs().get()); + } + if (options.zookeeperBarrierTimeout().isPresent()) { + builder.zookeeper(new ConfigserverConfig.Zookeeper.Builder().barrierTimeout(options.zookeeperBarrierTimeout().get())); + } + if (options.rpcPort().isPresent()) { + builder.rpcport(options.rpcPort().get()); + } + if (options.multiTenant().isPresent()) { + builder.multitenant(options.multiTenant().get()); + } + if (options.payloadCompressionType().isPresent()) { + builder.payloadCompressionType(ConfigserverConfig.PayloadCompressionType.Enum.valueOf(options.payloadCompressionType().get())); + } + for (ConfigServer server : getConfigServers()) { + ConfigserverConfig.Zookeeperserver.Builder zkBuilder = new ConfigserverConfig.Zookeeperserver.Builder(); + zkBuilder.hostname(server.hostName); + if (options.zookeeperClientPort().isPresent()) { + zkBuilder.port(options.zookeeperClientPort().get()); + } + builder.zookeeperserver(zkBuilder); + } + if (options.environment().isPresent()) { + builder.environment(options.environment().get()); + } + if (options.region().isPresent()) { + builder.region(options.region().get()); + } + if (options.defaultFlavor().isPresent()) { + builder.defaultFlavor(options.defaultFlavor().get()); + } + if (options.defaultAdminFlavor().isPresent()) { + builder.defaultAdminFlavor(options.defaultAdminFlavor().get()); + } + if (options.defaultContainerFlavor().isPresent()) { + builder.defaultContainerFlavor(options.defaultContainerFlavor().get()); + } + if (options.defaultContentFlavor().isPresent()) { + builder.defaultContentFlavor(options.defaultContentFlavor().get()); + } + + builder.serverId(HostName.getLocalhost()); + if (!containerCluster.getHttp().getHttpServer().getConnectorFactories().isEmpty()) { + builder.httpport(containerCluster.getHttp().getHttpServer().getConnectorFactories().get(0).getListenPort()); + } + if (options.useVespaVersionInRequest().isPresent()) { + builder.useVespaVersionInRequest(options.useVespaVersionInRequest().get()); + } else if (options.multiTenant().isPresent()) { + builder.useVespaVersionInRequest(options.multiTenant().get()); + } + if (options.hostedVespa().isPresent()) { + builder.hostedVespa(options.hostedVespa().get()); + } + if (options.numParallelTenantLoaders().isPresent()) { + builder.numParallelTenantLoaders(options.numParallelTenantLoaders().get()); + } + } + + private String[] getConfigModelPluginDirs() { + if (options.configModelPluginDirs().length > 0) { + return options.configModelPluginDirs(); + } else { + return new String[]{Defaults.getDefaults().vespaHome() + "lib/jars/config-models"}; + } + } + + private ConfigServer[] getConfigServers() { + if (options.allConfigServers().length > 0) { + return options.allConfigServers(); + } else { + return new ConfigServer[]{new ConfigServer(HostName.getLocalhost(), Optional.<Integer>empty()) }; + } + } + + private ZookeeperServerConfig.Server.Builder getZkServer(ConfigServer server, int id) { + ZookeeperServerConfig.Server.Builder builder = new ZookeeperServerConfig.Server.Builder(); + if (options.zookeeperElectionPort().isPresent()) { + builder.electionPort(options.zookeeperElectionPort().get()); + } + if (options.zookeeperQuorumPort().isPresent()) { + builder.quorumPort(options.zookeeperQuorumPort().get()); + } + builder.hostname(server.hostName); + builder.id(id); + return builder; + } + + @Override + public void getConfig(ScoreBoardConfig.Builder builder) { + builder.applicationName("configserver"); + builder.flushTime(60); + builder.step(60); + } + + @Override + public void getConfig(StatisticsConfig.Builder builder) { + builder.collectionintervalsec(60.0); + builder.loggingintervalsec(60.0); + } + + @Override + public void getConfig(HealthMonitorConfig.Builder builder) { + builder.snapshot_interval(60.0); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java new file mode 100644 index 00000000000..3bb4f9d09f5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.configserver.option; + +import java.util.Optional; + +/** + * @author tonytv + */ +public interface CloudConfigOptions { + + class ConfigServer { + public final String hostName; + public final Optional<Integer> port; + + public ConfigServer(String hostName, Optional<Integer> port) { + this.hostName = hostName; + this.port = port; + } + } + + + Optional<Integer> rpcPort(); + Optional<Boolean> multiTenant(); + Optional<Boolean> hostedVespa(); + + ConfigServer[] allConfigServers(); + Optional<Integer> zookeeperClientPort(); + String[] configModelPluginDirs(); + Optional<Long> sessionLifeTimeSecs(); + + //TODO: which unit? + Optional<Long> zookeeperBarrierTimeout(); + Optional<Integer> zookeeperElectionPort(); + Optional<Integer> zookeeperQuorumPort(); + Optional<String> payloadCompressionType(); + Optional<String> environment(); + Optional<String> region(); + Optional<String> defaultFlavor(); + Optional<String> defaultAdminFlavor(); + Optional<String> defaultContainerFlavor(); + Optional<String> defaultContentFlavor(); + Optional<Boolean> useVespaVersionInRequest(); + Optional<Integer> numParallelTenantLoaders(); +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/package-info.java new file mode 100644 index 00000000000..a6f5bfa895a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/package-info.java @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author tonytv + */ +@ExportPackage +package com.yahoo.vespa.model.container.configserver.option; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java new file mode 100644 index 00000000000..a1c98396ffc --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/ContainerDocproc.java @@ -0,0 +1,178 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.docproc; + +import com.yahoo.collections.Pair; +import com.yahoo.config.docproc.DocprocConfig; +import com.yahoo.container.jdisc.config.SessionConfig; +import com.yahoo.container.jdisc.ContainerMbusConfig; +import com.yahoo.config.docproc.SchemamappingConfig; +import com.yahoo.docproc.jdisc.messagebus.MbusRequestContext; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.ContainerSubsystem; +import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; +import edu.umd.cs.findbugs.annotations.Nullable; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author einarmr + * @author gjoranv + * @since 5.1.9 + */ +public class ContainerDocproc extends ContainerSubsystem<DocprocChains> + implements + ContainerMbusConfig.Producer, + SchemamappingConfig.Producer, + DocprocConfig.Producer +{ + public final Options options; + + // Whether or not to prefer sending to a local node. + private boolean preferLocalNode = false; + + // The number of nodes to use per client. + private int numNodesPerClient = 0; + + private Map<Pair<String, String>, String> fieldNameSchemaMap = new HashMap<>(); + + public ContainerDocproc(ContainerCluster cluster, DocprocChains chains) { + this(cluster, chains, new Options(false, null, null, null, null, null, null)); + } + + public ContainerDocproc(ContainerCluster cluster, DocprocChains chains, Options options) { + this(cluster, chains, options, true); + } + + private void addSource( + final ContainerCluster cluster, final String name, final SessionConfig.Type.Enum type) { + final MbusClient mbusClient = new MbusClient(name, type); + mbusClient.addClientBindings("mbus://*/" + mbusClient.getSessionName()); + cluster.addComponent(mbusClient); + } + + public ContainerDocproc(ContainerCluster cluster, DocprocChains chains, Options options, boolean addSourceClientProvider) { + super(chains); + assert (options != null) : "Null Options for " + this + " under cluster " + cluster.getName(); + this.options = options; + + if (addSourceClientProvider) { + addSource(cluster, "source", SessionConfig.Type.SOURCE); + addSource(cluster, MbusRequestContext.internalNoThrottledSource, SessionConfig.Type.INTERNAL); + } + } + + public boolean isCompressDocuments() { + return options.compressDocuments; + } + + public boolean isPreferLocalNode() { + return preferLocalNode; + } + + public void setPreferLocalNode(boolean preferLocalNode) { + this.preferLocalNode = preferLocalNode; + } + + public int getNumNodesPerClient() { + return numNodesPerClient; + } + + public void setNumNodesPerClient(int numNodesPerClient) { + this.numNodesPerClient = numNodesPerClient; + } + + @Override + public void getConfig(ContainerMbusConfig.Builder builder) { + builder.maxpendingcount(getMaxMessagesInQueue()); + if (getMaxQueueMbSize() != null) + builder.maxpendingsize(getMaxQueueMbSize()); //yes, this shall be set in megabytes. + } + + private int getMaxMessagesInQueue() { + if (options.maxMessagesInQueue != null) { + return options.maxMessagesInQueue; + } + + //maxmessagesinqueue has not been set for this node. let's try to give a good value anyway: + return 2048 * getChains().allChains().allComponents().size(); + //intentionally high, getMaxQueueMbSize() will probably kick in before this one! + } + + @Nullable + private Integer getMaxQueueMbSize() { + return options.maxQueueMbSize; + } + + + private Integer getMaxQueueTimeMs() { + return options.maxQueueTimeMs; + } + + @Override + public void getConfig(DocprocConfig.Builder builder) { + if (getMaxQueueTimeMs() != null) { + builder.maxqueuetimems(getMaxQueueTimeMs()); + } + } + + public ProcessingHandler<DocprocChains> getDocprocHandler() { + return ((DocprocChains) getChains()).getDocprocHandler(); + } + + @Override + public void getConfig(SchemamappingConfig.Builder builder) { + Map<Pair<String, String>, String> allMappings = new HashMap<>(); + for (DocprocChain chain : getChains().allChains().allComponents()) { + for (DocumentProcessor processor : chain.getInnerComponents()) { + allMappings.putAll(fieldNameSchemaMap()); + allMappings.putAll(chain.fieldNameSchemaMap()); + allMappings.putAll(processor.fieldNameSchemaMap()); + for (Map.Entry<Pair<String,String>, String> e : allMappings.entrySet()) { + String doctype = e.getKey().getFirst(); + String from = e.getKey().getSecond(); + String to = e.getValue(); + builder.fieldmapping(new SchemamappingConfig.Fieldmapping.Builder(). + chain(chain.getId().stringValue()). + docproc(processor.getGlobalComponentId().stringValue()). + indocument(from). + inprocessor(to). + doctype(doctype!=null?doctype:"")); + } + allMappings.clear(); + } + } + } + + /** + * The field name schema map that applies to this whole chain + * @return doctype,from → to + */ + public Map<Pair<String,String>,String> fieldNameSchemaMap() { + return fieldNameSchemaMap; + } + + public static class Options { + // Whether or not to compress documents after processing them. + public final boolean compressDocuments; + + public final Integer maxMessagesInQueue; + public final Integer maxQueueMbSize; + public final Integer maxQueueTimeMs; + + public final Double maxConcurrentFactor; + public final Double documentExpansionFactor; + public final Integer containerCoreMemory; + + public Options(boolean compressDocuments, Integer maxMessagesInQueue, Integer maxQueueMbSize, Integer maxQueueTimeMs, Double maxConcurrentFactor, Double documentExpansionFactor, Integer containerCoreMemory) { + this.compressDocuments = compressDocuments; + this.maxMessagesInQueue = maxMessagesInQueue; + this.maxQueueMbSize = maxQueueMbSize; + this.maxQueueTimeMs = maxQueueTimeMs; + this.maxConcurrentFactor = maxConcurrentFactor; + this.documentExpansionFactor = documentExpansionFactor; + this.containerCoreMemory = containerCoreMemory; + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChain.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChain.java new file mode 100644 index 00000000000..28d0b298f64 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChain.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.docproc; + +import com.yahoo.collections.Pair; +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.vespa.model.container.component.chain.Chain; + +import java.util.Map; + +import static com.yahoo.container.core.ChainsConfig.Chains.Type; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class DocprocChain extends Chain<DocumentProcessor> { + private Map<Pair<String, String>, String> fieldNameSchemaMap; + private static final Type.Enum TYPE = Type.Enum.DOCPROC; + + public DocprocChain(ChainSpecification specWithoutInnerComponents, Map<Pair<String,String>, String> fieldNameSchemaMap) { + super(specWithoutInnerComponents); + this.fieldNameSchemaMap = fieldNameSchemaMap; + } + + /** + * The field name schema map that applies to this whole chain + * @return doctype,from → to + */ + public Map<Pair<String,String>,String> fieldNameSchemaMap() { + return fieldNameSchemaMap; + } + + public String getServiceName() { + return getParent().getParent().getParent().getConfigId() + "/" + getSessionName(); + } + + public String getSessionName() { + return "chain." + getComponentId().stringValue(); + } + + public Type.Enum getType() { + return TYPE; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java new file mode 100644 index 00000000000..e1dbbf185c7 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocprocChains.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.docproc; + +import com.yahoo.component.ComponentId; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.container.jdisc.config.SessionConfig; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.chain.Chains; +import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; + +/** + * @author einarmr + * @since 5.1.9 + */ +public class DocprocChains extends Chains<DocprocChain> { + private final ProcessingHandler<DocprocChains> docprocHandler; + + public DocprocChains(AbstractConfigProducer parent, String subId) { + super(parent, subId); + docprocHandler = new ProcessingHandler<>(this, "com.yahoo.docproc.jdisc.DocumentProcessingHandler"); + addComponent(docprocHandler); + } + + public ProcessingHandler<DocprocChains> getDocprocHandler() { + return docprocHandler; + } + + private void addComponent(Component component) { + if (!(getParent() instanceof ContainerCluster)) { + return; + } + ((ContainerCluster) getParent()).addComponent(component); + } + + + public void addServersAndClientsForChains() { + if (getParent() instanceof ContainerCluster) { + for (DocprocChain chain: getChainGroup().getComponents()) + addServerAndClientForChain((ContainerCluster) getParent(), chain); + } + } + + private void addServerAndClientForChain(ContainerCluster cluster, DocprocChain docprocChain) { + docprocHandler.addServerBindings("mbus://*/" + docprocChain.getSessionName()); + + cluster.addMbusServer(ComponentId.fromString(docprocChain.getSessionName())); + + MbusClient client = new MbusClient(docprocChain.getSessionName(), SessionConfig.Type.INTERMEDIATE); + client.addClientBindings("mbus://*/" + client.getSessionName()); + addComponent(client); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocumentProcessor.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocumentProcessor.java new file mode 100644 index 00000000000..789b578eb13 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/DocumentProcessor.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.docproc; + +import com.yahoo.collections.Pair; +import com.yahoo.vespa.model.container.component.chain.ChainedComponent; +import com.yahoo.vespa.model.container.docproc.model.DocumentProcessorModel; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class DocumentProcessor extends ChainedComponent<DocumentProcessorModel> { + + public static final String INDEXER = "com.yahoo.docprocs.indexing.IndexingProcessor"; + + private final Map<Pair<String, String>, String> fieldNameSchemaMap; + + public DocumentProcessor(DocumentProcessorModel model) { + super(model); + this.fieldNameSchemaMap = model.fieldNameSchemaMap(); + } + + /** + * The field name schema map that applies to this docproc + * @return doctype,from → to + */ + public Map<Pair<String,String>,String> fieldNameSchemaMap() { + return fieldNameSchemaMap; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/MbusClient.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/MbusClient.java new file mode 100644 index 00000000000..87038115e0c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/MbusClient.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.docproc; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.container.jdisc.config.SessionConfig; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.container.component.Handler; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class MbusClient extends Handler<AbstractConfigProducer<?>> implements SessionConfig.Producer { + private static final ComponentSpecification CLASSNAME = + ComponentSpecification.fromString("com.yahoo.container.jdisc.messagebus.MbusClientProvider"); + + private final String sessionName; + private final SessionConfig.Type.Enum type; + + public MbusClient(String sessionName, SessionConfig.Type.Enum type) { + super(new ComponentModel(new BundleInstantiationSpecification(createId(sessionName), CLASSNAME, null))); + this.sessionName = sessionName; + this.type = type; + } + + private static ComponentId createId(String sessionName) { + return ComponentId.fromString(sessionName).nestInNamespace( + ComponentId.fromString("MbusClient")); + } + + @Override + public void getConfig(SessionConfig.Builder sb) { + sb. + name(sessionName). + type(type); + } + + public String getSessionName() { + return sessionName; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/model/DocumentProcessorModel.java b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/model/DocumentProcessorModel.java new file mode 100644 index 00000000000..1f166540b94 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/docproc/model/DocumentProcessorModel.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.docproc.model; + +import com.yahoo.collections.Pair; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.component.chain.dependencies.Dependencies; +import com.yahoo.component.chain.model.ChainedComponentModel; +import net.jcip.annotations.Immutable; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +@Immutable +public class DocumentProcessorModel extends ChainedComponentModel { + private Map<Pair<String, String>, String> fieldNameSchemaMap = new HashMap<>(); + + public DocumentProcessorModel(BundleInstantiationSpecification bundleInstantiationSpec, Dependencies dependencies, Map<Pair<String, String>, String> fieldNameSchemaMap) { + super(bundleInstantiationSpec, dependencies); + this.fieldNameSchemaMap.putAll(fieldNameSchemaMap); + } + + /** + * The field name schema map that applies to this docproc + * @return doctype,from → to + */ + public Map<Pair<String,String>,String> fieldNameSchemaMap() { + return fieldNameSchemaMap; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java new file mode 100644 index 00000000000..24aeda9ed84 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ConnectorFactory.java @@ -0,0 +1,149 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.http; + +import com.yahoo.component.ComponentId; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.jdisc.http.ConnectorConfig; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.container.component.SimpleComponent; +import org.w3c.dom.Element; + +import static com.yahoo.component.ComponentSpecification.fromString; +import static com.yahoo.jdisc.http.ConnectorConfig.Ssl.KeyStoreType; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.21.0 + */ +public class ConnectorFactory extends SimpleComponent implements ConnectorConfig.Producer { + + private final String name; + private volatile int listenPort; + private final Element legacyConfig; + + public ConnectorFactory(final String name, final int listenPort, final Element legacyConfig) { + super(new ComponentModel( + new BundleInstantiationSpecification(new ComponentId(name), + fromString("com.yahoo.jdisc.http.server.jetty.ConnectorFactory"), + fromString("jdisc_http_service")) + + )); + + + this.name = name; + this.listenPort = listenPort; + this.legacyConfig = legacyConfig; + } + + @Override + public void getConfig(ConnectorConfig.Builder connectorBuilder) { + if (legacyConfig != null) { + { + Element tcpKeepAliveEnabled = XML.getChild(legacyConfig, "tcpKeepAliveEnabled"); + if (tcpKeepAliveEnabled != null) { + connectorBuilder.tcpKeepAliveEnabled(Boolean.valueOf(XML.getValue(tcpKeepAliveEnabled).trim())); + } + } + { + Element tcpNoDelayEnabled = XML.getChild(legacyConfig, "tcpNoDelayEnabled"); + if (tcpNoDelayEnabled != null) { + connectorBuilder.tcpNoDelay(Boolean.valueOf(XML.getValue(tcpNoDelayEnabled).trim())); + } + } + { + Element tcpListenBacklogLength = XML.getChild(legacyConfig, "tcpListenBacklogLength"); + if (tcpListenBacklogLength != null) { + connectorBuilder.acceptQueueSize(Integer.parseInt(XML.getValue(tcpListenBacklogLength).trim())); + } + } + { + Element idleConnectionTimeout = XML.getChild(legacyConfig, "idleConnectionTimeout"); + if (idleConnectionTimeout != null) { + connectorBuilder.idleTimeout(Double.parseDouble(XML.getValue(idleConnectionTimeout).trim())); + } + } + { + Element soLinger = XML.getChild(legacyConfig, "soLinger"); + if (soLinger != null) { + + connectorBuilder.soLingerTime((int) Double.parseDouble(XML.getValue(soLinger).trim())); + } + } + { + Element sendBufferSize = XML.getChild(legacyConfig, "sendBufferSize"); + if (sendBufferSize != null) { + connectorBuilder.outputBufferSize(Integer.parseInt(XML.getValue(sendBufferSize).trim())); + } + } + { + Element maxHeaderSize = XML.getChild(legacyConfig, "maxHeaderSize"); + if (maxHeaderSize != null) { + connectorBuilder.headerCacheSize(Integer.parseInt(XML.getValue(maxHeaderSize).trim())); + } + } + + Element ssl = XML.getChild(legacyConfig, "ssl"); + Element sslEnabled = XML.getChild(ssl, "enabled"); + if (ssl != null && + sslEnabled != null && + Boolean.parseBoolean(XML.getValue(sslEnabled).trim())) { + ConnectorConfig.Ssl.Builder sslBuilder = new ConnectorConfig.Ssl.Builder(); + sslBuilder.enabled(true); + { + Element keyStoreType = XML.getChild(ssl, "keyStoreType"); + if (keyStoreType != null) { + sslBuilder.keyStoreType(KeyStoreType.Enum.valueOf(XML.getValue(keyStoreType).trim())); + } + } + { + Element keyStorePath = XML.getChild(ssl, "keyStorePath"); + if (keyStorePath != null) { + sslBuilder.keyStorePath(XML.getValue(keyStorePath).trim()); + } + } + { + Element trustStorePath = XML.getChild(ssl, "trustStorePath"); + if (trustStorePath != null) { + sslBuilder.trustStorePath(XML.getValue(trustStorePath).trim()); + } + } + { + Element keyDBKey = XML.getChild(ssl, "keyDBKey"); + if (keyDBKey != null) { + sslBuilder.keyDbKey(XML.getValue(keyDBKey).trim()); + } + } + { + Element algorithm = XML.getChild(ssl, "algorithm"); + if (algorithm != null) { + sslBuilder.sslKeyManagerFactoryAlgorithm(XML.getValue(algorithm).trim()); + } + } + { + Element protocol = XML.getChild(ssl, "protocol"); + if (protocol != null) { + sslBuilder.protocol(XML.getValue(protocol).trim()); + } + } + connectorBuilder.ssl(sslBuilder); + } + } + + connectorBuilder.listenPort(listenPort); + connectorBuilder.name(name); + } + + public String getName() { + return name; + } + + public int getListenPort() { + return listenPort; + } + + public void setListenPort(int httpPort) { + this.listenPort = httpPort; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Filter.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Filter.java new file mode 100644 index 00000000000..f9a416a6fd6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Filter.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.http; + +import com.yahoo.component.chain.model.ChainedComponentModel; +import com.yahoo.vespa.model.container.component.chain.ChainedComponent; + +/** + * @author tonytv + * @author gjoranv + */ +public class Filter extends ChainedComponent<ChainedComponentModel> { + + public Filter(ChainedComponentModel model) { + super(model); + } + + public FilterConfigProvider addAndInjectConfigProvider() { + FilterConfigProvider filterConfigProvider = new FilterConfigProvider(model); + addComponent(filterConfigProvider); + inject(filterConfigProvider); + return filterConfigProvider; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/FilterChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/FilterChains.java new file mode 100644 index 00000000000..f38c99bd631 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/FilterChains.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.http; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.container.component.SimpleComponent; +import com.yahoo.vespa.model.container.component.chain.Chain; +import com.yahoo.vespa.model.container.component.chain.Chains; + +/** + * @author tonytv + */ +public class FilterChains extends Chains<Chain<Filter>> { + + public FilterChains(AbstractConfigProducer parent) { + super(parent, "filters"); + + addChild(new SimpleComponent("com.yahoo.container.http.filter.FilterChainRepository")); + } + + public boolean hasChain(ComponentId filterChain) { + for (Chain<Filter> chain : allChains().allComponents()) { + if (chain.getId().equals(filterChain)) + return true; + } + return false; + } + + public boolean hasChainThatInherits(ComponentId filterChain) { + for (Chain<Filter> chain : allChains().allComponents()) { + for (ComponentSpecification spec : chain.getChainSpecification().inheritance.chainSpecifications) { + if(spec.toId().equals(filterChain)) + return true; + } + } + return false; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/FilterConfigProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/FilterConfigProvider.java new file mode 100644 index 00000000000..5c6ce3454a8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/FilterConfigProvider.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.http; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.model.ChainedComponentModel; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.container.core.http.HttpFilterConfig; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.component.SimpleComponent; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +import static com.yahoo.container.core.http.HttpFilterConfig.Param; + +/** + * @author gjoranv + * @since 5.1.23 + */ +public class FilterConfigProvider extends SimpleComponent implements HttpFilterConfig.Producer { + + private static final ComponentSpecification filterConfigProviderClass = + ComponentSpecification.fromString(com.yahoo.container.FilterConfigProvider.class.getName()); + + private final ChainedComponentModel filterModel; + private HashMap<String, String> configMap = new LinkedHashMap<>(); + + public FilterConfigProvider(ChainedComponentModel filterModel) { + super(new ComponentModel( + new BundleInstantiationSpecification( + configProviderId(filterModel.getComponentId()), + filterConfigProviderClass, + null))); + + this.filterModel = filterModel; + } + + @Override + public void getConfig(HttpFilterConfig.Builder builder) { + builder.filterName(filterModel.getComponentId().stringValue()) + .filterClass(filterModel.getClassId().stringValue()); + + for (Map.Entry<String, String> param : configMap.entrySet()) { + builder.param( + new Param.Builder() + .name(param.getKey()) + .value(param.getValue())); + } + } + + public String putConfig(String key, String value) { + return configMap.put(key, value); + } + + static ComponentId configProviderId(ComponentId filterId) { + return ComponentId.fromString("filterConfig").nestInNamespace(filterId); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java new file mode 100644 index 00000000000..7c44ce8e44f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/Http.java @@ -0,0 +1,123 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.http; + +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.container.jdisc.config.HttpServerConfig; +import com.yahoo.jdisc.http.ServerConfig; +import com.yahoo.vespa.model.container.component.chain.Chain; +import com.yahoo.vespa.model.container.component.chain.ChainedComponent; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +// This can be collapsed into JettyHttpServer now +/** + * @author tonytv + */ +public class Http extends AbstractConfigProducer<AbstractConfigProducer<?>> + implements HttpServerConfig.Producer, ServerConfig.Producer { + + public static class Binding { + public final ComponentSpecification filterId; + public final String binding; + + public Binding(ComponentSpecification filterId, String binding) { + this.filterId = filterId; + this.binding = binding; + } + } + + private FilterChains filterChains; + private JettyHttpServer httpServer; + public final List<Binding> bindings; + + public Http(List<Binding> bindings) { + super( "http"); + this.bindings = Collections.unmodifiableList(bindings); + } + + public void setFilterChains(FilterChains filterChains) { + this.filterChains = filterChains; + } + + public FilterChains getFilterChains() { + return filterChains; + } + + public JettyHttpServer getHttpServer() { + return httpServer; + } + + public void setHttpServer(JettyHttpServer newServer) { + JettyHttpServer oldServer = this.httpServer; + this.httpServer = newServer; + + if (oldServer == null && newServer != null) { + addChild(newServer); + } else if (newServer == null && oldServer != null) { + removeChild(oldServer); + } else if (newServer == null && oldServer == null) { + //do nothing + } else { + //none of them are null + removeChild(oldServer); + addChild(newServer); + } + } + + public void removeAllServers() { + setHttpServer(null); + } + + public List<Binding> getBindings() { + return bindings; + } + + @Override + public void getConfig(HttpServerConfig.Builder builder) { + for (Binding binding: bindings) + builder.filter(filterBindings(binding)); + } + + @Override + public void getConfig(ServerConfig.Builder builder) { + for (final Binding binding : bindings) { + builder.filter( + new ServerConfig.Filter.Builder() + .id(binding.filterId.stringValue()) + .binding(binding.binding)); + } + } + + static HttpServerConfig.Filter.Builder filterBindings(Binding binding) { + HttpServerConfig.Filter.Builder builder = new HttpServerConfig.Filter.Builder(); + builder.id(binding.filterId.stringValue()). + binding(binding.binding); + return builder; + } + + + @Override + public void validate() throws Exception { + validate(bindings); + } + + void validate(Collection<Binding> bindings) { + if (!bindings.isEmpty()) { + if (filterChains == null) + throw new IllegalArgumentException("Null FilterChains is not allowed when there are filter bindings!"); + + ComponentRegistry<ChainedComponent<?>> filters = filterChains.componentsRegistry(); + ComponentRegistry<Chain<Filter>> chains = filterChains.allChains(); + + for (Binding binding: bindings) { + if (filters.getComponent(binding.filterId) == null && chains.getComponent(binding.filterId) == null) + throw new RuntimeException("Can't find filter " + binding.filterId + " for binding " + binding.binding); + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java new file mode 100644 index 00000000000..30a9e12bf92 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/JettyHttpServer.java @@ -0,0 +1,73 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.http; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.jdisc.http.ServerConfig; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.component.SimpleComponent; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static com.yahoo.component.ComponentSpecification.fromString; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.16.0 + */ +public class JettyHttpServer extends SimpleComponent implements ServerConfig.Producer { + + private List<ConnectorFactory> connectorFactories = new ArrayList<>(); + + public JettyHttpServer(ComponentId id) { + super(new ComponentModel( + new BundleInstantiationSpecification(id, + fromString("com.yahoo.jdisc.http.server.jetty.JettyHttpServer"), + fromString("jdisc_http_service")) + )); + final FilterBindingsProviderComponent filterBindingsProviderComponent = new FilterBindingsProviderComponent(id); + addChild(filterBindingsProviderComponent); + inject(filterBindingsProviderComponent); + } + + public void addConnector(ConnectorFactory connectorFactory) { + connectorFactories.add(connectorFactory); + addChild(connectorFactory); + } + + public void removeConnector(ConnectorFactory connectorFactory) { + if (connectorFactory == null) { + return; + } + removeChild(connectorFactory); + connectorFactories.remove(connectorFactory); + } + + public List<ConnectorFactory> getConnectorFactories() { + return Collections.unmodifiableList(connectorFactories); + } + + @Override + public void getConfig(ServerConfig.Builder builder) { + } + + static ComponentModel providerComponentModel(final ComponentId parentId, String className) { + final ComponentSpecification classNameSpec = new ComponentSpecification( + className); + return new ComponentModel(new BundleInstantiationSpecification( + classNameSpec.nestInNamespace(parentId), + classNameSpec, + null)); + } + + public static final class FilterBindingsProviderComponent extends SimpleComponent { + public FilterBindingsProviderComponent(final ComponentId parentId) { + super(providerComponentModel(parentId, "com.yahoo.container.jdisc.FilterBindingsProvider")); + } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/package-info.java new file mode 100644 index 00000000000..304a6cd5052 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.model.container.http; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterBuilder.java new file mode 100644 index 00000000000..9f033160f6d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterBuilder.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.http.xml; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.builder.xml.dom.DomComponentBuilder; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.ChainedComponentModelBuilder; +import com.yahoo.vespa.model.container.http.Filter; +import com.yahoo.vespa.model.container.http.FilterConfigProvider; +import org.w3c.dom.Element; + +/** + * @author tonytv + * @author gjoranv + */ +public class FilterBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Filter> { + + protected Filter doBuild(AbstractConfigProducer ancestor, Element filterElement) { + ChainedComponentModelBuilder modelBuilder = new ChainedComponentModelBuilder(filterElement); + Filter filter = new Filter(modelBuilder.build()); + DomComponentBuilder.addChildren(ancestor, filterElement, filter); + addFilterConfig(filterElement, filter); + + return filter; + } + + private static void addFilterConfig(Element filterElement, Filter filter) { + Element filterConfigElement = XML.getChild(filterElement, "filter-config"); + if (filterConfigElement == null) + return; + + FilterConfigProvider filterConfigProvider = filter.addAndInjectConfigProvider(); + putFilterConfig(filterConfigElement, filterConfigProvider); + } + + private static void putFilterConfig(Element filterConfigElement, FilterConfigProvider filterConfigProvider) { + for (Element e : XML.getChildren(filterConfigElement)) { + filterConfigProvider.putConfig(e.getTagName(), XML.getValue(e)); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainBuilder.java new file mode 100644 index 00000000000..65f12cf4e3f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainBuilder.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.http.xml; + +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainBuilderBase; +import com.yahoo.vespa.model.container.component.chain.Chain; +import com.yahoo.vespa.model.container.http.Filter; +import org.w3c.dom.Element; + +import java.util.Collection; +import java.util.Collections; +import java.util.Map; + +import static com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder.ComponentType; + +/** + * @author tonytv + */ +public class FilterChainBuilder extends DomChainBuilderBase<Filter, Chain<Filter>> { + private static Collection<ComponentType<Filter>> allowedComponentTypes = Collections.singleton(ComponentType.filter); + + + public FilterChainBuilder(Map<String, ComponentType> outerFilterTypeByComponentName) { + super(allowedComponentTypes, outerFilterTypeByComponentName); + } + + @Override + protected Chain<Filter> buildChain(AbstractConfigProducer ancestor, Element producerSpec, ChainSpecification specWithoutInnerComponents) { + return new Chain<>(specWithoutInnerComponents); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainsBuilder.java new file mode 100644 index 00000000000..df38402be91 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/FilterChainsBuilder.java @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.http.xml; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.builder.xml.dom.chains.ChainsBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.ComponentsBuilder.ComponentType; +import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainBuilderBase; +import com.yahoo.vespa.model.builder.xml.dom.chains.DomChainsBuilder; +import com.yahoo.vespa.model.container.component.chain.Chain; +import com.yahoo.vespa.model.container.http.Filter; +import com.yahoo.vespa.model.container.http.FilterChains; +import org.w3c.dom.Element; + +import java.util.*; + +/** + * @author tonytv + */ +public class FilterChainsBuilder extends DomChainsBuilder<Filter, Chain<Filter>, FilterChains> { + private static final Collection<ComponentType<Filter>> allowedComponentTypes = + Collections.singleton(ComponentType.filter); + + //TODO: simplify + private static final Map<String, Class<? extends DomChainBuilderBase<? extends Filter, ? extends Chain<Filter>>>> chainType2BuilderClass = + Collections.unmodifiableMap( + new LinkedHashMap<String, Class<? extends DomChainBuilderBase<? extends Filter, ? extends Chain<Filter>>>>() {{ + put("chain", FilterChainBuilder.class); // TODO: remove when 'chain' under 'http' is removed from xml schema + put("request-chain", FilterChainBuilder.class); + put("response-chain", FilterChainBuilder.class); + }}); + + public FilterChainsBuilder() { + super(null, allowedComponentTypes, null); + } + + @Override + protected FilterChains newChainsInstance(AbstractConfigProducer parent) { + return new FilterChains(parent); + } + + @Override + protected ChainsBuilder<Filter, Chain<Filter>> readChains( + AbstractConfigProducer ancestor, + List<Element> allChainsElems, Map<String, ComponentsBuilder.ComponentType> outerComponentTypeByComponentName) { + + return new ChainsBuilder<>(ancestor, allChainsElems, outerComponentTypeByComponentName, chainType2BuilderClass); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java new file mode 100644 index 00000000000..d3ccefc3e26 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/HttpBuilder.java @@ -0,0 +1,83 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.http.xml; + +import com.yahoo.component.ComponentSpecification; +import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.log.LogLevel; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.http.FilterChains; +import com.yahoo.vespa.model.container.http.Http; +import org.w3c.dom.Element; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author tonytv + * @author gjoranv + */ +public class HttpBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Http> { + + @Override + protected Http doBuild(AbstractConfigProducer ancestor, Element spec) { + FilterChains filterChains; + List<Http.Binding> bindings = new ArrayList<>(); + + Element filteringElem = XML.getChild(spec, "filtering"); + if (filteringElem != null) { + filterChains = new FilterChainsBuilder().build(ancestor, filteringElem); + bindings = readFilterBindings(filteringElem); + } else { + filterChains = new FilterChainsBuilder().newChainsInstance(ancestor); + } + + Http http = new Http(bindings); + http.setFilterChains(filterChains); + + buildHttpServers(ancestor, http, spec); + + return http; + } + + private List<Http.Binding> readFilterBindings(Element filteringSpec) { + List<Http.Binding> result = new ArrayList<>(); + + for (Element child: XML.getChildren(filteringSpec)) { + String tagName = child.getTagName(); + if ((tagName.equals("request-chain") || tagName.equals("response-chain"))) { + ComponentSpecification chainId = XmlHelper.getIdRef(child); + + for (Element bindingSpec: XML.getChildren(child, "binding")) { + String binding = XML.getValue(bindingSpec); + result.add(new Http.Binding(chainId, binding)); + } + } + } + return result; + } + + private void buildHttpServers(AbstractConfigProducer ancestor, Http http, Element spec) { + http.setHttpServer(new JettyHttpServerBuilder().build(ancestor, spec)); + } + + static int readPort(Element spec, DeployState deployState) { + String portString = spec.getAttribute("port"); + + int port = Integer.parseInt(portString); + if (port < 0) + throw new IllegalArgumentException(String.format("Invalid port %d.", port)); + + int legalPortInHostedVespa = Container.BASEPORT; + if (deployState.isHostedVespa() && port != legalPortInHostedVespa) { + deployState.getDeployLogger().log(LogLevel.WARNING, + String.format("Trying to set port to %d for http server with id %s. You cannot set port to anything else than %s", + port, spec.getAttribute("id"), legalPortInHostedVespa)); + } + + return port; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java new file mode 100644 index 00000000000..be92c00652f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyConnectorBuilder.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.http.xml; + +import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.container.http.ConnectorFactory; +import org.w3c.dom.Element; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.21.0 + */ +public class JettyConnectorBuilder extends VespaDomBuilder.DomConfigProducerBuilder<ConnectorFactory> { + @Override + protected ConnectorFactory doBuild(AbstractConfigProducer ancestor, Element serverSpec) { + String name = XmlHelper.getIdString(serverSpec); + int port = HttpBuilder.readPort(serverSpec, ancestor.getRoot().getDeployState()); + + Element legacyServerConfig = XML.getChild(serverSpec, "config"); + if (legacyServerConfig != null) { + String configName = legacyServerConfig.getAttribute("name"); + if (!configName.equals("container.jdisc.config.http-server")) { + legacyServerConfig = null; + } + } + return new ConnectorFactory(name, port, legacyServerConfig); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java new file mode 100644 index 00000000000..c28849dba3f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/xml/JettyHttpServerBuilder.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.http.xml; + +import com.yahoo.component.ComponentId; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.container.http.ConnectorFactory; +import com.yahoo.vespa.model.container.http.JettyHttpServer; +import org.w3c.dom.Element; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.17.0 + */ +public class JettyHttpServerBuilder extends VespaDomBuilder.DomConfigProducerBuilder<JettyHttpServer> { + @Override + protected JettyHttpServer doBuild(AbstractConfigProducer ancestor, Element http) { + JettyHttpServer jettyHttpServer = new JettyHttpServer(new ComponentId("jdisc-jetty")); + for (Element serverSpec: XML.getChildren(http, "server")) { + ConnectorFactory connectorFactory = new JettyConnectorBuilder().build(ancestor, serverSpec); + jettyHttpServer.addConnector(connectorFactory); + } + + return jettyHttpServer; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/Jersey2Servlet.java b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/Jersey2Servlet.java new file mode 100644 index 00000000000..def9442f9e4 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/Jersey2Servlet.java @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.jersey; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.VersionSpecification; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.component.Servlet; + +/** + * @author tonytv + */ +public class Jersey2Servlet extends Servlet { + public static final String BUNDLE = "container-jersey2"; + public static final String CLASS = "com.yahoo.container.servlet.jersey.JerseyServletProvider"; + + private static final ComponentId REST_API_NAMESPACE = ComponentId.fromString("rest-api"); + + public Jersey2Servlet(String bindingPath) { + super(new ComponentModel( + new BundleInstantiationSpecification(idSpecFromPath(bindingPath), + ComponentSpecification.fromString(CLASS), + ComponentSpecification.fromString(BUNDLE))), + bindingPath + "/*"); + } + + private static ComponentSpecification idSpecFromPath(String path) { + return new ComponentSpecification( + RestApi.idFromPath(path), + VersionSpecification.emptyVersionSpecification, + REST_API_NAMESPACE); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/JerseyHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/JerseyHandler.java new file mode 100644 index 00000000000..3c7c9dde86a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/JerseyHandler.java @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.jersey; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.component.Handler; + +/** + * @author gjoranv + * @since 5.6 + */ +public class JerseyHandler extends Handler<AbstractConfigProducer<?>> { + + public static final String BUNDLE = "container-jersey"; + public static final String CLASS = "com.yahoo.container.jdisc.jersey.JerseyHandler"; + + public JerseyHandler(String bindingPath) { + super(new ComponentModel(bundleSpec(CLASS, BUNDLE, bindingPath))); + } + + public static BundleInstantiationSpecification bundleSpec(String className, String bundle, String bindingPath) { + return BundleInstantiationSpecification.getFromStrings( + className + "-" + RestApi.idFromPath(bindingPath), + className, + bundle); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/RestApi.java b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/RestApi.java new file mode 100644 index 00000000000..f546e43318f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/RestApi.java @@ -0,0 +1,84 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.jersey; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.container.config.jersey.JerseyInitConfig; +import com.yahoo.vespa.model.container.component.Component; + +import java.util.Optional; + +/** + * @author gjoranv + * @since 5.6 + */ +public class RestApi extends AbstractConfigProducer<AbstractConfigProducer<?>> implements + JerseyInitConfig.Producer +{ + public final boolean isJersey2; + private final String bindingPath; + private final Component<?, ?> jerseyHandler; + private RestApiContext restApiContext; + + public RestApi(String bindingPath, boolean isJersey2) { + super(idFromPath(bindingPath)); + this.bindingPath = bindingPath; + this.isJersey2 = isJersey2; + + jerseyHandler = isJersey2 ? + createJersey2Servlet(this.bindingPath): + createJersey1Handler(this.bindingPath); + addChild(jerseyHandler); + } + + public static String idFromPath(String path) { + return path.replaceAll("/", "|"); + } + + private Jersey2Servlet createJersey2Servlet(String bindingPath) { + return new Jersey2Servlet(bindingPath); + } + + private static JerseyHandler createJersey1Handler(String bindingPath) { + JerseyHandler jerseyHandler = new JerseyHandler(bindingPath); + jerseyHandler.addServerBindings(getBindings(bindingPath)); + return jerseyHandler; + } + + public String getBindingPath() { + return bindingPath; + } + + @Override + public void getConfig(JerseyInitConfig.Builder builder) { + builder.jerseyMapping(bindingPath); + } + + public void setRestApiContext(RestApiContext restApiContext) { + this.restApiContext = restApiContext; + addChild(restApiContext); + jerseyHandler.inject(restApiContext); + } + + public RestApiContext getContext() { return restApiContext; } + + public Optional<JerseyHandler> getJersey1Handler() { + return isJersey2 ? + Optional.empty(): + Optional.of((JerseyHandler)jerseyHandler); + } + + public Optional<Jersey2Servlet> getJersey2Servlet() { + return isJersey2 ? + Optional.of((Jersey2Servlet)jerseyHandler) : + Optional.empty(); + } + + private static String[] getBindings(String bindingPath) { + String bindingWithoutScheme = "://*/" + bindingPath + "/*"; + return new String[] {"http" + bindingWithoutScheme, "https" + bindingWithoutScheme}; + } + + public void prepare() { + restApiContext.prepare(); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/RestApiContext.java b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/RestApiContext.java new file mode 100644 index 00000000000..e18a40e1c6a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/RestApiContext.java @@ -0,0 +1,149 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.jersey; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.container.di.config.JerseyBundlesConfig; +import com.yahoo.container.di.config.JerseyInjectionConfig; +import com.yahoo.container.di.config.JerseyInjectionConfig.Inject; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.SimpleComponent; +import edu.umd.cs.findbugs.annotations.Nullable; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; +import java.util.logging.Logger; + +/** + * @author gjoranv + * @since 5.16 + */ +public class RestApiContext extends SimpleComponent implements + JerseyBundlesConfig.Producer, + JerseyInjectionConfig.Producer +{ + private static final Logger log = Logger.getLogger(RestApi.class.getName()); + public static final String CONTAINER_CLASS = "com.yahoo.container.di.config.RestApiContext"; + + private final List<BundleInfo> bundles = new ArrayList<>(); + + // class name -> componentId + private final Map<String, String> injectComponentForClass = new LinkedHashMap<>(); + + private final String bindingPath; + + @Nullable + private ContainerCluster containerCluster; + + public RestApiContext(AbstractConfigProducer<?> ancestor, String bindingPath) { + super(componentModel(bindingPath)); + this.bindingPath = bindingPath; + + if (ancestor instanceof ContainerCluster) + containerCluster = (ContainerCluster)ancestor; + + } + + private static ComponentModel componentModel(String bindingPath) { + return new ComponentModel(BundleInstantiationSpecification.getFromStrings( + CONTAINER_CLASS + "-" + RestApi.idFromPath(bindingPath), + CONTAINER_CLASS, + null)); + } + + @Override + public void getConfig(JerseyBundlesConfig.Builder builder) { + builder.bundles(createBundlesConfig(bundles)); + } + + private List<JerseyBundlesConfig.Bundles.Builder> createBundlesConfig(List<BundleInfo> bundles) { + List<JerseyBundlesConfig.Bundles.Builder> builders = new ArrayList<>(); + for (BundleInfo b : bundles) { + builders.add( + new JerseyBundlesConfig.Bundles.Builder() + .spec(b.spec) + .packages(b.getPackagesToScan()) + ); + } + return builders; + } + + public void addBundles(Collection<BundleInfo> newBundles) { + bundles.addAll(newBundles); + } + + @Override + public void getConfig(JerseyInjectionConfig.Builder builder) { + for (Map.Entry<String, String> i : injectComponentForClass.entrySet()) { + builder.inject(new Inject.Builder() + .forClass(i.getKey()) + .instance(i.getValue())); + } + } + + public void addInjections(Map<String, String> injections) { + injectComponentForClass.putAll(injections); + } + + @Override + public void validate() throws Exception { + super.validate(); + + if (bundles.isEmpty()) + log.warning("No bundles in rest-api '" + bindingPath + + "' - components will only be loaded from classpath."); + } + + public void prepare() { + if (containerCluster == null) return; + + containerCluster.getAllComponents().stream(). + filter(isCycleGeneratingComponent.negate()). + forEach(this::inject); + } + + + /* + * Example problem + * + * RestApiContext -> ApplicationStatusHandler -> ComponentRegistry<HttpServer> -> JettyHttpServer -> ComponentRegistry<Jersey2Servlet> -> RestApiContext + */ + private Predicate<Component> isCycleGeneratingComponent = component -> { + switch (component.getClassId().getName()) { + case CONTAINER_CLASS: + case JerseyHandler.CLASS: + case Jersey2Servlet.CLASS: + case "com.yahoo.jdisc.http.server.jetty.JettyHttpServer": + case "com.yahoo.container.handler.observability.ApplicationStatusHandler": + return true; + default: + return false; + } + }; + + public static class BundleInfo { + // SymbolicName[:Version] + public final String spec; + + private final List<String> packagesToScan = new ArrayList<>(); + + public BundleInfo(String spec) { + this.spec = spec; + } + + public List<String> getPackagesToScan() { + return packagesToScan; + } + + public void addPackageToScan(String pkg) { + packagesToScan.add(pkg); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/xml/RestApiBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/xml/RestApiBuilder.java new file mode 100644 index 00000000000..ed7239e0bbf --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/jersey/xml/RestApiBuilder.java @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.jersey.xml; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.container.jersey.RestApi; +import com.yahoo.vespa.model.container.jersey.RestApiContext; +import org.w3c.dom.Element; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalAttribute; + +/** + * @author gjoranv + * @since 5.6 + */ +public class RestApiBuilder extends VespaDomBuilder.DomConfigProducerBuilder<RestApi> { + + @Override + protected RestApi doBuild(AbstractConfigProducer ancestor, Element spec) { + String bindingPath = spec.getAttribute("path"); + boolean jersey2 = Boolean.parseBoolean(getOptionalAttribute(spec, "jersey2").orElse("false")); + RestApi restApi = new RestApi(bindingPath, jersey2); + + restApi.setRestApiContext( + createRestApiContext(ancestor, spec, bindingPath)); + return restApi; + } + + private RestApiContext createRestApiContext(AbstractConfigProducer ancestor, Element spec, String bindingPath) { + RestApiContext restApiContext = new RestApiContext(ancestor, bindingPath); + + restApiContext.addBundles(getBundles(spec)); + + return restApiContext; + } + + private List<RestApiContext.BundleInfo> getBundles(Element spec) { + List<RestApiContext.BundleInfo> bundles = new ArrayList<>(); + for (Element bundleElement : XML.getChildren(spec, "components")) { + bundles.add(getBundle(bundleElement)); + } + return bundles; + } + + private RestApiContext.BundleInfo getBundle(Element bundleElement) { + RestApiContext.BundleInfo bundle = new RestApiContext.BundleInfo( + bundleElement.getAttribute("bundle")); + + for (Element packageElement : XML.getChildren(bundleElement, "package")) + bundle.addPackageToScan(XML.getValue(packageElement)); + + return bundle; + } + + // TODO: use for naming injected components instead + private Map<String, String> getInjections(Element spec) { + Map<String, String> injectForClass = new LinkedHashMap<>(); + for (Element injectElement : XML.getChildren(spec, "inject")) { + injectForClass.put(injectElement.getAttribute("for-class"), + injectElement.getAttribute("component")); + } + return injectForClass; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/container/package-info.java new file mode 100644 index 00000000000..74acf6dcc7f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.model.container; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChain.java b/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChain.java new file mode 100644 index 00000000000..ae7bcb64f66 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChain.java @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.processing; + +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.vespa.model.container.component.chain.Chain; + +/** + * Represents a processing chain in the config model + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + * @since 5.1.6 + */ +public class ProcessingChain extends Chain<Processor> { + + public ProcessingChain(ChainSpecification specWithoutInnerProcessors) { + super(specWithoutInnerProcessors); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java new file mode 100644 index 00000000000..032aacccca8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/processing/ProcessingChains.java @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.processing; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.container.component.chain.Chains; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Root config producer for processing + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + * @since 5.1.6 + */ +public class ProcessingChains extends Chains<ProcessingChain> { + public static final String[] defaultBindings = new String[] + {"http://*/processing/*", "https://*/processing/*"}; + + + public ProcessingChains(AbstractConfigProducer parent, String subId) { + super(parent, subId); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/processing/Processor.java b/config-model/src/main/java/com/yahoo/vespa/model/container/processing/Processor.java new file mode 100644 index 00000000000..b16243576f9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/processing/Processor.java @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.processing; + +import com.yahoo.component.chain.model.ChainedComponentModel; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.container.component.chain.ChainedComponent; + +/** + * Representation of a Processor in the configuration model + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + * @since 5.1.6 + */ +public class Processor extends ChainedComponent<ChainedComponentModel> { + + public Processor(ChainedComponentModel model) { + super(model); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerHttpGateway.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerHttpGateway.java new file mode 100644 index 00000000000..88d4a0c8599 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerHttpGateway.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search; + +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerCluster; + +/** + * @author gjoranv + */ +public class ContainerHttpGateway extends Container { + + public ContainerHttpGateway(ContainerCluster parent, String name, int wantedPort) { + super(parent, name); + + // TODO: when this class is removed, all ports for the gateway will map to standard container ports + // this is just a tjuvtriks to keep the old gateway port allocation for now. + setBasePort(wantedPort); + } + + @Override + public String getServiceType() { return "container-httpgateway"; } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java new file mode 100644 index 00000000000..0caa84390f8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -0,0 +1,204 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search; + +import com.yahoo.binaryprefix.BinaryPrefix; +import com.yahoo.binaryprefix.BinaryScaledAmount; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.prelude.fastsearch.FS4ResourcePool; +import com.yahoo.prelude.semantics.SemanticRulesConfig; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.ContainerSubsystem; +import com.yahoo.vespa.model.container.search.searchchain.HttpProvider; +import com.yahoo.vespa.model.container.search.searchchain.LocalProvider; +import com.yahoo.vespa.model.container.search.searchchain.SearchChains; +import com.yahoo.vespa.model.search.*; +import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.search.config.QrStartConfig; +import com.yahoo.vespa.configdefinition.IlscriptsConfig; +import com.yahoo.container.QrSearchersConfig; +import com.yahoo.search.query.profile.config.QueryProfilesConfig; +import com.yahoo.search.pagetemplates.PageTemplatesConfig; + +import java.util.*; + +/** + * @author gjoranv + * @author tonytv + * @since 5.1.9 + */ +public class ContainerSearch extends ContainerSubsystem<SearchChains> + implements + IndexInfoConfig.Producer, + IlscriptsConfig.Producer, + QrSearchersConfig.Producer, + QrStartConfig.Producer, + QueryProfilesConfig.Producer, + SemanticRulesConfig.Producer, + PageTemplatesConfig.Producer +{ + private final List<AbstractSearchCluster> systems = new LinkedList<>(); + private Options options = null; + + //For legacy qrs clusters only. + private BinaryScaledAmount totalCacheSize = new BinaryScaledAmount(); + + private QueryProfiles queryProfiles; + private SemanticRules semanticRules; + private PageTemplates pageTemplates; + private boolean hostedVespa = false; + + public ContainerSearch(ContainerCluster cluster, SearchChains chains, Options options) { + super(chains); + this.options = options; + this.hostedVespa = cluster.isHostedVespa(); + cluster.addComponent(getFS4ResourcePool()); + } + + private Component<?, ComponentModel> getFS4ResourcePool() { + BundleInstantiationSpecification spec = BundleInstantiationSpecification. + getInternalSearcherSpecificationFromStrings(FS4ResourcePool.class.getName(), null); + return new Component<>(new ComponentModel(spec)); + } + + public void connectSearchClusters(Map<String, AbstractSearchCluster> searchClusters) { + systems.addAll(searchClusters.values()); + initializeSearchChains(searchClusters); + } + + // public for testing + public void initializeSearchChains(Map<String, ? extends AbstractSearchCluster> searchClusters) { + getChains().initialize(searchClusters, totalCacheSize); + + QrsCache defaultCacheOptions = getOptions().cacheSettings.get(""); + if (defaultCacheOptions != null) { + for (LocalProvider localProvider: getChains().localProviders()) { + localProvider.setCacheSize(defaultCacheOptions.size); + } + } + + for (LocalProvider localProvider: getChains().localProviders()) { + QrsCache cacheOptions = getOptions().cacheSettings.get(localProvider.getClusterName()); + if (cacheOptions != null) { + localProvider.setCacheSize(cacheOptions.size); + } + } + } + + public void setTotalCacheSize(BinaryScaledAmount totalCacheSize) { + this.totalCacheSize = totalCacheSize; + } + + public void setQueryProfiles(QueryProfiles queryProfiles) { + this.queryProfiles = queryProfiles; + } + + public void setSemanticRules(SemanticRules semanticRules) { + this.semanticRules = semanticRules; + } + + public void setPageTemplates(PageTemplates pageTemplates) { + this.pageTemplates = pageTemplates; + } + + @Override + public void getConfig(QueryProfilesConfig.Builder builder) { + if (queryProfiles!=null) queryProfiles.getConfig(builder); + } + + @Override + public void getConfig(SemanticRulesConfig.Builder builder) { + if (semanticRules!=null) semanticRules.getConfig(builder); + } + + @Override + public void getConfig(PageTemplatesConfig.Builder builder) { + if (pageTemplates!=null) pageTemplates.getConfig(builder); + } + + @Override + public void getConfig(QrStartConfig.Builder qsB) { + final QrStartConfig.Jvm.Builder internalBuilder = new QrStartConfig.Jvm.Builder(); + if (hostedVespa) { + internalBuilder.heapSizeAsPercentageOfPhysicalMemory(33); + } + qsB.jvm(internalBuilder.directMemorySizeCache(totalCacheSizeMb())); + } + + private int totalCacheSizeMb() { + if (!totalCacheSize.equals(new BinaryScaledAmount())) { + return (int) totalCacheSize.as(BinaryPrefix.mega); + } else { + return totalHttpProviderCacheSize(); + } + } + + private int totalHttpProviderCacheSize() { + int totalCacheSizeMb = 0; + for (HttpProvider provider: getChains().httpProviders()) + totalCacheSizeMb += provider.cacheSizeMB(); + + return totalCacheSizeMb; + } + + @Override + public void getConfig(IndexInfoConfig.Builder builder) { + for (AbstractSearchCluster sc : systems) { + sc.getConfig(builder); + } + } + + @Override + public void getConfig(IlscriptsConfig.Builder builder) { + for (AbstractSearchCluster sc : systems) { + sc.getConfig(builder); + } + } + + @Override + public void getConfig(QrSearchersConfig.Builder builder) { + for (int i = 0; i < systems.size(); i++) { + AbstractSearchCluster sys = findClusterWithId(systems, i); + QrSearchersConfig.Searchcluster.Builder scB = new QrSearchersConfig.Searchcluster.Builder(). + name(sys.getClusterName()); + for (AbstractSearchCluster.SearchDefinitionSpec spec : sys.getLocalSDS()) { + scB.searchdef(spec.getSearchDefinition().getName()); + } + scB.rankprofiles(new QrSearchersConfig.Searchcluster.Rankprofiles.Builder().configid(sys.getConfigId())); + scB.indexingmode(QrSearchersConfig.Searchcluster.Indexingmode.Enum.valueOf(sys.getIndexingModeName())); + if (sys instanceof IndexedSearchCluster) { + scB.rowbits(sys.getRowBits()); + for (Dispatch tld: ((IndexedSearchCluster)sys).getTLDs()) { + scB.dispatcher(new QrSearchersConfig.Searchcluster.Dispatcher.Builder(). + host(tld.getHostname()). + port(tld.getDispatchPort())); + } + } else { + scB.storagecluster(new QrSearchersConfig.Searchcluster.Storagecluster.Builder(). + routespec(((StreamingSearchCluster)sys).getStorageRouteSpec())); + } + builder.searchcluster(scB); + } + } + + private static AbstractSearchCluster findClusterWithId(List<AbstractSearchCluster> clusters, int index) { + for (AbstractSearchCluster sys : clusters) { + if (sys.getClusterIndex() == index) { + return sys; + } + } + throw new IllegalArgumentException("No search cluster with index " + index + " exists"); + } + + public Options getOptions() { + return options; + } + + /** + * Struct that encapsulates qrserver options. + */ + public static class Options { + public Map<String, QrsCache> cacheSettings = new LinkedHashMap<>(); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/DeclaredQueryProfileVariants.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DeclaredQueryProfileVariants.java new file mode 100644 index 00000000000..11903b55d67 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/DeclaredQueryProfileVariants.java @@ -0,0 +1,138 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search; + +import com.yahoo.search.query.profile.OverridableQueryProfile; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileVariants; + +import java.util.*; + +/** + * Represents a set of query profile variants (more or less) as they were declared - + * a helper to produce config, which is also using the "declared" representation + * + * @author bratseth + */ +public class DeclaredQueryProfileVariants { + + private Map<String, VariantQueryProfile> variantQueryProfiles =new LinkedHashMap<>(); + + public DeclaredQueryProfileVariants(QueryProfile profile) { + // Recreates the declared view (settings per set of dimensions) + // from the runtime view (dimension-value pairs per variable) + // yes, this is a little backwards, but the complexity of two representations + // is contained right here... + // TODO: This has become unnecessary as the variants now retains the original structure + for (Map.Entry<String,QueryProfileVariants.FieldValues> fieldValueEntry : profile.getVariants().getFieldValues().entrySet()) { + for (QueryProfileVariants.FieldValue fieldValue : fieldValueEntry.getValue().asList()) { + addVariant(fieldValueEntry.getKey(),fieldValue.getValue(),fieldValue.getDimensionValues().getValues()); + } + } + + for (QueryProfileVariants.FieldValue fieldValue : profile.getVariants().getInherited().asList()) { + for (QueryProfile inherited : (List<QueryProfile>)fieldValue.getValue()) + addVariantInherited(inherited,fieldValue.getDimensionValues().getValues()); + } + + dereferenceCompoundedVariants(profile,""); + } + + private void addVariant(String name,Object value,String[] dimensionValues) { + String dimensionString=toCanonicalString(dimensionValues); + VariantQueryProfile variant=variantQueryProfiles.get(dimensionString); + if (variant==null) { + variant=new VariantQueryProfile(dimensionValues); + variantQueryProfiles.put(dimensionString,variant); + } + variant.getValues().put(name,value); + } + + private void addVariantInherited(QueryProfile inherited,String[] dimensionValues) { + String dimensionString=toCanonicalString(dimensionValues); + VariantQueryProfile variant=variantQueryProfiles.get(dimensionString); + if (variant==null) { + variant=new VariantQueryProfile(dimensionValues); + variantQueryProfiles.put(dimensionString,variant); + } + variant.inherit(inherited); + } + + private void dereferenceCompoundedVariants(QueryProfile profile,String prefix) { + // A variant of a.b is represented as the value a pointing to an anonymous profile a + // having the variants + for (Map.Entry<String,Object> entry : profile.declaredContent().entrySet()) { + if ( ! (entry.getValue() instanceof QueryProfile)) continue; + QueryProfile subProfile=(QueryProfile)entry.getValue(); + // Export if defined implicitly in this, or if this contains overrides + if (!subProfile.isExplicit() || subProfile instanceof OverridableQueryProfile) { + String entryPrefix=prefix + entry.getKey() + "."; + dereferenceCompoundedVariants(subProfile.getVariants(),entryPrefix); + dereferenceCompoundedVariants(subProfile,entryPrefix); + } + } + + if (profile.getVariants()==null) return; + // We need to do the same dereferencing to overridables pointed to by variants of this + for (Map.Entry<String,QueryProfileVariants.FieldValues> fieldValueEntry : profile.getVariants().getFieldValues().entrySet()) { + for (QueryProfileVariants.FieldValue fieldValue : fieldValueEntry.getValue().asList()) { + if ( ! (fieldValue.getValue() instanceof QueryProfile)) continue; + QueryProfile subProfile=(QueryProfile)fieldValue.getValue(); + // Export if defined implicitly in this, or if this contains overrides + if (!subProfile.isExplicit() || subProfile instanceof OverridableQueryProfile) { + String entryPrefix=prefix + fieldValueEntry.getKey() + "."; + dereferenceCompoundedVariants(subProfile.getVariants(),entryPrefix); + dereferenceCompoundedVariants(subProfile,entryPrefix); + } + } + } + } + + private void dereferenceCompoundedVariants(QueryProfileVariants profileVariants,String prefix) { + if (profileVariants==null) return; + for (Map.Entry<String,QueryProfileVariants.FieldValues> fieldVariant : profileVariants.getFieldValues().entrySet()) { + for (QueryProfileVariants.FieldValue variantValue : fieldVariant.getValue().asList()) { + addVariant(prefix + fieldVariant.getKey(),variantValue.getValue(),variantValue.getDimensionValues().getValues()); + } + } + } + + public String toCanonicalString(String[] dimensionValues) { + StringBuilder b=new StringBuilder(); + for (String dimensionValue : dimensionValues) { + if (dimensionValue!=null) + b.append(dimensionValue); + else + b.append("*"); + b.append(","); + } + b.deleteCharAt(b.length()-1); // Remove last, + return b.toString(); + } + + public Map<String, VariantQueryProfile> getVariantQueryProfiles() { return variantQueryProfiles; } + + public class VariantQueryProfile { + + private Map<String,Object> values=new LinkedHashMap<>(); + + private List<QueryProfile> inherited=new ArrayList<>(); + + private String[] dimensionValues; + + public VariantQueryProfile(String[] dimensionValues) { + this.dimensionValues=dimensionValues; + } + + public String[] getDimensionValues() { return dimensionValues; } + + public void inherit(QueryProfile inheritedProfile) { + inherited.add(inheritedProfile); + } + + public List<QueryProfile> inherited() { return inherited; } + + public Map<String,Object> getValues() { return values; } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/PageTemplates.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/PageTemplates.java new file mode 100644 index 00000000000..c38001d4a83 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/PageTemplates.java @@ -0,0 +1,100 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search; + +import com.yahoo.io.reader.NamedReader; +import com.yahoo.search.pagetemplates.config.PageTemplateXMLReader; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.search.pagetemplates.PageTemplatesConfig; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.Reader; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +/** + * Owns the page templates to be handed to the qrs nodes. + * Owned by a container cluster. + * + * @author bratseth + */ +public class PageTemplates implements Serializable, PageTemplatesConfig.Producer { + + private List<String> pages = new ArrayList<>(); + + /** The number of pages in this, for reporting */ + //private int pages=0; + + /** Validates page templates in an application package. The passed readers will be closed. */ + public static void validate(ApplicationPackage applicationPackage) { + List<NamedReader> pageTemplateFiles=null; + try { + pageTemplateFiles=applicationPackage.getPageTemplateFiles(); + new PageTemplateXMLReader().read(pageTemplateFiles,true); // Parse XML for validation only + } + finally { + NamedReader.closeAll(pageTemplateFiles); + } + } + + /** Creates from an application package. The passed readers will be closed. */ + public static PageTemplates create(ApplicationPackage applicationPackage) { + List<NamedReader> pageTemplateFiles=null; + try { + pageTemplateFiles=applicationPackage.getPageTemplateFiles(); + return new PageTemplates(pageTemplateFiles); + } + finally { + NamedReader.closeAll(pageTemplateFiles); + } + } + + // We are representing these as XML rather than a structured config type because the structure + // is not easily representable by config (arbitrary nesting of many types of elements within each other) + // and config<->xml generation will not pull its weight in work and possible bugs. + // The XML content is already validated when we get here. + public PageTemplates(List<NamedReader> readers) { + for (NamedReader pageReader : readers) { + try { + pages.add(contentAsString(pageReader)); + } catch (IOException e) { + throw new IllegalArgumentException("Could not read page template '" + pageReader.getName() + "'",e); + } + } + } + + @Override + public void getConfig(PageTemplatesConfig.Builder builder) { + for (String page : pages) { + builder.page(page); + } + } + + private String contentAsString(Reader pageReader) throws IOException { + BufferedReader bufferedReader=new BufferedReader(pageReader); + StringBuilder b=new StringBuilder(); + String line; + while (null!=(line=bufferedReader.readLine())) { + b.append(line); + b.append("\n"); + } + return b.toString(); + } + + @Override + public String toString() { + return pages.toString(); + } + + /** + * The config produced by this + * + * @return page templates config + */ + public PageTemplatesConfig getConfig() { + PageTemplatesConfig.Builder ptB = new PageTemplatesConfig.Builder(); + getConfig(ptB); + return new PageTemplatesConfig(ptB); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QrsCache.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QrsCache.java new file mode 100644 index 00000000000..72dc2379dbf --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QrsCache.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search; + +/** + * A helper class to wrap a set of QRS cache settings. + * + * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + */ +public class QrsCache { + public final Integer size; + + public QrsCache(Integer size) { + this.size = size; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java new file mode 100644 index 00000000000..cc57a58ed47 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java @@ -0,0 +1,276 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search; + +import com.yahoo.search.query.profile.BackedOverridableQueryProfile; +import com.yahoo.search.query.profile.QueryProfile; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.search.query.profile.SubstituteString; +import com.yahoo.search.query.profile.types.FieldDescription; +import com.yahoo.search.query.profile.types.QueryProfileType; +import com.yahoo.search.query.profile.config.QueryProfilesConfig; + +import java.io.Serializable; +import java.util.*; +import java.util.Map.Entry; + +/** + * Owns the query profiles and query profile types to be handed to the qrs nodes. + * Owned by a container cluster + * + * @author bratseth + */ +public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer { + + private QueryProfileRegistry registry; + + /** + * Creates a new set of query profiles for which the config can be returned at request + * + * @param registry the registry containing the query profiles and types of this. + * The given registry cannot be frozen on calling this. + */ + public QueryProfiles(QueryProfileRegistry registry) { + this.registry = registry; + } + + public QueryProfiles() { + this.registry = new QueryProfileRegistry(); + } + + public QueryProfileRegistry getRegistry() { + return registry; + } + + @Override + public void getConfig(QueryProfilesConfig.Builder builder) { + for (QueryProfile profile : registry.allComponents()) { + builder.queryprofile(createConfig(profile)); + } + for (QueryProfileType profileType : registry.getTypeRegistry().allComponents()) { + if ( ! profileType.isBuiltin()) + builder.queryprofiletype(createConfig(profileType)); + } + } + + private QueryProfilesConfig.Queryprofile.Builder createConfig(QueryProfile profile) { + QueryProfilesConfig.Queryprofile.Builder qB = new QueryProfilesConfig.Queryprofile.Builder(); + qB.id(profile.getId().stringValue()); + if (profile.getType() != null) + qB.type(profile.getType().getId().stringValue()); + for (QueryProfile inherited : profile.inherited()) + qB.inherit(inherited.getId().stringValue()); + + if (profile.getVariants()!=null) { + for (String dimension : profile.getVariants().getDimensions()) + qB.dimensions(dimension); + } + addFieldChildren(qB, profile, ""); + addVariants(qB, profile); + return qB; + } + + private void addFieldChildren(QueryProfilesConfig.Queryprofile.Builder qpB, QueryProfile profile, String namePrefix) { + List<Map.Entry<String,Object>> content=new ArrayList<>(profile.declaredContent().entrySet()); + Collections.sort(content,new MapEntryKeyComparator()); + if (profile.getValue()!=null) { // Add "prefix with dot removed"=value: + QueryProfilesConfig.Queryprofile.Property.Builder propB = new QueryProfilesConfig.Queryprofile.Property.Builder(); + String fullName = namePrefix.substring(0, namePrefix.length() - 1); + Object value = profile.getValue(); + if (value instanceof SubstituteString) + value=value.toString(); // Send only types understood by configBuilder downwards + propB.name(fullName); + if (value!=null) propB.value(value.toString()); + qpB.property(propB); + } + for (Map.Entry<String,Object> field : content) { + addField(qpB, profile, field, namePrefix); + } + } + + private void addVariantFieldChildren(QueryProfilesConfig.Queryprofile.Queryprofilevariant.Builder qpB, + QueryProfile profile, String namePrefix) { + List<Map.Entry<String,Object>> content=new ArrayList<>(profile.declaredContent().entrySet()); + Collections.sort(content,new MapEntryKeyComparator()); + if (profile.getValue()!=null) { // Add "prefix with dot removed"=value: + QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder propB = new QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder(); + String fullName = namePrefix.substring(0, namePrefix.length() - 1); + Object value = profile.getValue(); + if (value instanceof SubstituteString) + value=value.toString(); // Send only types understood by configBuilder downwards + propB.name(fullName); + if (value!=null) propB.value(value.toString()); + qpB.property(propB); + } + for (Map.Entry<String,Object> field : content) { + addVariantField(qpB, field, namePrefix); + } + } + + private void addField(QueryProfilesConfig.Queryprofile.Builder qpB, + QueryProfile profile, Entry<String, Object> field, String namePrefix) { + String fullName=namePrefix + field.getKey(); + if (field.getValue() instanceof QueryProfile) { + QueryProfile subProfile=(QueryProfile)field.getValue(); + if ( ! subProfile.isExplicit()) { // Implicitly defined profile - add content + addFieldChildren(qpB, subProfile,fullName + "."); + } + else { // Reference to an id'ed profile - output reference plus any local overrides + QueryProfilesConfig.Queryprofile.Reference.Builder refB = new QueryProfilesConfig.Queryprofile.Reference.Builder(); + createReferenceFieldConfig(refB, profile, fullName, field.getKey(), ((BackedOverridableQueryProfile) subProfile).getBacking().getId().stringValue()); + qpB.reference(refB); + addFieldChildren(qpB, subProfile,fullName + "."); + } + } + else { // a primitive + qpB.property(createPropertyFieldConfig(profile, fullName, field.getKey(), field.getValue())); + } + } + + private void addVariantField(QueryProfilesConfig.Queryprofile.Queryprofilevariant.Builder qpB, + Entry<String, Object> field, String namePrefix) { + String fullName=namePrefix + field.getKey(); + if (field.getValue() instanceof QueryProfile) { + QueryProfile subProfile=(QueryProfile)field.getValue(); + if ( ! subProfile.isExplicit()) { // Implicitly defined profile - add content + addVariantFieldChildren(qpB, subProfile,fullName + "."); + } + else { // Reference to an id'ed profile - output reference plus any local overrides + QueryProfilesConfig.Queryprofile.Queryprofilevariant.Reference.Builder refB = new QueryProfilesConfig.Queryprofile.Queryprofilevariant.Reference.Builder(); + createVariantReferenceFieldConfig(refB, fullName, ((BackedOverridableQueryProfile) subProfile).getBacking().getId().stringValue()); + qpB.reference(refB); + addVariantFieldChildren(qpB, subProfile,fullName + "."); + } + } + else { // a primitive + qpB.property(createVariantPropertyFieldConfig(fullName, field.getValue())); + } + } + + private void addVariants(QueryProfilesConfig.Queryprofile.Builder qB, QueryProfile profile) { + if (profile.getVariants()==null) return; + DeclaredQueryProfileVariants declaredVariants=new DeclaredQueryProfileVariants(profile); + for (DeclaredQueryProfileVariants.VariantQueryProfile variant : declaredVariants.getVariantQueryProfiles().values()) { + QueryProfilesConfig.Queryprofile.Queryprofilevariant.Builder varB = new QueryProfilesConfig.Queryprofile.Queryprofilevariant.Builder(); + for (String dimensionValue : variant.getDimensionValues()) { + if (dimensionValue==null) + dimensionValue="*"; + varB.fordimensionvalues(dimensionValue); + } + for (QueryProfile inherited : variant.inherited()) + varB.inherit(inherited.getId().stringValue()); + + List<Map.Entry<String,Object>> content=new ArrayList<>(variant.getValues().entrySet()); + Collections.sort(content,new MapEntryKeyComparator()); + for (Map.Entry<String,Object> value : content) { + addVariantField(varB, value,""); + } + qB.queryprofilevariant(varB); + } + } + + private void createReferenceFieldConfig(QueryProfilesConfig.Queryprofile.Reference.Builder refB, QueryProfile profile, + String fullName, String localName, String stringValue) { + refB.name(fullName); + if (stringValue!=null) refB.value(stringValue); + Boolean overridable=null; + if (profile!=null) + overridable=profile.isDeclaredOverridable(localName, null); + if (overridable!=null) + refB.overridable(""+overridable); + } + + private void createVariantReferenceFieldConfig(QueryProfilesConfig.Queryprofile.Queryprofilevariant.Reference.Builder refB, + String fullName, String stringValue) { + refB.name(fullName); + if (stringValue!=null) refB.value(stringValue); + } + + private QueryProfilesConfig.Queryprofile.Property.Builder createPropertyFieldConfig( + QueryProfile profile, String fullName, String localName, Object value) { + QueryProfilesConfig.Queryprofile.Property.Builder propB = new QueryProfilesConfig.Queryprofile.Property.Builder(); + Boolean overridable=null; + if (value instanceof SubstituteString) + value=value.toString(); // Send only types understood by configBuilder downwards + propB.name(fullName); + if (value!=null) propB.value(value.toString()); + if (profile!=null) + overridable=profile.isDeclaredOverridable(localName, null); + if (overridable!=null) + propB.overridable(""+overridable); + return propB; + } + + private QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder createVariantPropertyFieldConfig( + String fullName, Object value) { + QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder propB = new QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder(); + if (value instanceof SubstituteString) + value=value.toString(); // Send only types understood by configBuilder downwards + propB.name(fullName); + if (value!=null) propB.value(value.toString()); + return propB; + } + + private QueryProfilesConfig.Queryprofiletype.Builder createConfig(QueryProfileType profileType) { + QueryProfilesConfig.Queryprofiletype.Builder qtB = new QueryProfilesConfig.Queryprofiletype.Builder(); + qtB.id(profileType.getId().stringValue()); + if (profileType.isDeclaredStrict()) + qtB.strict(true); + if (profileType.getDeclaredMatchAsPath()) + qtB.matchaspath(true); + for (QueryProfileType inherited : profileType.inherited()) + qtB.inherit(inherited.getId().stringValue()); + List<FieldDescription> fields=new ArrayList<>(profileType.declaredFields().values()); + Collections.sort(fields); + for (FieldDescription field : fields) + qtB.field(createConfig(field)); + return qtB; + } + + private QueryProfilesConfig.Queryprofiletype.Field.Builder createConfig(FieldDescription field) { + QueryProfilesConfig.Queryprofiletype.Field.Builder fB = new QueryProfilesConfig.Queryprofiletype.Field.Builder(); + fB. + name(field.getName()). + type(field.getType().stringValue()); + if ( ! field.isOverridable()) + fB.overridable(false); + if (field.isMandatory()) + fB.mandatory(true); + String aliases=toSpaceSeparatedString(field.getAliases()); + if (!aliases.isEmpty()) + fB.alias(aliases); + return fB; + } + + public String toSpaceSeparatedString(List<String> list) { + StringBuilder b=new StringBuilder(); + for (Iterator<String> i=list.iterator(); i.hasNext(); ) { + b.append(i.next()); + if (i.hasNext()) + b.append(" "); + } + return b.toString(); + } + + private static class MapEntryKeyComparator implements Comparator<Map.Entry<String,Object>> { + + public int compare(Map.Entry<String,Object> e1,Map.Entry<String,Object> e2) { + return e1.getKey().compareTo(e2.getKey()); + } + + public boolean equals(Object other) { + return other instanceof MapEntryKeyComparator; + } + + } + + /** + * The config produced by this + * @return query profiles config + */ + public QueryProfilesConfig getConfig() { + QueryProfilesConfig.Builder qB = new QueryProfilesConfig.Builder(); + getConfig(qB); + return new QueryProfilesConfig(qB); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfilesBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfilesBuilder.java new file mode 100644 index 00000000000..26e6ae75999 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfilesBuilder.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search; + +import com.yahoo.io.reader.NamedReader; +import com.yahoo.search.query.profile.config.QueryProfileXMLReader; +import com.yahoo.config.application.api.ApplicationPackage; + +import java.util.List; + +/** + * Reads the query profile and query profile types from an application package. The actual reading + * is delegated to a {@link com.yahoo.search.query.profile.config.QueryProfileXMLReader}. + * + * @author bratseth + */ +// TODO: Move into QueryProfiles +public class QueryProfilesBuilder { + + /** Build the set of query profiles for an application package */ + public QueryProfiles build(ApplicationPackage applicationPackage) { + List<NamedReader> queryProfileTypeFiles=null; + List<NamedReader> queryProfileFiles=null; + try { + queryProfileTypeFiles=applicationPackage.getQueryProfileTypeFiles(); + queryProfileFiles=applicationPackage.getQueryProfileFiles(); + return new QueryProfiles(new QueryProfileXMLReader().read(queryProfileTypeFiles,queryProfileFiles)); + } + finally { + NamedReader.closeAll(queryProfileTypeFiles); + NamedReader.closeAll(queryProfileFiles); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/SemanticRuleBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/SemanticRuleBuilder.java new file mode 100644 index 00000000000..3969054c1d1 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/SemanticRuleBuilder.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.io.IOUtils; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.prelude.semantics.RuleBase; +import com.yahoo.prelude.semantics.RuleImporter; +import com.yahoo.prelude.semantics.parser.ParseException; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Reads the semantic rules from the application package by delegating to RuleConfigDeriver. + * + * @author bratseth + */ +// TODO: Move into SemanticRules +public class SemanticRuleBuilder { + + /** Build the set of semantic rules for an application package */ + public SemanticRules build(ApplicationPackage applicationPackage) { + List<NamedReader> ruleBaseFiles = null; + try { + ruleBaseFiles = applicationPackage.getFiles(ApplicationPackage.RULES_DIR, "sr"); + return new SemanticRules(ruleBaseFiles.stream().map(this::toRuleBaseConfigView).collect(Collectors.toList())); + } + finally { + NamedReader.closeAll(ruleBaseFiles); + } + } + + private SemanticRules.RuleBase toRuleBaseConfigView(NamedReader reader) { + try { + String ruleBaseString = IOUtils.readAll(reader.getReader()); + boolean isDefault = ruleBaseString.contains("@default"); + return new SemanticRules.RuleBase(toName(reader.getName()), isDefault, ruleBaseString); + } + catch (IOException e) { + throw new IllegalArgumentException("Could not load rules bases", e); + } + } + + private String toName(String fileName) { + String shortName = new File(fileName).getName(); + return shortName.substring(0, shortName.length()-".sr".length()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/SemanticRules.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/SemanticRules.java new file mode 100644 index 00000000000..4e24ffd8b19 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/SemanticRules.java @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search; + +import com.yahoo.prelude.semantics.SemanticRulesConfig; +import java.io.Serializable; +import java.util.List; + +/** + * Returns the semantic rules config form a set of rule bases. + * Owned by a container cluster + * + * @author bratseth + */ +public class SemanticRules implements Serializable, SemanticRulesConfig.Producer { + + private List<RuleBase> ruleBases; + + public SemanticRules(List<RuleBase> ruleBases) { + this.ruleBases = ruleBases; + } + + @Override + public void getConfig(SemanticRulesConfig.Builder builder) { + for (RuleBase ruleBase : ruleBases) + builder.rulebase(ruleBase.getConfig()); + } + + /** A config view of a rule base */ + public static class RuleBase { + + private final String name; + private final boolean isDefault; + private final String rules; + + public RuleBase(String name, boolean isDefault, String rules) { + this.name = name; + this.isDefault = isDefault; + this.rules = rules; + } + + private SemanticRulesConfig.Rulebase.Builder getConfig() { + SemanticRulesConfig.Rulebase.Builder ruleBaseBuilder = new SemanticRulesConfig.Rulebase.Builder(); + ruleBaseBuilder.name(name); + ruleBaseBuilder.isdefault(isDefault); + ruleBaseBuilder.rules(rules); + return ruleBaseBuilder; + } + + + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java new file mode 100644 index 00000000000..0dbc7368954 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/FederationSearcher.java @@ -0,0 +1,269 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.collections.CollectionUtil; +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import com.yahoo.search.searchchain.model.federation.FederationSearcherModel; +import com.yahoo.search.federation.FederationConfig; +import com.yahoo.search.searchchain.model.federation.FederationSearcherModel.TargetSpec; +import com.yahoo.vespa.model.container.component.Component; + +import java.util.*; + +/** + * Config producer for the FederationSearcher. + * @author tonytv + */ +public class FederationSearcher extends Searcher<FederationSearcherModel> implements FederationConfig.Producer { + + private final Optional<Component> targetSelector; + + /** + * Generates config for a single search chain contained in a target. + */ + private static final class SearchChainConfig { + private final SearchChain searchChain; + //Zero if not applicable + final ComponentId providerId; + final FederationOptions targetOptions; + final List<String> documentTypes; + + SearchChainConfig(SearchChain searchChain, ComponentId providerId, + FederationOptions targetOptions, List<String> documentTypes) { + this.searchChain = searchChain; + this.providerId = providerId; + this.targetOptions = targetOptions; + this.documentTypes = documentTypes; + } + + public FederationConfig.Target.SearchChain.Builder getSearchChainConfig() { + FederationConfig.Target.SearchChain.Builder sB = new FederationConfig.Target.SearchChain.Builder(); + FederationOptions resolvedOptions = targetOptions.inherit(searchChain.federationOptions()); + sB. + searchChainId(searchChain.getGlobalComponentId().stringValue()). + timeoutMillis(resolvedOptions.getTimeoutInMilliseconds()). + requestTimeoutMillis(resolvedOptions.getRequestTimeoutInMilliseconds()). + optional(resolvedOptions.getOptional()). + useByDefault(resolvedOptions.getUseByDefault()). + documentTypes(documentTypes); + if (providerId != null) + sB.providerId(providerId.stringValue()); + return sB; + } + } + + /** + * One or more search chains that are handled as a single group, + * which can be federated to as a single entity. + */ + private static abstract class Target { + final ComponentId id; + final FederationOptions targetOptions; + + public Target(ComponentId id, FederationOptions targetOptions) { + this.id = id; + this.targetOptions = targetOptions; + } + + public FederationConfig.Target.Builder getTargetConfig() { + FederationConfig.Target.Builder tb = new FederationConfig.Target.Builder(); + tb. + id(id.stringValue()). + useByDefault(targetOptions.getUseByDefault()); + getSearchChainsConfig(tb); + return tb; + } + + protected abstract void getSearchChainsConfig(FederationConfig.Target.Builder tb); + } + + private static class SearchChainTarget extends Target { + private final SearchChainConfig searchChainConfig; + + public SearchChainTarget(SearchChain searchChain, + FederationOptions targetOptions) { + super(searchChain.getComponentId(), targetOptions); + searchChainConfig = new SearchChainConfig( + searchChain, + null, + targetOptions, + searchChain.getDocumentTypes()); + } + + @Override + protected void getSearchChainsConfig(FederationConfig.Target.Builder tB) { + tB.searchChain(searchChainConfig.getSearchChainConfig()); + } + } + + private static class SourceGroupTarget extends Target { + private final SearchChainConfig leaderConfig; + private final List<SearchChainConfig> participantsConfig = + new ArrayList<>(); + + public SourceGroupTarget(SourceGroup group, + FederationOptions targetOptions) { + super(group.getComponentId(), applyDefaultSourceGroupOptions(targetOptions)); + + leaderConfig = createConfig(group.leader(), targetOptions); + for (Source participant : group.participants()) { + participantsConfig.add( + createConfig(participant, targetOptions)); + } + } + + private static FederationOptions applyDefaultSourceGroupOptions(FederationOptions targetOptions) { + FederationOptions defaultSourceGroupOption = new FederationOptions().setUseByDefault(true); + return targetOptions.inherit(defaultSourceGroupOption); + } + + private SearchChainConfig createConfig(Source source, + FederationOptions targetOptions) { + return new SearchChainConfig( + source, + source.getParentProvider().getComponentId(), + targetOptions, + source.getDocumentTypes()); + } + + @Override + protected void getSearchChainsConfig(FederationConfig.Target.Builder tB) { + tB.searchChain(leaderConfig.getSearchChainConfig()); + for (SearchChainConfig participant : participantsConfig) { + tB.searchChain(participant.getSearchChainConfig()); + } + } + } + + private static class TargetResolver { + final ComponentRegistry<SearchChain> searchChainRegistry; + final SourceGroupRegistry sourceGroupRegistry; + + /** + * @return true if searchChain.id newer than sourceGroup.id + */ + private boolean newerVersion(SearchChain searchChain, + SourceGroup sourceGroup) { + if (searchChain == null || sourceGroup == null) { + return false; + } else { + return newerVersion(searchChain.getComponentId(), sourceGroup.getComponentId()); + } + } + + /** + * @return true if a newer than b + */ + private boolean newerVersion(ComponentId a, ComponentId b) { + return a.compareTo(b) > 0; + } + + + TargetResolver(ComponentRegistry<SearchChain> searchChainRegistry, + SourceGroupRegistry sourceGroupRegistry) { + this.searchChainRegistry = searchChainRegistry; + this.sourceGroupRegistry = sourceGroupRegistry; + } + + Target resolve(FederationSearcherModel.TargetSpec specification) { + SearchChain searchChain = searchChainRegistry.getComponent( + specification.sourceSpec); + SourceGroup sourceGroup = sourceGroupRegistry.getComponent( + specification.sourceSpec); + + if (searchChain == null && sourceGroup == null) { + return null; + } else if (sourceGroup == null || + newerVersion(searchChain, sourceGroup)) { + return new SearchChainTarget(searchChain, specification.federationOptions); + } else { + return new SourceGroupTarget(sourceGroup, specification.federationOptions); + } + } + } + + private final Map<ComponentId, Target> resolvedTargets = + new LinkedHashMap<>(); + + public FederationSearcher(FederationSearcherModel searcherModel, Optional<Component> targetSelector) { + super(searcherModel); + this.targetSelector = targetSelector; + + if (targetSelector.isPresent()) + addChild(targetSelector.get()); + } + + @Override + public void getConfig(FederationConfig.Builder builder) { + for (Target target : resolvedTargets.values()) { + builder.target(target.getTargetConfig()); + } + + if (targetSelector.isPresent()) { + builder.targetSelector(targetSelector.get().getGlobalComponentId().stringValue()); + } + } + + @Override + public void initialize() { + initialize(getSearchChains().allChains(), getSearchChains().allSourceGroups()); + } + + void initialize(ComponentRegistry<SearchChain> searchChainRegistry, + SourceGroupRegistry sourceGroupRegistry) { + TargetResolver targetResolver = new TargetResolver( + searchChainRegistry, sourceGroupRegistry); + + addSourceTargets(targetResolver, model.targets); + + if (model.inheritDefaultSources) + addDefaultTargets(targetResolver, searchChainRegistry); + } + + private void addSourceTargets(TargetResolver targetResolver, List<TargetSpec> targets) { + for (TargetSpec targetSpec : targets) { + + Target target = targetResolver.resolve(targetSpec); + if (target == null) { + throw new RuntimeException("Can't find source " + + targetSpec.sourceSpec + + " used as a source for federation '" + + getComponentId() + "'"); + } + + Target duplicate = resolvedTargets.put(target.id, target); + if (duplicate != null && !duplicate.targetOptions.equals(target.targetOptions)) { + throw new RuntimeException("Search chain " + target.id + " added twice with different federation options" + + " to the federation searcher " + getComponentId()); + } + } + } + + + private void addDefaultTargets(TargetResolver targetResolver, ComponentRegistry<SearchChain> searchChainRegistry) { + for (GenericTarget genericTarget : defaultTargets(searchChainRegistry.allComponents())) { + ComponentSpecification specification = genericTarget.getComponentId().toSpecification(); + + //Can't use genericTarget directly, as it might be part of a source group. + Target federationTarget = targetResolver.resolve(new TargetSpec(specification, new FederationOptions())); + //Do not replace manually added sources, as they might have manually configured federation options + if (!resolvedTargets.containsKey(federationTarget.id)) + resolvedTargets.put(federationTarget.id, federationTarget); + } + } + + + private static List<GenericTarget> defaultTargets(Collection<SearchChain> allSearchChains) { + Collection<Provider> providers = + CollectionUtil.filter(allSearchChains, Provider.class); + + List<GenericTarget> defaultTargets = new ArrayList<>(); + for (Provider provider : providers) { + defaultTargets.addAll(provider.defaultFederationTargets()); + } + return defaultTargets; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/GenericTarget.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/GenericTarget.java new file mode 100644 index 00000000000..9ed62f15244 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/GenericTarget.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.search.searchchain.model.federation.FederationOptions; + +/** + * A search chain that is intended to be used for federation (i.e. providers, sources) + * @author tonytv + */ +abstract public class GenericTarget extends SearchChain { + + private final FederationOptions federationOptions; + + public GenericTarget(ChainSpecification specWithoutInnerSearchers, FederationOptions federationOptions) { + super(specWithoutInnerSearchers); + this.federationOptions = federationOptions; + } + + @Override + public FederationOptions federationOptions() { + FederationOptions defaultOptions = new FederationOptions().setUseByDefault(useByDefault()); + return federationOptions.inherit(defaultOptions); + } + + /** The value for useByDefault in case the user have not specified any **/ + abstract protected boolean useByDefault(); + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProvider.java new file mode 100644 index 00000000000..ef6c6ef4df6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProvider.java @@ -0,0 +1,115 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.binaryprefix.BinaryPrefix; +import com.yahoo.binaryprefix.BinaryScaledAmount; +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.search.cache.QrBinaryCacheConfig; +import com.yahoo.search.cache.QrBinaryCacheRegionConfig; +import com.yahoo.search.federation.ProviderConfig; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import com.yahoo.search.searchchain.model.federation.HttpProviderSpec; + +import java.util.ArrayList; +import java.util.List; + +import static com.yahoo.search.federation.ProviderConfig.Node; +import static com.yahoo.search.federation.ProviderConfig.Yca; + + +/** + * A provider containing a http searcher. + * @author tonytv + */ +public class HttpProvider extends Provider implements ProviderConfig.Producer, + QrBinaryCacheConfig.Producer, + QrBinaryCacheRegionConfig.Producer { + + private final HttpProviderSpec providerSpec; + + //TODO: For backward compatibility only, eliminate this later + private BinaryScaledAmount cacheSize; + + public double getCacheWeight() { + return providerSpec.cacheWeight; + } + + /** + * TODO: remove, for backward compatibility only. + */ + public void setCacheSize(BinaryScaledAmount cacheSize) { + this.cacheSize = cacheSize; + } + + /* + * Config producer for the contained http searcher.. + */ + + public HttpProvider(ChainSpecification specWithoutInnerSearchers, FederationOptions federationOptions, HttpProviderSpec providerSpec) { + super(specWithoutInnerSearchers, federationOptions); + this.providerSpec = providerSpec; + } + + @Override + public void getConfig(ProviderConfig.Builder builder) { + if (providerSpec.path != null) + builder.path(providerSpec.path); + if (providerSpec.connectionParameters.readTimeout != null) + builder.readTimeout(providerSpec.connectionParameters.readTimeout ); + if (providerSpec.connectionParameters.connectionTimeout != null) + builder.connectionTimeout(providerSpec.connectionParameters.connectionTimeout); + if (providerSpec.connectionParameters.connectionPoolTimeout != null) + builder.connectionPoolTimeout(providerSpec.connectionParameters.connectionPoolTimeout); + if (providerSpec.connectionParameters.retries != null) + builder.retries(providerSpec.connectionParameters.retries); + + builder.node(getNodes(providerSpec.nodes)); + + if (providerSpec.ycaApplicationId != null) { + builder.yca(getYca(providerSpec)); + } + } + + private static Yca.Builder getYca(HttpProviderSpec providerSpec) { + Yca.Builder yca = new Yca.Builder() + .applicationId(providerSpec.ycaApplicationId); + + if (providerSpec.ycaProxy != null) { + yca.useProxy(true); + if (providerSpec.ycaProxy.host != null) { + yca.host(providerSpec.ycaProxy.host) + .port(providerSpec.ycaProxy.port); + } + } + if (providerSpec.ycaCertificateTtl != null) yca.ttl(providerSpec.ycaCertificateTtl); + if (providerSpec.ycaRetryWait != null) yca.ttl(providerSpec.ycaRetryWait); + return yca; + } + + private static List<Node.Builder> getNodes(List<HttpProviderSpec.Node> nodeSpecs) { + ArrayList<Node.Builder> nodes = new ArrayList<>(); + for (HttpProviderSpec.Node node : nodeSpecs) { + nodes.add( + new Node.Builder() + .host(node.host) + .port(node.port)); + } + return nodes; + } + + public int cacheSizeMB() { + return providerSpec.cacheSizeMB != null ? + providerSpec.cacheSizeMB : + (int) cacheSize.as(BinaryPrefix.mega); + } + + @Override + public void getConfig(QrBinaryCacheConfig.Builder builder) { + builder.cache_size(cacheSizeMB()); + } + + @Override + public void getConfig(QrBinaryCacheRegionConfig.Builder builder) { + builder.region_size(cacheSizeMB()); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProviderSearcher.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProviderSearcher.java new file mode 100644 index 00000000000..44f9879230d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/HttpProviderSearcher.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.binaryprefix.BinaryPrefix; +import com.yahoo.binaryprefix.BinaryScaledAmount; +import com.yahoo.component.chain.model.ChainedComponentModel; +import com.yahoo.search.searchchain.model.federation.HttpProviderSpec; +import java.util.List; + +/** + +* @author tonytv +*/ +public class HttpProviderSearcher extends Searcher<ChainedComponentModel> { + + + public HttpProviderSearcher(ChainedComponentModel model) { + super(model); + } + + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java new file mode 100644 index 00000000000..1dd4fb478ec --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java @@ -0,0 +1,188 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.component.chain.model.ChainedComponentModel; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.prelude.cluster.QrMonitorConfig; +import com.yahoo.search.config.dispatchprototype.SearchNodesConfig; +import com.yahoo.vespa.config.search.DispatchConfig; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.search.config.ClusterConfig; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import com.yahoo.search.searchchain.model.federation.LocalProviderSpec; +import com.yahoo.vespa.model.search.AbstractSearchCluster; +import com.yahoo.vespa.model.search.IndexedSearchCluster; +import com.yahoo.vespa.model.search.SearchNode; + +import java.util.*; + +/** + * Config producer for search chain responsible for sending queries to a local cluster. + * + * @author tonytv + */ +public class LocalProvider extends Provider implements + DocumentdbInfoConfig.Producer, + ClusterConfig.Producer, + AttributesConfig.Producer, + QrMonitorConfig.Producer, + RankProfilesConfig.Producer, + SearchNodesConfig.Producer, + DispatchConfig.Producer { + + private final LocalProviderSpec providerSpec; + private volatile AbstractSearchCluster searchCluster; + + + @Override + public void getConfig(ClusterConfig.Builder builder) { + assert (searchCluster != null) : "Null search cluster!"; + builder.clusterId(searchCluster.getClusterIndex()); + builder.clusterName(searchCluster.getClusterName()); + + if (providerSpec.cacheSize != null) + builder.cacheSize(providerSpec.cacheSize); + + if (searchCluster.getVisibilityDelay() != null) + builder.cacheTimeout(convertVisibilityDelay(searchCluster.getVisibilityDelay())); + } + + @Override + public void getConfig(RankProfilesConfig.Builder builder) { + searchCluster.getConfig(builder); + } + + @Override + public void getConfig(AttributesConfig.Builder builder) { + searchCluster.getConfig(builder); + } + + @Override + public void getConfig(QrMonitorConfig.Builder builder) { + int requestTimeout = federationOptions().getTimeoutInMilliseconds(); + if (requestTimeout != -1) { + builder.requesttimeout(requestTimeout); + } + } + + @Override + public void getConfig(final SearchNodesConfig.Builder builder) { + if (!(searchCluster instanceof IndexedSearchCluster)) { + log.warning("Could not build SearchNodesConfig: Only supported for IndexedSearchCluster, got " + + searchCluster.getClass().getCanonicalName()); + return; + } + final IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster) searchCluster; + for (final SearchNode searchNode : indexedSearchCluster.getSearchNodes()) { + builder.search_node( + new SearchNodesConfig.Search_node.Builder() + .host(searchNode.getHostName()) + .port(searchNode.getDispatchPort())); + } + } + + private void addProviderSearchers(LocalProviderSpec providerSpec) { + for (ChainedComponentModel searcherModel : providerSpec.searcherModels) { + addInnerComponent(new Searcher<>(searcherModel)); + } + } + + @Override + public ChainSpecification getChainSpecification() { + ChainSpecification spec = + super.getChainSpecification(); + return new ChainSpecification(spec.componentId, spec.inheritance, spec.phases(), + disableStemmingIfStreaming(spec.componentReferences)); + } + + //TODO: ugly, restructure this + private Set<ComponentSpecification> disableStemmingIfStreaming(Set<ComponentSpecification> searcherReferences) { + if (!searchCluster.isStreaming()) { + return searcherReferences; + } else { + Set<ComponentSpecification> filteredSearcherReferences = new LinkedHashSet<>(searcherReferences); + filteredSearcherReferences.remove( + toGlobalComponentId( + new ComponentId("com.yahoo.prelude.querytransform.StemmingSearcher")). + toSpecification()); + return filteredSearcherReferences; + } + } + + private ComponentId toGlobalComponentId(ComponentId searcherId) { + return searcherId.nestInNamespace(getComponentId()); + } + + public String getClusterName() { + return providerSpec.clusterName; + } + + public void setSearchCluster(AbstractSearchCluster searchCluster) { + assert (this.searchCluster == null); + this.searchCluster = searchCluster; + } + + public LocalProvider(ChainSpecification specWithoutInnerSearchers, + FederationOptions federationOptions, + LocalProviderSpec providerSpec) { + super(specWithoutInnerSearchers, federationOptions); + addProviderSearchers(providerSpec); + this.providerSpec = providerSpec; + } + + @Override + public List<String> getDocumentTypes() { + List<String> documentTypes = new ArrayList<>(); + + for (AbstractSearchCluster.SearchDefinitionSpec spec : searchCluster.getLocalSDS()) { + documentTypes.add(spec.getSearchDefinition().getSearch().getDocument().getName()); + } + + return documentTypes; + } + + @Override + public FederationOptions federationOptions() { + Double queryTimeoutInSeconds = searchCluster.getQueryTimeout(); + + return queryTimeoutInSeconds == null ? + super.federationOptions() : + super.federationOptions().inherit( + new FederationOptions().setTimeoutInMilliseconds((int) (queryTimeoutInSeconds * 1000))); + } + + @Override + public void getConfig(DocumentdbInfoConfig.Builder builder) { + searchCluster.getConfig(builder); + } + + /** + * For backward compatibility only, do not use. + */ + public void setCacheSize(Integer cacheSize) { + providerSpec.cacheSize = cacheSize; + } + + // The semantics of visibility delay in search is deactivating caches if the + // delay is less than 1.0, in qrs the cache is deactivated if the delay is 0 + // (or less). 1.0 seems a little arbitrary, so just doing the conversion + // here instead of having two totally independent implementations having to + // follow each other down in the modules. + private static Double convertVisibilityDelay(Double visibilityDelay) { + return (visibilityDelay < 1.0d) ? 0.0d : visibilityDelay; + } + + @Override + public void getConfig(DispatchConfig.Builder builder) { + if (!(searchCluster instanceof IndexedSearchCluster)) { + log.warning("Could not build DispatchConfig: Only supported for IndexedSearchCluster, got " + + searchCluster.getClass().getCanonicalName()); + return; + } + ((IndexedSearchCluster) searchCluster).getConfig(builder); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Provider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Provider.java new file mode 100644 index 00000000000..8dba2e8b589 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Provider.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import com.yahoo.vespa.model.container.component.ConfigProducerGroup; + +import java.util.Arrays; +import java.util.Collection; + +/** + * Base config producer for search chains that communicate with backends. + * + * @author tonytv + */ +public class Provider extends GenericTarget { + + private ConfigProducerGroup<Source> sources; + + public Provider(ChainSpecification specWithoutInnerSearchers, FederationOptions federationOptions) { + super(specWithoutInnerSearchers, federationOptions); + sources = new ConfigProducerGroup<>(this, "source"); + } + + public void addSource(Source source) { + sources.addComponent(source.getComponentId(), source); + } + + public Collection<Source> getSources() { + return sources.getComponents(); + } + + @Override + protected boolean useByDefault() { + return sources.getComponents().isEmpty(); + } + + public Collection<? extends GenericTarget> defaultFederationTargets() { + if (sources.getComponents().isEmpty()) { + return Arrays.asList(this); + } else { + return sources.getComponents(); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChain.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChain.java new file mode 100644 index 00000000000..148bab3f84a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChain.java @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import com.yahoo.vespa.model.container.component.chain.Chain; + +import java.util.Collections; +import java.util.List; + +/** + * Represents a search chain in the vespa model. + * + * @author tonytv + */ +public class SearchChain extends Chain<Searcher<?>> { + + public SearchChain(ChainSpecification specWithoutInnerSearchers) { + super(specWithoutInnerSearchers); + } + + public FederationOptions federationOptions() { + return new FederationOptions().setUseByDefault(true); + } + + //A list of documents types that this search chain provides results for, empty if unknown + public List<String> getDocumentTypes() { + return Collections.emptyList(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java new file mode 100644 index 00000000000..d13e13b232f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SearchChains.java @@ -0,0 +1,129 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.binaryprefix.BinaryScaledAmount; +import com.yahoo.collections.CollectionUtil; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.container.component.chain.Chains; +import com.yahoo.vespa.model.search.AbstractSearchCluster; +import com.yahoo.vespa.model.container.search.searchchain.defaultsearchchains.LocalClustersCreator; +import com.yahoo.vespa.model.container.search.searchchain.defaultsearchchains.VespaSearchChainsCreator; + +import java.util.Collection; +import java.util.Map; + +/** + * Root config producer of the whole search chains model (contains searchchains and searchers). + * + * @author tonytv + */ +public class SearchChains extends Chains<SearchChain> { + + private final SourceGroupRegistry sourceGroups = new SourceGroupRegistry(); + + public SearchChains(AbstractConfigProducer parent, String subId) { + super(parent, subId); + } + + public void initialize(Map<String, ? extends AbstractSearchCluster> searchClustersByName, BinaryScaledAmount totalProviderCacheSize) { + LocalClustersCreator.addDefaultLocalProviders(this, searchClustersByName.keySet()); + VespaSearchChainsCreator.addVespaSearchChains(this); + + validateSourceGroups(); //must be done before initializing searchers since they are used by FederationSearchers. + initializeComponents(searchClustersByName, totalProviderCacheSize); + } + + private void initializeComponents(Map<String, ? extends AbstractSearchCluster> searchClustersByName, + BinaryScaledAmount totalProviderCacheSize) { + setSearchClusterForLocalProvider(searchClustersByName); + setCacheSizeForHttpProviders(totalProviderCacheSize); + initializeComponents(); + } + + private void setCacheSizeForHttpProviders(BinaryScaledAmount totalProviderCacheSize) { + double totalCacheWeight = 0; + for (HttpProvider provider : httpProviders()) { + totalCacheWeight += provider.getCacheWeight(); + } + + final BinaryScaledAmount cacheUnit = totalProviderCacheSize.divide(totalCacheWeight); + for (HttpProvider provider : httpProviders()) { + provider.setCacheSize(cacheUnit.multiply(provider.getCacheWeight())); + } + } + + private void setSearchClusterForLocalProvider(Map<String, ? extends AbstractSearchCluster> clusterIndexByName) { + for (LocalProvider provider : localProviders()) { + AbstractSearchCluster cluster = clusterIndexByName.get(provider.getClusterName()); + if (cluster == null) { + throw new RuntimeException("No searchable content cluster with id '" + provider.getClusterName() + "'"); + } + provider.setSearchCluster(cluster); + } + } + + private void validateSourceGroups() { + for (SourceGroup sourceGroup : sourceGroups.groups()) { + sourceGroup.validate(); + + if (getChainGroup().getComponentMap().containsKey(sourceGroup.getComponentId())) { + throw new RuntimeException( + String.format("Same id used for a source and another search chain/provider: '%s'", + sourceGroup.getComponentId())); + } + } + } + + @Override + public void validate() throws Exception { + validateSourceGroups(); + super.validate(); + } + + public SourceGroupRegistry allSourceGroups() { + return sourceGroups; + } + + public Collection<LocalProvider> localProviders() { + return CollectionUtil.filter(allChains().allComponents(), LocalProvider.class); + } + + + public Collection<HttpProvider> httpProviders() { + return CollectionUtil.filter(allChains().allComponents(), HttpProvider.class); + } + + /* + * If searchChain is a provider, its sources must already have been attached. + */ + @Override + public void add(SearchChain searchChain) { + assert !(searchChain instanceof Source); + + super.add(searchChain); + + if (searchChain instanceof Provider) { + sourceGroups.addSources((Provider)searchChain); + } + } + + @Override + public ComponentRegistry<SearchChain> allChains() { + ComponentRegistry<SearchChain> allChains = new ComponentRegistry<>(); + for (SearchChain chain : getChainGroup().getComponents()) { + allChains.register(chain.getId(), chain); + if (chain instanceof Provider) + addSources(allChains, (Provider)chain); + } + allChains.freeze(); + return allChains; + } + + private void addSources(ComponentRegistry<SearchChain> chains, Provider provider) { + for (Source source : provider.getSources()) { + chains.register(source.getId(), source); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Searcher.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Searcher.java new file mode 100644 index 00000000000..653be591be3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Searcher.java @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.component.chain.model.ChainedComponentModel; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.container.component.chain.ChainedComponent; + +/** + * @author gjoranv + * @author tonytv + */ +public class Searcher<T extends ChainedComponentModel> extends ChainedComponent<T> { + + public Searcher(T model) { + super(model); + } + + protected SearchChains getSearchChains() { + AbstractConfigProducer ancestor = getParent(); + while (!(ancestor instanceof SearchChains)) { + ancestor = ancestor.getParent(); + } + return (SearchChains)ancestor; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Source.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Source.java new file mode 100644 index 00000000000..fe839b904d2 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/Source.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import com.yahoo.config.model.producer.AbstractConfigProducer; + +import java.util.Arrays; + + +/** + * Config producer for source, which is contained in a provider. + * + * @author tonytv + */ +public class Source extends GenericTarget { + + //Each source group must have exactly one leader, and an arbitrary number of participants + public enum GroupOption { + leader, + participant + } + + public final GroupOption groupOption; + + public Source(ChainSpecification specWithoutInnerSearchers, FederationOptions federationOptions, + GroupOption groupOption) { + super(specWithoutInnerSearchers, federationOptions); + this.groupOption = groupOption; + } + + @Override + public FederationOptions federationOptions() { + return super.federationOptions().inherit(getParentProvider().federationOptions()); + } + + @Override + protected boolean useByDefault() { + return false; + } + + public Provider getParentProvider() { + AbstractConfigProducer parent = getParent(); + while (!(parent instanceof Provider)) { + parent = parent.getParent(); + } + return (Provider)parent; + } + + @Override + public ChainSpecification getChainSpecification() { + return super.getChainSpecification().addInherits( + Arrays.asList(getParentProvider().getComponentId().toSpecification())); + } + + public ComponentId getGlobalComponentId() { + return getComponentId().nestInNamespace( + getParentProvider().getComponentId()); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroup.java new file mode 100644 index 00000000000..799bba45b04 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroup.java @@ -0,0 +1,95 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.component.ComponentId; + +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * A set of sources with the same name, + * each associated with a different provider, + * that fills the same role. + * @author tonytv + */ +final class SourceGroup { + private final ComponentId id; + private Source leader; + private final Set<Source> participants = + new LinkedHashSet<>(); + + private void setLeader(Source leader) { + assert (validMember(leader)); + + if (this.leader != null) { + throw new IllegalStateException( + "There can not be two default providers for the source " + + id); + } + + this.leader = leader; + } + + private void addParticipant(Source source) { + assert (validMember(source)); + assert (!source.equals(leader)); + + if (!participants.add(source)) { + throw new RuntimeException("Source added twice to the same group " + + source); + } + } + + private boolean validMember(Source leader) { + return leader.getComponentId().equals(id); + } + + public ComponentId getComponentId() { + return id; + } + + public SourceGroup(ComponentId id) { + this.id = id; + } + + public void add(Source source) { + assert source.getComponentId().equals(getComponentId()): + "Ids differ: " + source.getComponentId() + " -- " + getComponentId(); + + if (Source.GroupOption.leader == source.groupOption) { + setLeader(source); + } else { + addParticipant(source); + } + } + + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("Source id: ").append(id).append("\n"). + append("Leader provider: ").append( + leader.getParentProvider().getComponentId()).append("\n"). + append("Participants:"); + + for (Source participant : participants) { + builder.append("\n").append(" Provider: ").append( + participant.getParentProvider().getComponentId()); + } + return builder.toString(); + } + + public Source leader() { + return leader; + } + + public Collection<Source> participants() { + return Collections.unmodifiableCollection(participants); + } + + public void validate() { + if (leader == null) + throw new IllegalStateException("Missing leader for the source " + getComponentId() + + ". One of the sources must use the attribute id instead of idref."); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupRegistry.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupRegistry.java new file mode 100644 index 00000000000..fda962402d1 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/SourceGroupRegistry.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.model.ComponentAdaptor; +import com.yahoo.component.provider.ComponentRegistry; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + + +/** + * Owns all the source groups in the search chains model. + * @author tonytv + */ +class SourceGroupRegistry { + private final ComponentRegistry<ComponentAdaptor<SourceGroup>> sourceGroups + = new ComponentRegistry<>(); + + private void add(Source source) { + getGroup(source.getComponentId()).add(source); + } + + private SourceGroup getGroup(ComponentId sourceId) { + ComponentAdaptor<SourceGroup> group = + sourceGroups.getComponent(sourceId); + if (group == null) { + group = new ComponentAdaptor<>(sourceId, + new SourceGroup(sourceId)); + sourceGroups.register(group.getId(), group); + } + return group.model; + } + + void addSources(Provider provider) { + for (Source source : provider.getSources()) { + add(source); + } + } + + public Collection<SourceGroup> groups() { + List<SourceGroup> result = new ArrayList<>(); + for (ComponentAdaptor<SourceGroup> group : + sourceGroups.allComponents()) { + result.add(group.model); + } + return result; + } + + public SourceGroup getComponent(ComponentSpecification spec) { + ComponentAdaptor<SourceGroup> result = sourceGroups.getComponent(spec); + return (result != null)? + result.model : + null; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/LocalClustersCreator.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/LocalClustersCreator.java new file mode 100644 index 00000000000..43f4a28ff30 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/LocalClustersCreator.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain.defaultsearchchains; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.Phase; +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.search.searchchain.model.federation.FederationOptions; +import com.yahoo.search.searchchain.model.federation.LocalProviderSpec; +import com.yahoo.vespa.model.container.search.searchchain.LocalProvider; +import com.yahoo.vespa.model.container.search.searchchain.SearchChains; + +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * Adds default search chains for all local clusters not mentioned explicitly + * @author tonytv + */ +public class LocalClustersCreator { + static ChainSpecification emptySearchChainSpecification(String componentName) { + return new ChainSpecification( + new ComponentId(componentName), + VespaSearchChainsCreator.inheritsVespaPhases(), //TODO: refactor + Collections.<Phase>emptyList(), + Collections.<ComponentSpecification>emptySet()); + } + + static LocalProvider createDefaultLocalProvider(String clusterName) { + return new LocalProvider(emptySearchChainSpecification(clusterName), new FederationOptions(), + new LocalProviderSpec(clusterName, null)); + } + + static Set<String> presentClusters(SearchChains searchChains) { + Set<String> presentClusters = new LinkedHashSet<>(); + for (LocalProvider provider : searchChains.localProviders()) { + presentClusters.add(provider.getClusterName()); + } + return presentClusters; + } + + public static void addDefaultLocalProviders(SearchChains searchChains, Set<String> clusterNames) { + Set<String> missingClusters = new LinkedHashSet<>(clusterNames); + missingClusters.removeAll(presentClusters(searchChains)); + + for (String clusterName : missingClusters) { + searchChains.add(createDefaultLocalProvider(clusterName)); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/VespaSearchChainsCreator.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/VespaSearchChainsCreator.java new file mode 100644 index 00000000000..ed5fd3b759e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/defaultsearchchains/VespaSearchChainsCreator.java @@ -0,0 +1,141 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search.searchchain.defaultsearchchains; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.Phase; +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.component.chain.model.ChainedComponentModel; +import com.yahoo.search.searchchain.PhaseNames; +import com.yahoo.search.searchchain.model.VespaSearchers; +import com.yahoo.search.searchchain.model.federation.FederationSearcherModel; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.search.searchchain.*; + +import java.util.*; + +/** + * Creates the search chains vespaPhases, vespa and native. + * + * <p>TODO: refactor</p> + * @author tonytv + */ +public class VespaSearchChainsCreator { + private static class PhasesCreator { + private static Set<String> set(String successor) { + return successor == null ? null : new LinkedHashSet<>(Arrays.asList(successor)); + } + + private static String lastElement(String[] phases) { + return phases[phases.length - 1]; + } + + private static Phase createPhase(String phase, String before) { + return new Phase(phase, set(before), null); + } + + public static Collection<Phase> linearPhases(String... phases) { + List<Phase> result = new ArrayList<>(); + + for (int i=0; i < phases.length - 1; ++i) { + result.add( + createPhase(phases[i], phases[i+1])); + } + + if (phases.length > 0) { + result.add( + createPhase(lastElement(phases), null)); + } + + return result; + } + } + + private static Set<ComponentSpecification> noSearcherReferences() { + return Collections.emptySet(); + } + + private static Collection<Phase> noPhases() { + return Collections.emptySet(); + } + + private static ChainSpecification.Inheritance inherits(ComponentId chainId) { + Set<ComponentSpecification> inheritsSet = new LinkedHashSet<>(); + inheritsSet.add(chainId.toSpecification()); + return new ChainSpecification.Inheritance(inheritsSet, null); + } + + static ChainSpecification.Inheritance inheritsVespaPhases() { + return inherits(vespaPhasesSpecification().componentId); + } + + private static void addInnerSearchers(SearchChain searchChain, Collection<ChainedComponentModel> searcherModels) { + for (ChainedComponentModel searcherModel : searcherModels) { + searchChain.addInnerComponent(createSearcher(searcherModel)); + } + } + + private static Searcher<? extends ChainedComponentModel> createSearcher(ChainedComponentModel searcherModel) { + if (searcherModel instanceof FederationSearcherModel) { + return new FederationSearcher((FederationSearcherModel) searcherModel, Optional.<Component>empty()); + } else { + return new Searcher<>(searcherModel); + } + } + + + private static ChainSpecification nativeSearchChainSpecification() { + return new ChainSpecification( + new ComponentId("native"), + inheritsVespaPhases(), + noPhases(), + noSearcherReferences()); + } + + private static ChainSpecification vespaSearchChainSpecification() { + return new ChainSpecification( + new ComponentId("vespa"), + inherits(nativeSearchChainSpecification().componentId), + noPhases(), + noSearcherReferences()); + } + + + private static ChainSpecification vespaPhasesSpecification() { + return new ChainSpecification( + new ComponentId("vespaPhases"), + new ChainSpecification.Inheritance(null, null), + PhasesCreator.linearPhases( + PhaseNames.RAW_QUERY, + PhaseNames.TRANSFORMED_QUERY, + PhaseNames.BLENDED_RESULT, + PhaseNames.UNBLENDED_RESULT, + PhaseNames.BACKEND), + noSearcherReferences()); + } + + private static SearchChain createVespaPhases() { + return new SearchChain(vespaPhasesSpecification()); + } + + private static SearchChain createNative() { + SearchChain nativeChain = new SearchChain(nativeSearchChainSpecification()); + addInnerSearchers(nativeChain, VespaSearchers.nativeSearcherModels); + return nativeChain; + } + + private static SearchChain createVespa() { + SearchChain vespaChain = new SearchChain(vespaSearchChainSpecification()); + addInnerSearchers(vespaChain, VespaSearchers.vespaSearcherModels); + return vespaChain; + } + + public static void addVespaSearchChains(SearchChains searchChains) { + searchChains.add( + createVespaPhases()); + searchChains.add( + createNative()); + searchChains.add( + createVespa()); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java new file mode 100644 index 00000000000..f6352e9ac35 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java @@ -0,0 +1,96 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.container.core.AccessLogConfig; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.AccessLogComponent; +import com.yahoo.vespa.model.container.component.AccessLogComponent.AccessLogType; +import org.w3c.dom.Element; + +import java.util.Optional; + +import static com.yahoo.collections.CollectionUtil.firstMatching; +import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalAttribute; + +import static com.yahoo.config.model.builder.xml.XmlHelper.nullIfEmpty; + +/** + * @author tonytv + */ +public class AccessLogBuilder { + /* + * Valid values for the type attribute in services.xml + * Must be kept in sync with containercluster.rnc:AccessLog + */ + private enum AccessLogTypeLiteral { + VESPA("vespa"), + YAPACHE("yapache"), + DISABLED("disabled"); + + final String attributeValue; + + AccessLogTypeLiteral(String attributeValue) { + this.attributeValue = attributeValue; + } + + static AccessLogTypeLiteral fromAttributeValue(String value) { + return firstMatching( + AccessLogTypeLiteral.values(), + typeLiteral -> typeLiteral.attributeValue.equals(value)).get(); + } + } + + private static class DomBuilder extends VespaDomBuilder.DomConfigProducerBuilder<AccessLogComponent> { + private final AccessLogType accessLogType; + + public DomBuilder(AccessLogType accessLogType) { + this.accessLogType = accessLogType; + } + + @Override + protected AccessLogComponent doBuild(AbstractConfigProducer ancestor, Element spec) { + return new AccessLogComponent( + accessLogType, + fileNamePattern(spec), + rotationInterval(spec), + rotationScheme(spec), + symlinkName(spec)); + } + + private String symlinkName(Element spec) { + return nullIfEmpty(spec.getAttribute("symlinkName")); + } + + private AccessLogConfig.FileHandler.RotateScheme.Enum rotationScheme(Element spec) { + return AccessLogComponent.rotateScheme(nullIfEmpty(spec.getAttribute("rotationScheme"))); + } + + private String rotationInterval(Element spec) { + return nullIfEmpty(spec.getAttribute("rotationInterval")); + } + + private String fileNamePattern(Element spec) { + return nullIfEmpty(spec.getAttribute("fileNamePattern")); + } + } + + public static Optional<AccessLogComponent> buildIfNotDisabled(ContainerCluster cluster, Element accessLogSpec) { + AccessLogTypeLiteral typeLiteral = + getOptionalAttribute(accessLogSpec, "type"). + map(AccessLogTypeLiteral::fromAttributeValue). + orElse(AccessLogTypeLiteral.VESPA); + + switch (typeLiteral) { + case DISABLED: + return Optional.empty(); + case VESPA: + return Optional.of(new DomBuilder(AccessLogType.queryAccessLog).build(cluster, accessLogSpec)); + case YAPACHE: + return Optional.of(new DomBuilder(AccessLogType.yApacheAccessLog).build(cluster, accessLogSpec)); + default: + throw new InconsistentSchemaAndCodeError(); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java new file mode 100644 index 00000000000..544f4f93c69 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleInstantiationSpecificationBuilder.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.processing.handler.ProcessingHandler; +import com.yahoo.search.handler.SearchHandler; +import org.w3c.dom.Element; + +import java.util.*; + +/** + * This object builds a bundle instantiation spec from an XML element. + * + * @author gjoranv + */ +public class BundleInstantiationSpecificationBuilder { + + public static BundleInstantiationSpecification build(Element spec, boolean legacyMode) { + ComponentSpecification id = XmlHelper.getIdRef(spec); + ComponentSpecification classId = getComponentSpecification(spec, "class"); + ComponentSpecification bundle = getComponentSpecification(spec, "bundle"); + + BundleInstantiationSpecification instSpec = new BundleInstantiationSpecification(id, classId, bundle); + if ( ! legacyMode) // TODO: Remove? + validate(instSpec); + + return bundle == null ? setBundleForKnownClass(instSpec) : instSpec; + } + + private static BundleInstantiationSpecification setBundleForKnownClass(BundleInstantiationSpecification spec) { + return BundleMapper.getBundle(spec.getClassName()). + map(spec::inBundle). + orElse(spec); + } + + + private static void validate(BundleInstantiationSpecification instSpec) { + List<String> forbiddenClasses = Arrays.asList( + "com.yahoo.search.handler.SearchHandler", + "com.yahoo.processing.handler.ProcessingHandler"); + + for (String forbiddenClass: forbiddenClasses) { + if (forbiddenClass.equals(instSpec.getClassName())) { + throw new RuntimeException("Setting up " + forbiddenClass + " manually is not supported."); + } + } + } + + //null if missing + private static ComponentSpecification getComponentSpecification(Element spec, String attributeName) { + return (spec.hasAttribute(attributeName)) ? + new ComponentSpecification(spec.getAttribute(attributeName)) : + null; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java new file mode 100644 index 00000000000..e09d47e7bc1 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/BundleMapper.java @@ -0,0 +1,132 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.vespa.defaults.Defaults; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +/** + * @author gjoranv + * @author lulf + * @since 5.45 + */ +public class BundleMapper { + + public static final Path LIBRARY_PATH = Paths.get(Defaults.getDefaults().vespaHome() + "lib/jars"); + public static final String searchAndDocprocBundle = "container-search-and-docproc"; + + private static final Map<String, String> bundleFromClass; + private static final Map<String, Path> bundleFileFromClass; + + public static Optional<String> getBundle(String className) { + return Optional.ofNullable(bundleFromClass.get(className)); + } + + public static Optional<Path> getBundlePath(String className) { + return Optional.ofNullable(absoluteBundlePath(bundleFileFromClass.get(className))); + } + + public static Path absoluteBundlePath(Path fileName) { + if (fileName == null) return null; + return LIBRARY_PATH.resolve(fileName); + } + + /** + * TODO: This is a temporary hack to ensure that users can use our internal components without + * specifying the bundle in which the components reside. Ideally, this information + * should be generated during vespamodel build time. + * + * The container_maven_plugin has much of the logic in place, but needs to be extended. + */ + static { + bundleFromClass = new HashMap<>(); + bundleFileFromClass = new HashMap<>(); + + bundleFromClass.put("com.yahoo.docproc.AbstractConcreteDocumentFactory", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.docproc.DocumentProcessor", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.docproc.SimpleDocumentProcessor", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.docproc.util.JoinerDocumentProcessor", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.docproc.util.SplitterDocumentProcessor", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.example.TimingSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.language.simple.SimpleLinguistics", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.cluster.ClusterSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.fastsearch.FastSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.fastsearch.VespaBackEndSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.querytransform.CJKSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.querytransform.CollapsePhraseSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.querytransform.IndexCombinatorSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.querytransform.LiteralBoostSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.querytransform.NoRankingSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.querytransform.NonPhrasingSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.querytransform.NormalizingSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.querytransform.PhrasingSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.querytransform.RecallSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.querytransform.StemmingSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.searcher.BlendingSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.searcher.CachingSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.searcher.DocumentSourceSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.searcher.FieldCollapsingSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.searcher.FillSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.searcher.JSONDebugSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.searcher.JuniperSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.searcher.KeyValueSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.searcher.MultipleResultsSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.searcher.PosSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.searcher.QuerySnapshotSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.searcher.QueryValidatingSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.searcher.QuotingSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.searcher.ValidateSortingSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.semantics.SemanticSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.statistics.StatisticsSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.prelude.templates.SearchRendererAdaptor", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.Searcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.cluster.ClusterSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.cluster.PingableSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.federation.FederationSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.federation.ForwardingSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.federation.http.ConfiguredHTTPClientSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.federation.http.ConfiguredHTTPProviderSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.federation.http.HTTPClientSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.federation.http.HTTPProviderSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.federation.http.HTTPSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.federation.news.NewsSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.federation.vespa.VespaSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.grouping.GroupingQueryParser", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.grouping.GroupingValidator", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.grouping.vespa.GroupingExecutor", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.handler.SearchWithRendererHandler", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.pagetemplates.PageTemplate", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.pagetemplates.PageTemplateSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.pagetemplates.engine.Resolver", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.pagetemplates.engine.resolvers.DeterministicResolver", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.pagetemplates.engine.resolvers.RandomResolver", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.pagetemplates.model.Renderer", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.pagetemplates.model.Renderer", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.query.rewrite.QueryRewriteSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.query.rewrite.SearchChainDispatcherSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.query.rewrite.rewriters.GenericExpansionRewriter", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.query.rewrite.rewriters.MisspellRewriter", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.query.rewrite.rewriters.NameRewriter", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.querytransform.AllLowercasingSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.querytransform.DefaultPositionSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.querytransform.LegacyCombinator", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.querytransform.LowercasingSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.querytransform.NGramSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.querytransform.QueryCombinator", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.querytransform.VespaLowercasingSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.rendering.Renderer", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.rendering.SectionedRenderer", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.searchchain.ForkingSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.searchchain.example.ExampleSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.searchers.CacheControlSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.statistics.PeakQpsSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.search.statistics.TimingSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.vespa.streamingvisitors.MetricsSearcher", searchAndDocprocBundle); + bundleFromClass.put("com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher", searchAndDocprocBundle); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java new file mode 100644 index 00000000000..a82291cd159 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.config.application.Xml; +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.application.api.ApplicationPackage; + +import com.yahoo.path.Path; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.ContainerModel; +import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; +import com.yahoo.vespa.model.container.configserver.ConfigserverCluster; + +import org.w3c.dom.Element; +import org.w3c.dom.Node; + +import java.util.List; + +/** + * Builds the config model for the standalone config server. + * + * @author lulf + * @since 5.16 + */ +public class ConfigServerContainerModelBuilder extends ContainerModelBuilder { + + private final CloudConfigOptions options; + private final static String HOSTED_VESPA_INCLUDE_DIR = "hosted-vespa"; + + public ConfigServerContainerModelBuilder(CloudConfigOptions options) { + super(true, Networking.enable); + this.options = options; + } + + + @Override + public void doBuild(ContainerModel model, Element spec, ConfigModelContext modelContext) { + ApplicationPackage app = modelContext.getDeployState().getApplicationPackage(); + if (!app.getFiles(Path.fromString(HOSTED_VESPA_INCLUDE_DIR), ".xml").isEmpty()) { + app.validateIncludeDir(HOSTED_VESPA_INCLUDE_DIR); + List<Element> configModelElements = Xml.allElemsFromPath(app, HOSTED_VESPA_INCLUDE_DIR); + mergeInto(spec, configModelElements); + } + + ConfigserverCluster cluster = new ConfigserverCluster(modelContext.getParentProducer(), "configserver", options); + super.doBuild(model, spec, modelContext.modifyParent(cluster)); + cluster.setContainerCluster(model.getCluster()); + } + + private void mergeInto(Element destination, List<Element> configModelElements) { + for (Element jdiscElement: configModelElements) { + for (Node child = jdiscElement.getFirstChild(); child != null; child = child.getNextSibling()) { + Node copiedNode = destination.getOwnerDocument().importNode(child, true); + destination.appendChild(copiedNode); + } + } + } + + @Override + protected void addDefaultComponents(ContainerCluster containerCluster) { + // To avoid search specific stuff. + } + + @Override + protected void addDefaultHandlers(ContainerCluster containerCluster) { + addDefaultHandlersExceptStatus(containerCluster); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java new file mode 100644 index 00000000000..a8055afeea8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -0,0 +1,576 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.google.common.collect.ImmutableList; +import com.yahoo.component.Version; +import com.yahoo.config.application.Xml; +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.application.provider.IncludeDirs; +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.config.model.builder.xml.ConfigModelId; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.container.jdisc.config.MetricDefaultsConfig; +import com.yahoo.search.rendering.RendererRegistry; +import com.yahoo.text.XML; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.builder.xml.dom.DomClientProviderBuilder; +import com.yahoo.vespa.model.builder.xml.dom.DomComponentBuilder; +import com.yahoo.vespa.model.builder.xml.dom.DomFilterBuilder; +import com.yahoo.vespa.model.builder.xml.dom.DomHandlerBuilder; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.builder.xml.dom.NodesSpecification; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.builder.xml.dom.ServletBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.docproc.DomDocprocChainsBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.processing.DomProcessingBuilder; +import com.yahoo.vespa.model.builder.xml.dom.chains.search.DomSearchChainsBuilder; +import com.yahoo.vespa.model.clients.ContainerDocumentApi; +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.ContainerModel; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent; +import com.yahoo.vespa.model.container.component.chain.ProcessingHandler; +import com.yahoo.vespa.model.container.docproc.ContainerDocproc; +import com.yahoo.vespa.model.container.docproc.DocprocChains; +import com.yahoo.vespa.model.container.http.Http; +import com.yahoo.vespa.model.container.http.xml.HttpBuilder; +import com.yahoo.vespa.model.container.jersey.xml.RestApiBuilder; +import com.yahoo.vespa.model.container.processing.ProcessingChains; +import com.yahoo.vespa.model.container.search.ContainerSearch; +import com.yahoo.vespa.model.container.search.PageTemplates; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import com.yahoo.vespa.model.container.search.SemanticRules; +import com.yahoo.vespa.model.container.search.searchchain.SearchChains; +import com.yahoo.vespa.model.container.xml.document.DocumentFactoryBuilder; + +import org.w3c.dom.Element; +import org.w3c.dom.Node; + +import java.util.*; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +/** + * @author tonytv + */ +public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { + + /** + * Default path to vip status file for container in Hosted Vespa. + */ + static final String HOSTED_VESPA_STATUS_FILE = Defaults.getDefaults().vespaHome() + "var/mediasearch/oor/status.html"; + /** + * Path to vip status file for container in Hosted Vespa. Only used if set, else use HOSTED_VESPA_STATUS_FILE + */ + static final String HOSTED_VESPA_STATUS_FILE_YINST_SETTING = "cloudconfig_server__tenant_vip_status_file"; + + public enum Networking { disable, enable } + + private ApplicationPackage app; + private final boolean standaloneBuilder; + private final Networking networking; + protected DeployLogger log; + + public static final List<ConfigModelId> configModelIds = + ImmutableList.of(ConfigModelId.fromName("container"), ConfigModelId.fromName("jdisc")); + + private static final String xmlRendererId = RendererRegistry.xmlRendererId.getName(); + private static final String jsonRendererId = RendererRegistry.jsonRendererId.getName(); + + public ContainerModelBuilder(boolean standaloneBuilder, Networking networking) { + super(ContainerModel.class); + this.standaloneBuilder = standaloneBuilder; + this.networking = networking; + } + + @Override + public List<ConfigModelId> handlesElements() { + return configModelIds; + } + + @Override + public void doBuild(ContainerModel model, Element spec, ConfigModelContext modelContext) { + app = modelContext.getApplicationPackage(); + checkVersion(spec); + + this.log = modelContext.getDeployLogger(); + + ContainerCluster cluster = createContainerCluster(spec, modelContext); + addClusterContent(cluster, spec, modelContext); + addBundlesForPlatformComponents(cluster); + + model.setCluster(cluster); + } + + protected void addBundlesForPlatformComponents(ContainerCluster cluster) { + for (Component<?, ?> component : cluster.getAllComponents()) { + String componentClass = component.model.bundleInstantiationSpec.getClassName(); + BundleMapper.getBundlePath(componentClass). + ifPresent(cluster::addPlatformBundle); + } + } + + private ContainerCluster createContainerCluster(Element spec, final ConfigModelContext modelContext) { + return new VespaDomBuilder.DomConfigProducerBuilder<ContainerCluster>() { + @Override + protected ContainerCluster doBuild(AbstractConfigProducer ancestor, Element producerSpec) { + return new ContainerCluster(ancestor, modelContext.getProducerId(), modelContext.getProducerId()); + } + }.build(modelContext.getParentProducer(), spec); + } + + private void addClusterContent(ContainerCluster cluster, Element spec, ConfigModelContext modelContext) { + DocumentFactoryBuilder.buildDocumentFactories(cluster, spec); + + addConfiguredComponents(cluster, spec); + addHandlers(cluster, spec); + + addHttp(spec, cluster); + addRestApis(spec, cluster); + addServlets(spec, cluster); + addProcessing(spec, cluster); + addSearch(spec, cluster, modelContext.getDeployState().getQueryProfiles(), modelContext.getDeployState().getSemanticRules()); + addDocproc(spec, cluster); + addDocumentApi(spec, cluster); //NOTE: Document API must be set up _after_ search! + + addAccessLogs(cluster, spec); + addRoutingAliases(cluster, spec); + addNodes(cluster, spec); + + addClientProviders(spec, cluster); + addServerProviders(spec, cluster); + addLegacyFilters(spec, cluster); + + addDefaultHandlers(cluster); + addStatusHandlers(cluster, modelContext); + addDefaultComponents(cluster); + setDefaultMetricConsumerFactory(cluster); + + //TODO: overview handler, see DomQrserverClusterBuilder + //TODO: cache options. + } + + private void addRoutingAliases(ContainerCluster cluster, Element spec) { + Element aliases = XML.getChild(spec, "aliases"); + for (Element alias : XML.getChildren(aliases, "service-alias")) { + cluster.serviceAliases().add(XML.getValue(alias)); + } + for (Element alias : XML.getChildren(aliases, "endpoint-alias")) { + cluster.endpointAliases().add(XML.getValue(alias)); + } + } + + private void addConfiguredComponents(ContainerCluster cluster, Element spec) { + for (Element components : XML.getChildren(spec, "components")) { + addIncludes(components); + addConfiguredComponents(cluster, components, "component"); + } + addConfiguredComponents(cluster, spec, "component"); + } + + protected void addDefaultComponents(ContainerCluster cluster) { + } + + protected void setDefaultMetricConsumerFactory(ContainerCluster cluster) { + cluster.setDefaultMetricConsumerFactory(MetricDefaultsConfig.Factory.Enum.STATE_MONITOR); + } + + protected void addDefaultHandlers(ContainerCluster cluster) { + addDefaultHandlersExceptStatus(cluster); + } + + protected void addStatusHandlers(ContainerCluster cluster, ConfigModelContext configModelContext) { + if (configModelContext.getDeployState().isHostedVespa()) { + String name = "status.html"; + Optional<String> statusFile = Optional.ofNullable(System.getenv(HOSTED_VESPA_STATUS_FILE_YINST_SETTING)); + cluster.addComponent( + new FileStatusHandlerComponent(name + "-status-handler", statusFile.orElse(HOSTED_VESPA_STATUS_FILE), + "http://*/" + name, "https://*/" + name)); + } else { + cluster.addVipHandler(); + } + } + + /** + * Intended for use by legacy builders only. + * Will be called during building when using ContainerModelBuilder. + */ + public static void addDefaultHandler_legacyBuilder(ContainerCluster cluster) { + addDefaultHandlersExceptStatus(cluster); + cluster.addVipHandler(); + } + + protected static void addDefaultHandlersExceptStatus(ContainerCluster cluster) { + cluster.addDefaultRootHandler(); + cluster.addMetricStateHandler(); + cluster.addApplicationStatusHandler(); + cluster.addStatisticsHandler(); + } + + private void addClientProviders(Element spec, ContainerCluster cluster) { + for (Element clientSpec: XML.getChildren(spec, "client")) { + cluster.addComponent(new DomClientProviderBuilder().build(cluster, clientSpec)); + } + } + + private void addServerProviders(Element spec, ContainerCluster cluster) { + addConfiguredComponents(cluster, spec, "server"); + } + + private void addLegacyFilters(Element spec, ContainerCluster cluster) { + for (Component component : buildLegacyFilters(cluster, spec)) { + cluster.addComponent(component); + } + } + + private List<Component> buildLegacyFilters(AbstractConfigProducer ancestor, + Element spec) { + List<Component> components = new ArrayList<>(); + + for (Element node : XML.getChildren(spec, "filter")) { + components.add(new DomFilterBuilder().build(ancestor, node)); + } + return components; + } + + protected void addAccessLogs(ContainerCluster cluster, Element spec) { + List<Element> accessLogElements = getAccessLogElements(spec); + + for (Element accessLog : accessLogElements) { + AccessLogBuilder.buildIfNotDisabled(cluster, accessLog).ifPresent(cluster::addComponent); + } + + if (accessLogElements.isEmpty() && cluster.getSearch() != null) + cluster.addDefaultSearchAccessLog(); + } + + protected final List<Element> getAccessLogElements(Element spec) { + return XML.getChildren(spec, "accesslog"); + } + + + protected void addHttp(Element spec, ContainerCluster cluster) { + Element httpElement = XML.getChild(spec, "http"); + if (httpElement != null) { + cluster.setHttp(buildHttp(cluster, httpElement)); + } + } + + private Http buildHttp(ContainerCluster cluster, Element httpElement) { + Http http = new HttpBuilder().build(cluster, httpElement); + + if (networking == Networking.disable) + http.removeAllServers(); + + return http; + } + + protected void addRestApis(Element spec, ContainerCluster cluster) { + for (Element restApiElem : XML.getChildren(spec, "rest-api")) { + cluster.addRestApi( + new RestApiBuilder().build(cluster, restApiElem)); + } + } + + private void addServlets(Element spec, ContainerCluster cluster) { + for (Element servletElem : XML.getChildren(spec, "servlet")) { + cluster.addServlet( + new ServletBuilder().build(cluster, servletElem)); + } + } + + private void addDocumentApi(Element spec, ContainerCluster cluster) { + ContainerDocumentApi containerDocumentApi = buildDocumentApi(cluster, spec); + if (containerDocumentApi != null) { + cluster.setDocumentApi(containerDocumentApi); + } + } + + private void addDocproc(Element spec, ContainerCluster cluster) { + ContainerDocproc containerDocproc = buildDocproc(cluster, spec); + if (containerDocproc != null) { + cluster.setDocproc(containerDocproc); + + ContainerDocproc.Options docprocOptions = containerDocproc.options; + cluster.setMbusParams(new ContainerCluster.MbusParams( + docprocOptions.maxConcurrentFactor, docprocOptions.documentExpansionFactor, docprocOptions.containerCoreMemory)); + } + } + + private void addSearch(Element spec, ContainerCluster cluster, QueryProfiles queryProfiles, SemanticRules semanticRules) { + Element searchElement = XML.getChild(spec, "search"); + if (searchElement != null) { + addIncludes(searchElement); + cluster.setSearch(buildSearch(cluster, searchElement, queryProfiles, semanticRules)); + + addSearchHandler(cluster, searchElement); + validateAndAddConfiguredComponents(cluster, searchElement, "renderer", ContainerModelBuilder::validateRendererElement); + } + } + + private void addProcessing(Element spec, ContainerCluster cluster) { + Element processingElement = XML.getChild(spec, "processing"); + if (processingElement != null) { + addIncludes(processingElement); + cluster.setProcessingChains(new DomProcessingBuilder(null).build(cluster, processingElement), + serverBindings(processingElement, ProcessingChains.defaultBindings)); + validateAndAddConfiguredComponents(cluster, processingElement, "renderer", ContainerModelBuilder::validateRendererElement); + } + } + + private ContainerSearch buildSearch(ContainerCluster containerCluster, Element producerSpec, + QueryProfiles queryProfiles, SemanticRules semanticRules) { + SearchChains searchChains = new DomSearchChainsBuilder(null, false).build(containerCluster, producerSpec); + + ContainerSearch containerSearch = new ContainerSearch(containerCluster, searchChains, new ContainerSearch.Options()); + + applyApplicationPackageDirectoryConfigs(containerCluster.getRoot().getDeployState().getApplicationPackage(), containerSearch); + containerSearch.setQueryProfiles(queryProfiles); + containerSearch.setSemanticRules(semanticRules); + + return containerSearch; + } + + private void applyApplicationPackageDirectoryConfigs(ApplicationPackage applicationPackage,ContainerSearch containerSearch) { + PageTemplates.validate(applicationPackage); + containerSearch.setPageTemplates(PageTemplates.create(applicationPackage)); + } + + private void addHandlers(ContainerCluster cluster, Element spec) { + for (Element component: XML.getChildren(spec, "handler")) { + cluster.addComponent( + new DomHandlerBuilder().build(cluster, component)); + } + } + + private void checkVersion(Element spec) { + String version = spec.getAttribute("version"); + + if (!Version.fromString(version).equals(new Version(1))) { + throw new RuntimeException("Expected container version to be 1.0, but got " + version); + } + } + + private void addNodes(ContainerCluster cluster, Element spec) { + if (standaloneBuilder) { + addStandaloneNode(cluster); + } else { + addNodesFromXml(cluster, spec); + } + } + + private void addStandaloneNode(ContainerCluster cluster) { + Container container = new Container(cluster, "standalone"); + cluster.addContainers(Collections.singleton(container)); + } + + private void addNodesFromXml(ContainerCluster cluster, Element spec) { + Element nodesElement = XML.getChild(spec, "nodes"); + if (nodesElement == null) { // default single node on localhost + Container container = new Container(cluster, "container.0"); + HostResource host = allocateSingleNodeHost(cluster, log); + container.setHostResource(host); + if ( ! container.isInitialized() ) // TODO: Fold this into initService + container.initService(); + cluster.addContainers(Collections.singleton(container)); + } + else { + String defaultJvmArgs = nodesElement.getAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME); + String defaultPreLoad = null; + if (nodesElement.hasAttribute(VespaDomBuilder.PRELOAD_ATTRIB_NAME)) { + defaultPreLoad = nodesElement.getAttribute(VespaDomBuilder.PRELOAD_ATTRIB_NAME); + } + boolean useCpuSocketAffinity = false; + if (nodesElement.hasAttribute(VespaDomBuilder.CPU_SOCKET_AFFINITY_ATTRIB_NAME)) { + useCpuSocketAffinity = Boolean.parseBoolean(nodesElement.getAttribute(VespaDomBuilder.CPU_SOCKET_AFFINITY_ATTRIB_NAME)); + } + List<Container> result = new ArrayList<>(); + result.addAll(createNodesFromXmlNodeCount(cluster, nodesElement)); + addNodesFromXmlNodeList(cluster, spec, nodesElement, result); + applyDefaultJvmArgs(result, defaultJvmArgs); + applyRoutingAliasProperties(result, cluster); + if (defaultPreLoad != null) { + applyDefaultPreload(result, defaultPreLoad); + } + if (useCpuSocketAffinity) { + AbstractService.distributeCpuSocketAffinity(result); + } + + cluster.addContainers(result); + } + } + + private void applyRoutingAliasProperties(List<Container> result, ContainerCluster cluster) { + if (!cluster.serviceAliases().isEmpty()) { + result.forEach(container -> { + container.setProp("servicealiases", cluster.serviceAliases().stream().collect(Collectors.joining(","))); + }); + } + if (!cluster.endpointAliases().isEmpty()) { + result.forEach(container -> { + container.setProp("endpointaliases", cluster.endpointAliases().stream().collect(Collectors.joining(","))); + }); + } + } + + private HostResource allocateSingleNodeHost(ContainerCluster cluster, DeployLogger logger) { + if (cluster.isHostedVespa()) { + ClusterSpec clusterSpec = ClusterSpec.from(ClusterSpec.Type.container, ClusterSpec.Id.from(cluster.getName()), Optional.empty()); + return cluster.getHostSystem().allocateHosts(clusterSpec, Capacity.fromNodeCount(1), 1, logger).keySet().iterator().next(); + } else { + return cluster.getHostSystem().getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC); + } + } + + private void addNodesFromXmlNodeList(ContainerCluster cluster, + Element spec, Element nodesElement, List<Container> result) { + int nodeCount = 0; + for (Element nodeElem: XML.getChildren(nodesElement, "node")) { + Container container = new ContainerServiceBuilder("container." + nodeCount).build(cluster, nodeElem); + result.add(container); + ++nodeCount; + } + } + + private List<Container> createNodesFromXmlNodeCount(ContainerCluster cluster, Element nodesElement) { + List<Container> result = new ArrayList<>(); + if (nodesElement.hasAttribute("count")) { + NodesSpecification nodesSpecification = NodesSpecification.from(new ModelElement(nodesElement)); + Map<HostResource, ClusterMembership> hosts = nodesSpecification.provision(cluster.getRoot().getHostSystem(), ClusterSpec.Type.container, ClusterSpec.Id.from(cluster.getName()), Optional.empty(), log); + for (Map.Entry<HostResource, ClusterMembership> entry : hosts.entrySet()) { + String id = "container." + entry.getValue().index(); + Container container = new Container(cluster, id, entry.getValue().retired()); + container.setHostResource(entry.getKey()); + container.initService(); + result.add(container); + } + } + return result; + } + + private void applyDefaultJvmArgs(List<Container> containers, String defaultJvmArgs) { + for (Container container: containers) { + if (container.getJvmArgs().isEmpty()) + container.prependJvmArgs(defaultJvmArgs); + } + } + + private void applyDefaultPreload(List<Container> containers, String defaultPreLoad) { + for (Container container: containers) { + container.setPreLoad(defaultPreLoad); + } + } + + private void addSearchHandler(ContainerCluster cluster, Element searchElement) { + ProcessingHandler<SearchChains> searchHandler = new ProcessingHandler<>( + cluster.getSearch().getChains(), "com.yahoo.search.handler.SearchHandler"); + + String[] defaultBindings = {"http://*/search/*", "https://*/search/*"}; + for (String binding: serverBindings(searchElement, defaultBindings)) { + searchHandler.addServerBindings(binding); + } + + cluster.addComponent(searchHandler); + } + + private String[] serverBindings(Element searchElement, String... defaultBindings) { + List<Element> bindings = XML.getChildren(searchElement, "binding"); + if (bindings.isEmpty()) + return defaultBindings; + + return toBindingList(bindings); + } + + private String[] toBindingList(List<Element> bindingElements) { + List<String> result = new ArrayList<>(); + + for (Element element: bindingElements) { + String text = element.getTextContent().trim(); + if (!text.isEmpty()) + result.add(text); + } + + return result.toArray(new String[result.size()]); + } + + private ContainerDocumentApi buildDocumentApi(ContainerCluster cluster, Element spec) { + Element documentApiElement = XML.getChild(spec, "document-api"); + if (documentApiElement == null) { + return null; + } + + ContainerDocumentApi.Options documentApiOptions = DocumentApiOptionsBuilder.build(documentApiElement); + return new ContainerDocumentApi(cluster, documentApiOptions); + } + + private ContainerDocproc buildDocproc(ContainerCluster cluster, Element spec) { + Element docprocElement = XML.getChild(spec, "document-processing"); + if (docprocElement == null) + return null; + + addIncludes(docprocElement); + DocprocChains chains = new DomDocprocChainsBuilder(null, false).build(cluster, docprocElement); + + ContainerDocproc.Options docprocOptions = DocprocOptionsBuilder.build(docprocElement); + return new ContainerDocproc(cluster, chains, docprocOptions, !standaloneBuilder); + } + + private void addIncludes(Element parentElement) { + List<Element> includes = XML.getChildren(parentElement, IncludeDirs.INCLUDE); + if (includes == null || includes.isEmpty()) { + return; + } + if (app == null) { + throw new IllegalArgumentException("Element <include> given in XML config, but no application package given."); + } + for (Element include : includes) { + addInclude(parentElement, include); + } + } + + private void addInclude(Element parentElement, Element include) { + String dirName = include.getAttribute(IncludeDirs.DIR); + app.validateIncludeDir(dirName); + + List<Element> includedFiles = Xml.allElemsFromPath(app, dirName); + for (Element includedFile : includedFiles) { + List<Element> includedSubElements = XML.getChildren(includedFile); + for (Element includedSubElement : includedSubElements) { + Node copiedNode = parentElement.getOwnerDocument().importNode(includedSubElement, true); + parentElement.appendChild(copiedNode); + } + } + } + + public static void addConfiguredComponents(ContainerCluster cluster, Element spec, String componentName) { + for (Element node : XML.getChildren(spec, componentName)) { + cluster.addComponent(new DomComponentBuilder().build(cluster, node)); + } + } + + public static void validateAndAddConfiguredComponents(ContainerCluster cluster, Element spec, String componentName, Consumer<Element> elementValidator) { + for (Element node : XML.getChildren(spec, componentName)) { + elementValidator.accept(node); // throws exception here if something is wrong + cluster.addComponent(new DomComponentBuilder().build(cluster, node)); + } + } + + /** + * Disallow renderers named "DefaultRenderer" or "JsonRenderer" + */ + private static void validateRendererElement(Element element) { + String idAttr = element.getAttribute("id"); + + if (idAttr.equals(xmlRendererId) || idAttr.equals(jsonRendererId)) { + throw new IllegalArgumentException(String.format("Renderer id %s is reserved for internal use", idAttr)); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java new file mode 100644 index 00000000000..785ab1f7504 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerServiceBuilder.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.component.ComponentSpecification; +import com.yahoo.config.model.ConfigModelUtils; +import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.container.Container; +import org.w3c.dom.Element; + +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Logger; + +/** + * @author tonytv + */ +public class ContainerServiceBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Container> { + + private final String id; + + public ContainerServiceBuilder(String id) { + this.id = id; + } + + @Override + protected Container doBuild(AbstractConfigProducer parent, Element nodeElem) { + return new Container(parent, id, readServerPortOverrides(nodeElem)); + } + + private List<Container.PortOverride> readServerPortOverrides(Element spec) { + List<Container.PortOverride> portOverrides = new ArrayList<>(); + + for (Element serverPort: XML.getChildren(spec, "server-port")) { + ComponentSpecification serverId = XmlHelper.getIdRef(serverPort); + int port = Integer.parseInt(serverPort.getAttribute("port")); + + portOverrides.add(new Container.PortOverride(serverId, port)); + } + + return portOverrides; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocprocOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocprocOptionsBuilder.java new file mode 100644 index 00000000000..0b13187bd58 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocprocOptionsBuilder.java @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.vespa.model.container.docproc.ContainerDocproc; +import org.w3c.dom.Element; + +/** + * Extracted from DomDocProcClusterBuilder + */ +public class DocprocOptionsBuilder { + public static ContainerDocproc.Options build(Element spec) { + return new ContainerDocproc.Options( + getCompression(spec), + getMaxMessagesInQueue(spec), + getSizeInMegabytes(spec.getAttribute("maxqueuebytesize")), + getTime(spec.getAttribute("maxqueuewait")), + getFactor(spec.getAttribute("maxconcurrentfactor")), + getFactor(spec.getAttribute("documentexpansionfactor")), + getInt(spec.getAttribute("containercorememory"))); + } + + private static Integer getInt(String integer) { + return integer == null || integer.trim().isEmpty() ? + null: + Integer.parseInt(integer); + } + + private static boolean getCompression(Element spec) { + return (spec.hasAttribute("compressdocuments") && spec.getAttribute("compressdocuments").equals("true")); + } + + private static Double getFactor(String factor) { + return factor == null || factor.trim().isEmpty() ? + null : + Double.parseDouble(factor); + } + + + private static Integer getMaxMessagesInQueue(Element spec) { + // get max queue size (number of messages), if set + Integer maxMessagesInQueue = null; + if (spec.hasAttribute("maxmessagesinqueue")) { + maxMessagesInQueue = Integer.valueOf(spec.getAttribute("maxmessagesinqueue")); + } + return maxMessagesInQueue; + } + + private static Integer getSizeInMegabytes(String size) { + if (size == null) { + return null; + } + size = size.trim(); + if (size.isEmpty()) { + return null; + } + + Integer megabyteSize; + if (size.endsWith("m")) { + size = size.substring(0, size.length() - 1); + megabyteSize = Integer.parseInt(size); + } else if (size.endsWith("g")) { + size = size.substring(0, size.length() - 1); + megabyteSize = Integer.parseInt(size) * 1024; + } else { + throw new IllegalArgumentException("Heap sizes for docproc must be set to Xm or Xg, where X is an integer specifying megabytes or gigabytes, respectively."); + } + return megabyteSize; + } + + private static Integer getTime(String intStr) { + if (intStr == null) { + return null; + } + intStr = intStr.trim(); + if (intStr.isEmpty()) { + return null; + } + + return 1000 * (int)Double.parseDouble(intStr); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java new file mode 100644 index 00000000000..df2090db166 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java @@ -0,0 +1,119 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.text.XML; +import com.yahoo.vespa.model.clients.ContainerDocumentApi; +import org.w3c.dom.Element; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.logging.Logger; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @since 5.1.11 + */ +public class DocumentApiOptionsBuilder { + private static final Logger log = Logger.getLogger(DocumentApiOptionsBuilder.class.getName()); + private static final String[] DEFAULT_BINDINGS = {"http://*/", "https://*/"}; + + public static ContainerDocumentApi.Options build(Element spec) { + return new ContainerDocumentApi.Options( + getBindings(spec), + getAbortOnDocumentError(spec), + getRoute(spec), + getMaxPendingDocs(spec), + getMaxPendingBytes(spec), + getRetryEnabled(spec), + getRetryDelay(spec), + getTimeout(spec), + getTracelevel(spec), + getMbusPort(spec), + getDocprocChain(spec)); + } + + private static List<String> getBindings(Element spec) { + Collection<Element> bindingElems = XML.getChildren(spec, "binding"); + if (bindingElems.isEmpty()) + return Arrays.asList(DEFAULT_BINDINGS); + + List<String> bindings = new ArrayList<>(); + for (Element e :bindingElems) { + String binding = getBinding(e); + bindings.add(binding); + } + return bindings; + } + + private static String getBinding(Element e) { + String binding = XML.getValue(e); + if (! binding.endsWith("/")) { + log.warning("Adding a trailing '/' to the document-api binding: " + binding + " -> " + binding + "/"); + binding = binding + "/"; + } + return binding; + } + + private static String getCleanValue(Element spec, String name) { + Element elem = XML.getChild(spec, name); + if (elem == null) { + return null; + } + String value = elem.getFirstChild().getNodeValue(); + if (value == null) { + return null; + } + value = value.trim(); + return value.isEmpty() ? null : value; + } + + private static String getDocprocChain(Element spec) { + return getCleanValue(spec, "docprocchain"); + } + + private static Integer getMbusPort(Element spec) { + String value = getCleanValue(spec, "mbusport"); + return value == null ? null : Integer.parseInt(value); + } + + private static Integer getTracelevel(Element spec) { + String value = getCleanValue(spec, "tracelevel"); + return value == null ? null : Integer.parseInt(value); + } + + private static Double getTimeout(Element spec) { + String value = getCleanValue(spec, "timeout"); + return value == null ? null : Double.parseDouble(value); + } + + private static Double getRetryDelay(Element spec) { + String value = getCleanValue(spec, "retrydelay"); + return value == null ? null : Double.parseDouble(value); + } + + private static Boolean getRetryEnabled(Element spec) { + String value = getCleanValue(spec, "retryenabled"); + return value == null ? null : Boolean.parseBoolean(value); + } + + private static Integer getMaxPendingBytes(Element spec) { + String value = getCleanValue(spec, "maxpendingbytes"); + return value == null ? null : Integer.parseInt(value); + } + + private static Integer getMaxPendingDocs(Element spec) { + String value = getCleanValue(spec, "maxpendingdocs"); + return value == null ? null : Integer.parseInt(value); + } + + private static String getRoute(Element spec) { + return getCleanValue(spec, "route"); + } + + private static Boolean getAbortOnDocumentError(Element spec) { + String value = getCleanValue(spec, "abortondocumenterror"); + return value == null ? null : Boolean.parseBoolean(value); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/InconsistentSchemaAndCodeError.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/InconsistentSchemaAndCodeError.java new file mode 100644 index 00000000000..46e7271fe9b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/InconsistentSchemaAndCodeError.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +/** + * @author tonytv + */ +public class InconsistentSchemaAndCodeError extends Error {} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ManhattanContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ManhattanContainerModelBuilder.java new file mode 100644 index 00000000000..f6ed6c2eb7d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ManhattanContainerModelBuilder.java @@ -0,0 +1,172 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.component.ComponentId; +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.container.jdisc.config.MetricDefaultsConfig; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.AccessLogComponent; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.container.component.FileStatusHandlerComponent; +import com.yahoo.vespa.model.container.http.ConnectorFactory; +import com.yahoo.vespa.model.container.http.FilterChains; +import com.yahoo.vespa.model.container.http.Http; +import com.yahoo.vespa.model.container.http.JettyHttpServer; +import org.w3c.dom.Element; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.logging.Level; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.yahoo.collections.CollectionUtil.first; +import static com.yahoo.container.core.AccessLogConfig.FileHandler.RotateScheme; +import static com.yahoo.vespa.model.container.xml.BundleMapper.absoluteBundlePath; + +/** + * @author tonytv + */ +public final class ManhattanContainerModelBuilder extends ContainerModelBuilder { + + static final String MANHATTAN_FILE_NAME_PATTERN = Defaults.getDefaults().vespaHome() + "logs/jdisc_core/access.%Y-%m-%d-%H"; + static final String MANHATTAN_ROTATION_INTERVAL = "0 60 ..."; + static final RotateScheme.Enum MANHATTAN_ROTATION_SCHEME = RotateScheme.DATE; + static final String MANHATTAN_SYMLINK_NAME = "access"; + + public interface BundleFiles { + // TODO: move constants to the DH code base. + Set<Path> dhBundles = new HashSet<>(Arrays.asList( + Paths.get("apache_avro/avro.jar"), + Paths.get("apache_avro/commons-compress.jar"), + Paths.get("apache_avro/paranamer.jar"), + Paths.get("apache_avro/jackson-core-asl.jar"), + Paths.get("apache_avro/jackson-mapper-asl.jar"), + Paths.get("dh_rainbow_client_api_java.jar"), + Paths.get("dh_rainbow_util_batch_java.jar"), + Paths.get("dh_rainbow_util_java.jar"))); + } + + private final int httpPort; + private JettyHttpServer jettyHttpServer; + + public ManhattanContainerModelBuilder(int httpPort) { + super(true, Networking.enable); + this.httpPort = httpPort; + } + + @Override + protected void addBundlesForPlatformComponents(ContainerCluster cluster) { + super.addBundlesForPlatformComponents(cluster); + BundleFiles.dhBundles.forEach( + bundleFile -> cluster.addPlatformBundle(absoluteBundlePath(bundleFile))); + } + + @Override + protected void setDefaultMetricConsumerFactory(ContainerCluster cluster) { + cluster.setDefaultMetricConsumerFactory(MetricDefaultsConfig.Factory.Enum.YAMAS_SCOREBOARD); + } + + @Override + protected void addAccessLogs(ContainerCluster cluster, Element spec) { + warnIfAccessLogsDefined(spec); + + checkNotNull(jettyHttpServer, "addHttp must be called first"); + cluster.addComponent(createManhattanAccessLog()); + } + + private Component createManhattanAccessLog() { + return new AccessLogComponent(AccessLogComponent.AccessLogType.yApacheAccessLog, + MANHATTAN_FILE_NAME_PATTERN, + MANHATTAN_ROTATION_INTERVAL, + MANHATTAN_ROTATION_SCHEME, + MANHATTAN_SYMLINK_NAME); + } + + private void warnIfAccessLogsDefined(Element spec) { + List<Element> accessLogElements = getAccessLogElements(spec); + if (!accessLogElements.isEmpty()) { + logManhattanInfo("Ignoring " + accessLogElements.size() + + " access log elements in services.xml, using default yapache access logging instead."); + } + } + + @Override + protected void addDefaultHandlers(ContainerCluster cluster) { + addDefaultHandlersExceptStatus(cluster); + } + + @Override + protected void addStatusHandlers(ContainerCluster cluster, ConfigModelContext configModelContext) { + addStatusHandlerForJDiscStatusPackage(cluster, "status.html"); //jdisc_status + addStatusHandlerForJDiscStatusPackage(cluster, "akamai"); //jdisc_akamai + } + + private static void addStatusHandlerForJDiscStatusPackage(ContainerCluster cluster, String name) { + cluster.addComponent( + new FileStatusHandlerComponent(name + "-status-handler", Defaults.getDefaults().vespaHome() + "libexec/jdisc/" + name, + "http://*/" + name, "https://*/" + name)); + } + + @Override + protected void addHttp(Element spec, ContainerCluster cluster) { + super.addHttp(spec, cluster); + ensureHasHttp(cluster); + ensureOneHttpServer(cluster.getHttp()); + } + + private void ensureHasHttp(ContainerCluster cluster) { + if (cluster.getHttp() == null) + cluster.setHttp(createHttp()); + } + + private Http createHttp() { + Http http = new Http(Collections.<Http.Binding>emptyList()); + http.setFilterChains(new FilterChains(http)); + return http; + } + + private void ensureOneHttpServer(Http http) { + if (http.getHttpServer() == null || http.getHttpServer().getConnectorFactories().isEmpty()) { + JettyHttpServer jettyHttpServer = new JettyHttpServer(new ComponentId("main-http-server")); + http.setHttpServer(jettyHttpServer); + ConnectorFactory connectorFactory = new ConnectorFactory("main-http-connector", + httpPort, null); + http.getHttpServer().addConnector(connectorFactory); + } else { + removeAllButOneConnector(http.getHttpServer()); + ConnectorFactory connectorFactory = first(http.getHttpServer().getConnectorFactories()); + connectorFactory.setListenPort(httpPort); + } + jettyHttpServer = http.getHttpServer(); + } + + private void removeAllButOneConnector(JettyHttpServer jettyHttpServer) { + int removed = 0; + + if (jettyHttpServer.getConnectorFactories().size() > 1) { + for (int i = jettyHttpServer.getConnectorFactories().size() - 1; i > 0; i--) { + ConnectorFactory c = jettyHttpServer.getConnectorFactories().get(i); + jettyHttpServer.removeConnector(c); + ++removed; + } + } + + if (removed > 0) { + logManhattanInfo("Using only the first http server " + jettyHttpServer.getConnectorFactories().get(0).getName()); + } + } + + private static <E> List<E> tail(List<E> list) { + return list.subList(1, list.size()); + } + + private void logManhattanInfo(String message) { + log.log(Level.INFO, "[Manhattan] " + message); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/document/DocumentFactoryBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/document/DocumentFactoryBuilder.java new file mode 100644 index 00000000000..96ab9fd2f52 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/document/DocumentFactoryBuilder.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.xml.document; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.Component; +import org.w3c.dom.Element; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Handles the document bindings (concrete document types). Register the concrete document factories as components. + * + * @author vegardh + * @since 5.1.10 + */ +public class DocumentFactoryBuilder { + private static final String CONCRETE_DOC_FACTORY_CLASS = "ConcreteDocumentFactory"; + + public static void buildDocumentFactories(ContainerCluster cluster, Element spec) { + Map<String, String> types = new LinkedHashMap<>(); + for (Element e : XML.getChildren(spec, "document")) { + String type = e.getAttribute("type"); + String clazz = e.getAttribute("class"); + // Empty pkg is forbidden in the documentgen Mojo. + if (clazz.indexOf('.')<0) throw new IllegalArgumentException("Malformed class for <document> binding, must be a full class with package: "+clazz); + String pkg = clazz.substring(0, clazz.lastIndexOf('.')); + String concDocFactory=pkg+"."+CONCRETE_DOC_FACTORY_CLASS; + String bundle = e.getAttribute("bundle"); + Component<AbstractConfigProducer<?>, ComponentModel> component = new Component<>( + new ComponentModel(BundleInstantiationSpecification.getFromStrings(concDocFactory, concDocFactory, bundle))); + if (!cluster.getComponentsMap().containsKey(component.getComponentId())) cluster.addComponent(component); + types.put(type, concDocFactory); + } + cluster.concreteDocumentTypes().putAll(types); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/package-info.java new file mode 100644 index 00000000000..2ae232adcd6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.model.container.xml; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/BucketSplitting.java b/config-model/src/main/java/com/yahoo/vespa/model/content/BucketSplitting.java new file mode 100644 index 00000000000..e320e44c237 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/BucketSplitting.java @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.content.cluster.ContentCluster; + +/** + * Represents configuration for bucket splitting. + */ +public class BucketSplitting implements StorDistributormanagerConfig.Producer { + Integer maxDocuments; + Integer splitSize; + Integer minSplitCount; + boolean useInlineBucketSplitting; + + public static class Builder { + public BucketSplitting build(ContentCluster cluster, ModelElement clusterElem) { + ModelElement tuning = clusterElem.getChild("tuning"); + if (tuning == null) { + return new BucketSplitting(cluster.isMemfilePersistence(), null, null, null); + } + + ModelElement bucketSplitting = tuning.getChild("bucket-splitting"); + if (bucketSplitting != null) { + Integer maxDocuments = bucketSplitting.getIntegerAttribute("max-documents"); + Integer splitSize = bucketSplitting.getIntegerAttribute("max-size"); + Integer minSplitCount = bucketSplitting.getIntegerAttribute("minimum-bits"); + + return new BucketSplitting(cluster.isMemfilePersistence(), maxDocuments, splitSize, minSplitCount); + } + + return new BucketSplitting(cluster.isMemfilePersistence(), null, null, null); + } + } + + public BucketSplitting(boolean useInlineBucketSplitting, Integer maxDocuments, Integer splitSize, Integer minSplitCount) { + this.maxDocuments = maxDocuments; + this.splitSize = splitSize; + this.minSplitCount = minSplitCount; + this.useInlineBucketSplitting = useInlineBucketSplitting; + } + + @Override + public void getConfig(StorDistributormanagerConfig.Builder builder) { + if (maxDocuments != null) { + builder.splitcount(maxDocuments); + builder.joincount(maxDocuments / 2); + } + if (splitSize != null) { + builder.splitsize(splitSize); + builder.joinsize(splitSize / 2); + } + if (minSplitCount != null) { + builder.minsplitcount(minSplitCount); + } + + builder.inlinebucketsplitting(useInlineBucketSplitting); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java new file mode 100644 index 00000000000..fd1bc8a3362 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java @@ -0,0 +1,121 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.vespa.config.content.FleetcontrollerConfig; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.model.producer.AbstractConfigProducerRoot; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.utils.Duration; +import org.w3c.dom.Element; + +/** + * Config generation for common parameters for all fleet controllers. + * + * TODO: Author + */ +public class ClusterControllerConfig extends AbstractConfigProducer implements FleetcontrollerConfig.Producer { + + public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<ClusterControllerConfig> { + String clusterName; + ModelElement clusterElement; + + public Builder(String clusterName, ModelElement clusterElement) { + this.clusterName = clusterName; + this.clusterElement = clusterElement; + } + + @Override + protected ClusterControllerConfig doBuild(AbstractConfigProducer ancestor, Element producerSpec) { + ModelElement tuning = null; + + ModelElement clusterTuning = clusterElement.getChild("tuning"); + if (clusterTuning != null) { + tuning = clusterTuning.getChild("cluster-controller"); + } + + if (tuning != null) { + return new ClusterControllerConfig(ancestor, clusterName, + tuning.childAsDuration("init-progress-time"), + tuning.childAsDuration("transition-time"), + tuning.childAsLong("max-premature-crashes"), + tuning.childAsDuration("stable-state-period"), + tuning.childAsDouble("min-distributor-up-ratio"), + tuning.childAsDouble("min-storage-up-ratio"), + clusterElement.childAsInteger("tuning.bucket-splitting.minimum-bits")); + } else { + return new ClusterControllerConfig(ancestor, clusterName, null, null, null, null, null, null, + clusterElement.childAsInteger("tuning.bucket-splitting.minimum-bits")); + } + } + } + + String clusterName; + Duration initProgressTime; + Duration transitionTime; + Long maxPrematureCrashes; + Duration stableStateTimePeriod; + Double minDistributorUpRatio; + Double minStorageUpRatio; + Integer minSplitBits; + + private ClusterControllerConfig(AbstractConfigProducer parent, + String clusterName, + Duration initProgressTime, + Duration transitionTime, + Long maxPrematureCrashes, + Duration stableStateTimePeriod, + Double minDistributorUpRatio, + Double minStorageUpRatio, + Integer minSplitBits) { + super(parent, "fleetcontroller"); + + this.clusterName = clusterName; + this.initProgressTime = initProgressTime; + this.transitionTime = transitionTime; + this.maxPrematureCrashes = maxPrematureCrashes; + this.stableStateTimePeriod = stableStateTimePeriod; + this.minDistributorUpRatio = minDistributorUpRatio; + this.minStorageUpRatio = minStorageUpRatio; + this.minSplitBits = minSplitBits; + } + + @Override + public void getConfig(FleetcontrollerConfig.Builder builder) { + AbstractConfigProducerRoot root = getRoot(); + if (root instanceof VespaModel) { + String zooKeeperAddress = + root.getAdmin().getZooKeepersConfigProvider().getZooKeepersConnectionSpec(); + builder.zookeeper_server(zooKeeperAddress); + } else { + builder.zookeeper_server(""); + } + + builder.index(0); + builder.cluster_name(clusterName); + builder.fleet_controller_count(getChildren().size()); + + if (initProgressTime != null) { + builder.init_progress_time((int) initProgressTime.getMilliSeconds()); + } + if (transitionTime != null) { + builder.storage_transition_time((int) transitionTime.getMilliSeconds()); + } + if (maxPrematureCrashes != null) { + builder.max_premature_crashes(maxPrematureCrashes.intValue()); + } + if (stableStateTimePeriod != null) { + builder.stable_state_time_period((int) stableStateTimePeriod.getMilliSeconds()); + } + if (minDistributorUpRatio != null) { + builder.min_distributor_up_ratio(minDistributorUpRatio); + } + if (minStorageUpRatio != null) { + builder.min_storage_up_ratio(minStorageUpRatio); + } + if (minSplitBits != null) { + builder.ideal_distribution_bits(minSplitBits); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java new file mode 100644 index 00000000000..b9fe10e4873 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -0,0 +1,347 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.config.model.ApplicationConfigProducerRoot; +import com.yahoo.config.model.ConfigModel; +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.config.model.ConfigModelRepoAdder; +import com.yahoo.config.model.admin.AdminModel; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.model.producer.AbstractConfigProducerRoot; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.model.*; +import com.yahoo.vespa.model.builder.VespaModelBuilder; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.ContainerModel; +import com.yahoo.vespa.model.container.docproc.ContainerDocproc; +import com.yahoo.vespa.model.container.docproc.DocprocChain; +import com.yahoo.vespa.model.container.docproc.DocprocChains; +import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.search.AbstractSearchCluster; +import com.yahoo.vespa.model.search.IndexedSearchCluster; +import com.yahoo.vespa.model.search.IndexingDocprocChain; +import com.yahoo.vespa.model.search.SearchNode; + +import java.util.*; +import java.util.logging.Logger; + +/** + * The config model from a content tag in services. + * This consists mostly of a ContentCluster. + * + * @author balder + */ +public class Content extends ConfigModel { + + private static final Logger log = Logger.getLogger(Content.class.getName()); + + private ContentCluster cluster; + private Optional<ContainerCluster> ownedIndexingCluster = Optional.empty(); + private final boolean hostedVespa; + + // Dependencies to other models + private final AdminModel adminModel; + private final Collection<ContainerModel> containers; // to find or add the docproc container + + @SuppressWarnings({ "UnusedDeclaration"}) // Created by reflection in ConfigModelRepo + public Content(ConfigModelContext modelContext, AdminModel adminModel, Collection<ContainerModel> containers) { + super(modelContext); + modelContext.getParentProducer().getRoot(); + hostedVespa = modelContext.getDeployState().isHostedVespa(); + this.adminModel = adminModel; + this.containers = containers; + } + + /** Returns the admin model of this system */ + public AdminModel adminModel() { return adminModel; } + + /** Called by DomContentBuilder during build */ + public void setCluster(ContentCluster cluster, ConfigModelContext configModelContext) { + this.cluster = cluster; + initializeIndexingClusters(containers, + configModelContext.getConfigModelRepoAdder(), + (ApplicationConfigProducerRoot)configModelContext.getParentProducer()); + } + + public ContentCluster getCluster() { return cluster; } + + /** + * Returns indexing cluster implicitly created by this, + * or empty if an explicit cluster is used (or if called before the build phase) + */ + public Optional<ContainerCluster> ownedIndexingCluster() { return ownedIndexingCluster; } + + public void createTlds(ConfigModelRepo modelRepo) { + IndexedSearchCluster indexedCluster = cluster.getSearch().getIndexed(); + if (indexedCluster == null) { + return; + } + + SimpleConfigProducer tldParent = new SimpleConfigProducer(indexedCluster, "tlds"); + for (ConfigModel model : modelRepo.asMap().values()) { + if (!(model instanceof ContainerModel)) { + continue; + } + + ContainerCluster containerCluster = ((ContainerModel) model).getCluster(); + if (containerCluster.getSearch() == null) { + continue; // this is not a qrs cluster + } + + log.log(LogLevel.DEBUG, "Adding tlds for indexed cluster " + indexedCluster.getClusterName() + ", container cluster " + containerCluster.getName()); + indexedCluster.addTldsWithSameIdsAsContainers(tldParent, containerCluster); + } + indexedCluster.setupDispatchGroups(); + } + + /** Select/creates and initializes the indexing cluster coupled to this */ + private void initializeIndexingClusters(Collection<ContainerModel> containers, + ConfigModelRepoAdder configModelRepoAdder, + ApplicationConfigProducerRoot root) { + if (getCluster().getSearch().hasIndexedCluster()) + initializeOrSetExistingIndexingCluster(getCluster().getSearch().getIndexed(), hostedVespa, + containers, configModelRepoAdder, root); + } + + private void initializeOrSetExistingIndexingCluster(IndexedSearchCluster indexedSearchCluster, + boolean isHostedVespa, + Collection<ContainerModel> containers, + ConfigModelRepoAdder configModelRepoAdder, + ApplicationConfigProducerRoot root) { + if (indexedSearchCluster.hasExplicitIndexingCluster()) { + setExistingIndexingCluster(indexedSearchCluster, containers); + } else if (isHostedVespa) { + setContainerAsIndexingCluster(indexedSearchCluster, containers, configModelRepoAdder, root); + } else { + createImplicitIndexingCluster(indexedSearchCluster, configModelRepoAdder, root); + } + } + + private void setContainerAsIndexingCluster(IndexedSearchCluster indexedSearchCluster, + Collection<ContainerModel> containers, + ConfigModelRepoAdder configModelRepoAdder, + ApplicationConfigProducerRoot root) { + if (containers.isEmpty()) { + createImplicitIndexingCluster(indexedSearchCluster, configModelRepoAdder, root); + } else { + ContainerCluster targetCluster = getContainerWithDocproc(containers); + if (targetCluster == null) + targetCluster = getContainerWithSearch(containers); + if (targetCluster == null) + targetCluster = containers.iterator().next().getCluster(); + + addDocproc(targetCluster); + indexedSearchCluster.setIndexingClusterName(targetCluster.getName()); + addIndexingChainsTo(targetCluster, indexedSearchCluster); + } + } + + private void setExistingIndexingCluster(IndexedSearchCluster cluster, Collection<ContainerModel> containers) { + String indexingClusterName = cluster.getIndexingClusterName(); + ContainerModel containerModel = findByName(indexingClusterName, containers); + if (containerModel == null) + throw new RuntimeException("Content cluster '" + cluster.getClusterName() + "' refers to docproc " + + "cluster '" + indexingClusterName + "', but this cluster does not exist."); + addIndexingChainsTo(containerModel.getCluster(), cluster); + } + + private ContainerModel findByName(String name, Collection<ContainerModel> containers) { + for (ContainerModel container : containers) + if (container.getId().equals(name)) + return container; + return null; + } + + private void addIndexingChainsTo(ContainerCluster indexer, IndexedSearchCluster cluster) { + addIndexingChain(indexer); + DocprocChain indexingChain; + ComponentRegistry<DocprocChain> allChains = indexer.getDocprocChains().allChains(); + if (cluster.hasExplicitIndexingChain()) { + indexingChain = allChains.getComponent(cluster.getIndexingChainName()); + if (indexingChain == null) { + throw new RuntimeException("Indexing cluster " + cluster.getClusterName() + " refers to docproc " + + "chain " + cluster.getIndexingChainName() + " for indexing, which does not exist."); + } else { + checkThatExplicitIndexingChainInheritsCorrectly(allChains, indexingChain.getChainSpecification()); + } + } else { + indexingChain = allChains.getComponent(IndexingDocprocChain.NAME); + } + + cluster.setIndexingChain(indexingChain); + } + + private static boolean checkParentChain(ComponentRegistry<DocprocChain> allChains, ChainSpecification chainSpec) { + if (IndexingDocprocChain.NAME.equals(chainSpec.componentId.stringValue())) { + return true; + } + + ChainSpecification.Inheritance inheritance = chainSpec.inheritance; + for (ComponentSpecification parentComponentSpec : inheritance.chainSpecifications) { + ChainSpecification parentSpec = getChainSpec(allChains, parentComponentSpec); + checkParentChain(allChains, parentSpec); + } + + return false; + } + + private static ChainSpecification getChainSpec(ComponentRegistry<DocprocChain> allChains, ComponentSpecification componentSpec) { + DocprocChain docprocChain = allChains.getComponent(componentSpec); + if (docprocChain == null) { + throw new IllegalArgumentException("Chain '" + componentSpec + "' not found."); + } + return docprocChain.getChainSpecification(); + } + + private static void addIndexingChain(ContainerCluster containerCluster) { + DocprocChain chainAlreadyPresent = containerCluster.getDocprocChains().allChains(). + getComponent(new ComponentId(IndexingDocprocChain.NAME)); + if (chainAlreadyPresent != null) { + if (chainAlreadyPresent instanceof IndexingDocprocChain) { + return; + } else { + throw new IllegalArgumentException("A docproc chain may not have the ID '" + + IndexingDocprocChain.NAME + ", since this is reserved by Vespa. Please use a different ID."); + } + } + + containerCluster.getDocprocChains().add(new IndexingDocprocChain()); + } + + /** Create a new container cluster for indexing and add it to the Vespa model */ + private void createImplicitIndexingCluster(IndexedSearchCluster cluster, + ConfigModelRepoAdder configModelRepoAdder, + ApplicationConfigProducerRoot root) { + String indexerName = cluster.getIndexingClusterName(); + AbstractConfigProducer p = root.getChildren().get(ContainerModel.DOCPROC_RESERVED_NAME); + if (p == null) + p = new SimpleConfigProducer(root, ContainerModel.DOCPROC_RESERVED_NAME); + ConfigModelContext context = ConfigModelContext.createFromParentAndId(configModelRepoAdder, p, ContainerModel.DOCPROC_RESERVED_NAME); + ContainerCluster indexingCluster = new ContainerCluster(context.getParentProducer(), "cluster." + indexerName, indexerName); + ContainerModel indexingClusterModel = new ContainerModel(ConfigModelContext.createFromParentAndId(configModelRepoAdder, p, indexingCluster.getSubId())); + indexingClusterModel.setCluster(indexingCluster); + configModelRepoAdder.add(indexingClusterModel); + ownedIndexingCluster = Optional.of(indexingCluster); + + ContainerModelBuilder.addDefaultHandler_legacyBuilder(indexingCluster); + + addDocproc(indexingCluster); + + List<Container> nodes = new ArrayList<>(); + int index = 0; + Set<HostResource> processedHosts = new LinkedHashSet<>(); + boolean isElastic = cluster.isElastic(); + for (SearchNode searchNode : cluster.getSearchNodes()) { + HostResource host = searchNode.getHostResource(); + if (!processedHosts.contains(host)) { + String containerName = String.valueOf(isElastic ? searchNode.getDistributionKey() : index++); + Container docprocService = new Container(indexingCluster, containerName); + docprocService.setBasePort(host.nextAvailableBaseport(docprocService.getPortCount())); + docprocService.setHostResource(host); + docprocService.initService(); + nodes.add(docprocService); + processedHosts.add(host); + } + } + indexingCluster.addContainers(nodes); + + addIndexingChain(indexingCluster); + cluster.setIndexingChain(indexingCluster.getDocprocChains().allChains().getComponent(IndexingDocprocChain.NAME)); + } + + private void addDocproc(ContainerCluster cluster) { + if (cluster.getDocproc() == null) { + DocprocChains chains = new DocprocChains(cluster, "docprocchains"); + ContainerDocproc containerDocproc = new ContainerDocproc(cluster, chains); + cluster.setDocproc(containerDocproc); + } + } + + private ContainerCluster getContainerWithDocproc(Collection<ContainerModel> containers) { + for (ContainerModel container : containers) + if (container.getCluster().getDocproc() != null) + return container.getCluster(); + return null; + } + + private static ContainerCluster getContainerWithSearch(Collection<ContainerModel> containers) { + for (ContainerModel container : containers) + if (container.getCluster().getSearch() != null) + return container.getCluster(); + return null; + } + + private static void checkThatExplicitIndexingChainInheritsCorrectly(ComponentRegistry<DocprocChain> allChains, ChainSpecification chainSpec) { + ChainSpecification.Inheritance inheritance = chainSpec.inheritance; + boolean found = false; + for (ComponentSpecification componentSpec : inheritance.chainSpecifications) { + ChainSpecification parentSpec = getChainSpec(allChains, componentSpec); + found = checkParentChain(allChains, parentSpec); + if (found) { + break; + } + } + if (!found) { + throw new IllegalArgumentException("Docproc chain '" + chainSpec.componentId + "' does not inherit from 'indexing' chain."); + } + } + + public static List<Content> getContent(ConfigModelRepo pc) { + List<Content> contents = new ArrayList<>(); + + for (ConfigModel model : pc.asMap().values()) { + if (model instanceof Content) { + contents.add((Content)model); + } + } + + return contents; + } + + public static List<AbstractSearchCluster> getSearchClusters(ConfigModelRepo pc) { + List<AbstractSearchCluster> clusters = new ArrayList<>(); + + for (ContentCluster c : getContentClusters(pc)) { + clusters.addAll(c.getSearch().getClusters().values()); + } + + return clusters; + } + + public static List<ContentCluster> getContentClusters(ConfigModelRepo pc) { + List<ContentCluster> clusters = new ArrayList<>(); + + for (Content c : getContent(pc)) { + clusters.add(c.getCluster()); + } + + return clusters; + } + + @Override + public void prepare(ConfigModelRepo models) { + if (cluster.getRootGroup().useCpuSocketAffinity()) { + setCpuSocketAffinity(); + } + if (cluster.getRootGroup().getMmapNoCoreLimit().isPresent()) { + for (AbstractService s : cluster.getSearch().getSearchNodes()) { + s.setMMapNoCoreLimit(cluster.getRootGroup().getMmapNoCoreLimit().get()); + } + } + cluster.prepare(); + } + + private void setCpuSocketAffinity() { + // Currently only distribute affinity for search nodes + AbstractService.distributeCpuSocketAffinity(cluster.getSearch().getSearchNodes()); + } +} + diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java new file mode 100644 index 00000000000..77a90e8a515 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java @@ -0,0 +1,80 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.metrics.MetricsmanagerConfig; +import com.yahoo.vespa.config.content.LoadTypeConfig; +import com.yahoo.vespa.config.content.core.StorCommunicationmanagerConfig; +import com.yahoo.vespa.config.content.core.StorServerConfig; +import com.yahoo.vespa.config.content.core.StorStatusConfig; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.application.validation.RestartConfigs; + +/** + * Common class for config producers for storage and distributor nodes. + * + * TODO: Author + */ +@RestartConfigs({StorCommunicationmanagerConfig.class, StorStatusConfig.class, + StorServerConfig.class, LoadTypeConfig.class, MetricsmanagerConfig.class}) +public abstract class ContentNode extends AbstractService + implements StorCommunicationmanagerConfig.Producer, StorStatusConfig.Producer, StorServerConfig.Producer { + + protected int distributionKey; + String rootDirectory; + + public ContentNode(AbstractConfigProducer parent, String clusterName, String rootDirectory, int distributionKey) { + super(parent, "" + distributionKey); + this.distributionKey = distributionKey; + initialize(distributionKey); + + this.rootDirectory = rootDirectory; + + setProp("clustertype", "content"); + setProp("clustername", clusterName); + setProp("index", distributionKey); + } + + public int getDistributionKey() { + return distributionKey; + } + + @Override + public void getConfig(StorServerConfig.Builder builder) { + builder.root_folder(rootDirectory); + builder.node_index(distributionKey); + } + + public void initialize(int distributionKey) { + this.distributionKey = distributionKey; + portsMeta.on(0).tag("messaging"); + portsMeta.on(1).tag("rpc").tag("status"); + portsMeta.on(2).tag("http").tag("status").tag("state"); + + monitorService(); + } + + @Override + public int getPortCount() { return 3; } + + + @Override + public void getConfig(StorCommunicationmanagerConfig.Builder builder) { + builder.mbusport(getRelativePort(0)); + builder.rpcport(getRelativePort(1)); + } + + @Override + public void getConfig(StorStatusConfig.Builder builder) { + builder.httpport(getRelativePort(2)); + } + + @Override + public int getHealthPort() { + return getRelativePort(2); + } + + public String getRootDirectory() { + return rootDirectory; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearch.java new file mode 100644 index 00000000000..fdfb7f75188 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearch.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class ContentSearch { + + private final Double queryTimeout; + private final Double visibilityDelay; + + private ContentSearch(Builder builder) { + queryTimeout = builder.queryTimeout; + visibilityDelay = builder.visibilityDelay; + } + + public Double getQueryTimeout() { + return queryTimeout; + } + + public Double getVisibilityDelay() { + return visibilityDelay; + } + + public static class Builder { + + private Double queryTimeout; + private Double visibilityDelay; + + public ContentSearch build() { + return new ContentSearch(this); + } + + public Builder setQueryTimeout(Double queryTimeout) { + this.queryTimeout = queryTimeout; + return this; + } + + public Builder setVisibilityDelay(Double visibilityDelay) { + this.visibilityDelay = visibilityDelay; + return this; + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java new file mode 100644 index 00000000000..b5cbb23233a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -0,0 +1,299 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.vespa.config.search.DispatchConfig; +import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.documentmodel.DocumentTypeRepo; +import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.builder.UserConfigBuilder; +import com.yahoo.vespa.model.builder.xml.dom.DomSearchTuningBuilder; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.content.cluster.DomResourceLimitsBuilder; +import com.yahoo.vespa.model.search.*; +import org.w3c.dom.Element; + + +import java.util.*; + +/** + * Encapsulates the various options for search in a content model. + * Wraps a search cluster from com.yahoo.vespa.model.search. + */ +public class ContentSearchCluster extends AbstractConfigProducer implements ProtonConfig.Producer, DispatchConfig.Producer { + + private final boolean flushOnShutdown; + + /** If this is set up for streaming search, it is modelled as one search cluster per search definition */ + private Map<String, AbstractSearchCluster> clusters = new TreeMap<>(); + + /** The single, indexed search cluster this sets up (supporting multiple document types), or null if none */ + private IndexedSearchCluster indexedCluster; + + private final String clusterName; + Map<String, NewDocumentType> documentDefinitions; + + /** The search nodes of this if it does not have an indexed cluster */ + List<SearchNode> nonIndexed = new ArrayList<>(); + + Map<StorageGroup, NodeSpec> groupToSpecMap = new LinkedHashMap<>(); + private DocumentTypeRepo repo = null; + private Optional<ResourceLimits> resourceLimits = Optional.empty(); + + public void prepare() { + repo = getRoot().getDeployState().getDocumentModel().getDocumentManager(); + } + + public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<ContentSearchCluster> { + + private final Map<String, NewDocumentType> documentDefinitions; + + public Builder(Map<String, NewDocumentType> documentDefinitions) { + this.documentDefinitions = documentDefinitions; + } + + @Override + protected ContentSearchCluster doBuild(AbstractConfigProducer ancestor, Element producerSpec) { + ModelElement clusterElem = new ModelElement(producerSpec); + String clusterName = ContentCluster.getClusterName(clusterElem); + Boolean flushOnShutdown = clusterElem.childAsBoolean("engine.proton.flush-on-shutdown"); + + ContentSearchCluster search = new ContentSearchCluster(ancestor, clusterName, documentDefinitions, flushOnShutdown != null ? flushOnShutdown : false); + + ModelElement tuning = clusterElem.getChildByPath("engine.proton.tuning"); + if (tuning != null) { + search.setTuning(new DomSearchTuningBuilder().build(search, tuning.getXml())); + } + ModelElement protonElem = clusterElem.getChildByPath("engine.proton"); + if (protonElem != null) { + search.setResourceLimits(DomResourceLimitsBuilder.build(protonElem)); + } + + buildAllStreamingSearchClusters(clusterElem, clusterName, search); + buildIndexedSearchCluster(clusterElem, clusterName, search); + return search; + } + + private Double getQueryTimeout(ModelElement clusterElem) { + return clusterElem.childAsDouble("engine.proton.query-timeout"); + } + + private void buildAllStreamingSearchClusters(ModelElement clusterElem, String clusterName, ContentSearchCluster search) { + ModelElement docElem = clusterElem.getChild("documents"); + + if (docElem == null) { + return; + } + + for (ModelElement docType : docElem.subElements("document")) { + String mode = docType.getStringAttribute("mode"); + if ("streaming".equals(mode)) { + buildStreamingSearchCluster(clusterElem, clusterName, search, docType); + } + } + } + + private void buildStreamingSearchCluster(ModelElement clusterElem, String clusterName, ContentSearchCluster search, ModelElement docType) { + StreamingSearchCluster cluster = new StreamingSearchCluster(search, clusterName + "." + docType.getStringAttribute("type"), 0, clusterName, clusterName); + + List<ModelElement> def = new ArrayList<>(); + def.add(docType); + search.addSearchCluster(cluster, getQueryTimeout(clusterElem), def); + } + + private void buildIndexedSearchCluster(ModelElement clusterElem, + String clusterName, + ContentSearchCluster search) { + List<ModelElement> indexedDefs = getIndexedSearchDefinitions(clusterElem); + if (!indexedDefs.isEmpty()) { + IndexedSearchCluster isc = new IndexedElasticSearchCluster(search, clusterName, 0); + isc.setRoutingSelector(clusterElem.childAsString("documents.selection")); + + Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay"); + if (visibilityDelay != null) { + isc.setVisibilityDelay(visibilityDelay); + } + + search.addSearchCluster(isc, getQueryTimeout(clusterElem), indexedDefs); + } + } + + private List<ModelElement> getIndexedSearchDefinitions(ModelElement clusterElem) { + List<ModelElement> indexedDefs = new ArrayList<>(); + ModelElement docElem = clusterElem.getChild("documents"); + if (docElem == null) { + return indexedDefs; + } + + for (ModelElement docType : docElem.subElements("document")) { + String mode = docType.getStringAttribute("mode"); + if ("index".equals(mode)) { + indexedDefs.add(docType); + } + } + return indexedDefs; + } + } + + private ContentSearchCluster(AbstractConfigProducer parent, + String clusterName, + Map<String, NewDocumentType> documentDefinitions, boolean flushOnShutdown) + { + super(parent, "search"); + this.clusterName = clusterName; + this.documentDefinitions = documentDefinitions; + this.flushOnShutdown = flushOnShutdown; + } + + void addSearchCluster(SearchCluster cluster, Double queryTimeout, List<ModelElement> documentDefs) { + addSearchDefinitions(documentDefs, cluster); + + if (queryTimeout != null) { + cluster.setQueryTimeout(queryTimeout); + } + cluster.defaultDocumentsConfig(); + cluster.deriveSearchDefinitions(new ArrayList<>()); + addCluster(cluster); + } + + private void addSearchDefinitions(List<ModelElement> searchDefs, AbstractSearchCluster sc) { + for (ModelElement e : searchDefs) { + SearchDefinitionXMLHandler searchDefinitionXMLHandler = new SearchDefinitionXMLHandler(e); + SearchDefinition searchDefinition = + searchDefinitionXMLHandler.getResponsibleSearchDefinition(sc.getRoot().getDeployState().getSearchDefinitions()); + if (searchDefinition == null) + throw new RuntimeException("Search definition parsing error or file does not exist: '" + + searchDefinitionXMLHandler.getName() + "'"); + + // TODO: remove explicit building of user configs when the complete content model is built using builders. + sc.getLocalSDS().add(new AbstractSearchCluster.SearchDefinitionSpec(searchDefinition, + UserConfigBuilder.build(e.getXml(), sc.getRoot().getDeployState(), sc.getRoot().deployLogger()))); + //need to get the document names from this sdfile + sc.addDocumentNames(searchDefinition); + } + } + + public ContentSearchCluster addCluster(AbstractSearchCluster sc) { + if (clusters.containsKey(sc.getClusterName())) { + throw new IllegalArgumentException("I already have registered cluster '" + sc.getClusterName() + "'"); + } + if (sc instanceof IndexedSearchCluster) { + if (indexedCluster != null) { + throw new IllegalArgumentException("I already have one indexed cluster named '" + indexedCluster.getClusterName()); + } + indexedCluster = (IndexedSearchCluster)sc; + } + clusters.put(sc.getClusterName(), sc); + return this; + } + + public List<SearchNode> getSearchNodes() { + return hasIndexedCluster() ? getIndexed().getSearchNodes() : nonIndexed; + } + + public SearchNode addSearchNode(ContentNode node, StorageGroup parentGroup, ModelElement element) { + AbstractConfigProducer parent = hasIndexedCluster() ? getIndexed() : this; + + NodeSpec spec = getNextSearchNodeSpec(parentGroup); + SearchNode snode; + TransactionLogServer tls; + if (element == null) { + snode = SearchNode.create(parent, "" + node.getDistributionKey(), node.getDistributionKey(), spec, clusterName, node, flushOnShutdown); + snode.setHostResource(node.getHostResource()); + snode.initService(); + + tls = new TransactionLogServer(snode, clusterName); + tls.setHostResource(snode.getHostResource()); + tls.initService(); + } else { + snode = new SearchNode.Builder(""+node.getDistributionKey(), spec, clusterName, node, flushOnShutdown).build(parent, element.getXml()); + tls = new TransactionLogServer.Builder(clusterName).build(snode, element.getXml()); + } + snode.setTls(tls); + if (hasIndexedCluster()) { + getIndexed().addSearcher(snode); + } else { + nonIndexed.add(snode); + } + return snode; + } + + private NodeSpec getNextSearchNodeSpec(StorageGroup parentGroup) { + NodeSpec spec = groupToSpecMap.get(parentGroup); + if (spec == null) { + spec = new NodeSpec(groupToSpecMap.size(), 0); + } else { + spec = new NodeSpec(spec.rowId(), spec.partitionId() + 1); + } + groupToSpecMap.put(parentGroup, spec); + return spec; + } + + private Tuning tuning; + + public void setTuning(Tuning t) { + tuning = t; + } + + public void setResourceLimits(ResourceLimits resourceLimits) { + this.resourceLimits = Optional.of(resourceLimits); + } + + public boolean usesHierarchicDistribution() { + return indexedCluster != null && groupToSpecMap.size() > 1; + } + + public void handleRedundancy(Redundancy redundancy) { + if (usesHierarchicDistribution()) { + indexedCluster.setMaxNodesDownPerFixedRow((redundancy.effectiveFinalRedundancy() / groupToSpecMap.size()) - 1); + } + } + + @Override + public void getConfig(ProtonConfig.Builder builder) { + double visibilityDelay = hasIndexedCluster() ? getIndexed().getVisibilityDelay() : 0.0; + for (NewDocumentType type : documentDefinitions.values()) { + ProtonConfig.Documentdb.Builder ddbB = new ProtonConfig.Documentdb.Builder(); + ddbB.inputdoctypename(type.getFullName().getName()) + .configid(getConfigId()) + .visibilitydelay(visibilityDelay); + if (hasIndexedCluster()) { + getIndexed().fillDocumentDBConfig(type.getFullName().getName(), ddbB); + } + builder.documentdb(ddbB); + } + for (AbstractSearchCluster sc : getClusters().values()) { + if (sc instanceof StreamingSearchCluster) { + NewDocumentType type = repo.getDocumentType(((StreamingSearchCluster)sc).getSdConfig().getSearch().getName()); + ProtonConfig.Documentdb.Builder ddbB = new ProtonConfig.Documentdb.Builder(); + ddbB.inputdoctypename(type.getFullName().getName()).configid(getConfigId()); + builder.documentdb(ddbB); + } + } + int numDocumentDbs = builder.documentdb.size(); + builder.initialize(new ProtonConfig.Initialize.Builder().threads(numDocumentDbs + 1)); + + if (resourceLimits.isPresent()) { + resourceLimits.get().getConfig(builder); + } + + if (tuning != null) { + tuning.getConfig(builder); + } + } + + @Override + public void getConfig(DispatchConfig.Builder builder) { + if (hasIndexedCluster()) { + getIndexed().getConfig(builder); + } + } + + public Map<String, AbstractSearchCluster> getClusters() { return clusters; } + public IndexedSearchCluster getIndexed() { return indexedCluster; } + public boolean hasIndexedCluster() { return indexedCluster != null; } + public String getClusterName() { return clusterName; } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchSpec.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchSpec.java new file mode 100644 index 00000000000..ad69cced45b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchSpec.java @@ -0,0 +1,83 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import java.util.ArrayList; +import java.util.List; + +/** + * Represents the dispatch setup for a content cluster. + * This EITHER has a number of dispatch groups OR a an explicit list of groups, + * for unknown reasons (talk to Geir). + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class DispatchSpec { + + private final Integer numDispatchGroups; + private final List<Group> groups; + + private DispatchSpec(Builder builder) { + numDispatchGroups = builder.numDispatchGroups; + groups = builder.groups; + } + + public Integer getNumDispatchGroups() { return numDispatchGroups; } + + public List<Group> getGroups() { + return groups; + } + + public boolean valid() { + return numDispatchGroups != null || groups != null; + } + + /** + * Reference to a node which is contained in a dispatch group. + */ + public static class Node { + private final int distributionKey; + public Node(int distributionKey) { + this.distributionKey = distributionKey; + } + public int getDistributionKey() { + return distributionKey; + } + } + + /** + * A dispatch group with a list of nodes contained in that group. + */ + public static class Group { + private final List<Node> nodes = new ArrayList<>(); + public Group() { + + } + public Group addNode(Node node) { + nodes.add(node); + return this; + } + public List<Node> getNodes() { + return nodes; + } + } + + public static class Builder { + + private Integer numDispatchGroups; + private List<Group> groups; + + public DispatchSpec build() { + return new DispatchSpec(this); + } + + public Builder setNumDispatchGroups(Integer numDispatchGroups) { + this.numDispatchGroups = numDispatchGroups; + return this; + } + + public Builder setGroups(List<Group> groups) { + this.groups = groups; + return this; + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributionBitCalculator.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributionBitCalculator.java new file mode 100644 index 00000000000..b6462d5aa9b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributionBitCalculator.java @@ -0,0 +1,79 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.vespa.model.content.cluster.ContentCluster; + +public class DistributionBitCalculator { + + public static int getDistributionBits(int nodes, ContentCluster.DistributionMode mode) { + if (mode == ContentCluster.DistributionMode.STRICT) { + if (nodes < 5) { + return 8; // Require few buckets in small clusters to ease testing + } else if (nodes < 15) { + return 16; + } else if (nodes < 200) { + return 21; + } else if (nodes < 800) { + return 25; + } else if (nodes < 1500) { + return 28; + } else if (nodes < 5000) { + return 30; + } else { + return 32; + } + } else if (mode == ContentCluster.DistributionMode.LOOSE) { + if (nodes < 5) { + return 8; + } else if (nodes < 200) { + return 16; + } else { + return 24; + } + } else if (mode == ContentCluster.DistributionMode.LEGACY) { + if (nodes == 1) { // min 1 bucket/node + return 1; + } else if (nodes == 2) { // min 128 buckets/node + return 8; + } else if (nodes <= 6) { // min 5462 buckets/node + return 14; + } else if (nodes <= 8) { // min 9362 buckets/node + return 16; + } else if (nodes <= 10) { // min 14563 buckets/node + return 17; + } else if (nodes <= 12) { // min 23832 buckets/node + return 18; + } else if (nodes <= 20) { // min 40329 buckets/node + return 19; + } else if (nodes <= 32) { // min 49933 buckets/node + return 20; + } else if (nodes <= 64) { // min 63550 buckets/node + return 21; + } else if (nodes <= 100) { // min 64528 buckets/node + return 22; + } else if (nodes <= 256) { // min 83056 buckets/node + return 23; + } else if (nodes <= 350) { // min 65281 buckets/node + return 24; + } else if (nodes <= 500) { // min 95597 buckets/node + return 25; + } else if (nodes <= 1024) { // min 133950 buckets/node + return 26; + } else if (nodes <= 2048) { // min 130944 buckets/node + return 27; + } else if (nodes <= 4096) { // min 130008 buckets/node + return 28; + } else if (nodes <= 8192) { // min 131040 buckets/node + return 29; + } else if (nodes <= 16384) { // min 131056 buckets/node + return 30; + } else if (nodes <= 32768) { // min 131064 buckets/node + return 31; + } else { // min 131068 buckets/node + return 32; + } + } else { + throw new IllegalArgumentException("We don't know how to handle mode " + mode); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java new file mode 100644 index 00000000000..5458bd366d5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Distributor.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.vespa.config.content.core.StorServerConfig; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.content.engines.PersistenceEngine; +import org.w3c.dom.Element; + +/** + * Represents specific configuration for a given distributor node. + */ +public class Distributor extends ContentNode { + + PersistenceEngine provider; + + public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<Distributor> { + ModelElement clusterXml; + PersistenceEngine persistenceProvider; + + public Builder(ModelElement clusterXml, PersistenceEngine persistenceProvider) { + this.clusterXml = clusterXml; + this.persistenceProvider = persistenceProvider; + } + + @Override + protected Distributor doBuild(AbstractConfigProducer ancestor, Element producerSpec) { + return new Distributor((DistributorCluster)ancestor, new ModelElement(producerSpec).getIntegerAttribute("distribution-key"), + clusterXml.getIntegerAttribute("distributor-base-port"), persistenceProvider); + } + } + + Distributor(DistributorCluster parent, int distributionKey, Integer distributorBasePort, PersistenceEngine provider) { + super(parent, parent.getClusterName(), StorageNode.rootFolder + parent.getClusterName() + "/distributor/" + + distributionKey, distributionKey); + + this.provider = provider; + + if (distributorBasePort != null) { + setBasePort(distributorBasePort); + } + } + + @Override + public void getConfig(StorServerConfig.Builder builder) { + super.getConfig(builder); + provider.getConfig(builder); + } + + @Override + public String getStartupCommand() { + return "exec sbin/distributord -c $VESPA_CONFIG_ID"; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java new file mode 100644 index 00000000000..dc27da7461d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -0,0 +1,151 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; +import com.yahoo.vespa.config.content.core.StorServerConfig; +import com.yahoo.document.select.DocumentSelector; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.metrics.MetricsmanagerConfig; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import org.w3c.dom.Element; + +import java.util.logging.Logger; + + +/** + * Generates distributor-specific configuration. + */ +public class DistributorCluster extends AbstractConfigProducer<Distributor> + implements StorDistributormanagerConfig.Producer, StorServerConfig.Producer, MetricsmanagerConfig.Producer { + + public static final Logger log = Logger.getLogger(DistributorCluster.class.getPackage().toString()); + + private static class GcOptions { + public final int interval; + public final String selection; + + public GcOptions(int interval, String selection) { + this.interval = interval; + this.selection = selection; + } + } + + private final ContentCluster parent; + private final BucketSplitting bucketSplitting; + private final GcOptions gc; + private final boolean hasIndexedDocumentType; + + public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<DistributorCluster> { + ContentCluster parent; + + public Builder(ContentCluster parent) { + this.parent = parent; + } + + private String prepareGCSelection(ModelElement documentNode, String selStr) throws ParseException { + DocumentSelector s = new DocumentSelector(selStr); + boolean enableGC = false; + if (documentNode != null) { + enableGC = documentNode.getBooleanAttribute("garbage-collection", false); + } + if (!enableGC) { + return null; + } + + return s.toString(); + } + + private int getGCInterval(ModelElement documentNode) throws ParseException { + int gcInterval = 3600; + if (documentNode != null) { + gcInterval = documentNode.getIntegerAttribute("garbage-collection-interval", gcInterval); + } + return gcInterval; + } + + private GcOptions parseGcOptions(ModelElement documentNode) { + String gcSelection = parent.getRoutingSelector(); + int gcInterval; + try { + if (gcSelection != null) { + gcSelection = prepareGCSelection(documentNode, gcSelection); + } + gcInterval = getGCInterval(documentNode); + } catch (ParseException e) { + throw new IllegalArgumentException("Failed to parse garbage collection selection", e); + } + return new GcOptions(gcInterval, gcSelection); + } + + private boolean documentModeImpliesIndexing(String mode) { + return "index".equals(mode); + } + + private boolean clusterContainsIndexedDocumentType(ModelElement documentsNode) { + // The presence of at least one <document> and its mode attribute is schema-enforced. + return documentsNode.subElements("document").stream() + .anyMatch(node -> documentModeImpliesIndexing(node.getStringAttribute("mode"))); + } + + @Override + protected DistributorCluster doBuild(AbstractConfigProducer ancestor, Element producerSpec) { + final ModelElement clusterElement = new ModelElement(producerSpec); + final ModelElement documentsNode = clusterElement.getChild("documents"); + final GcOptions gc = parseGcOptions(documentsNode); + final boolean hasIndexedDocumentType = clusterContainsIndexedDocumentType(documentsNode); + + return new DistributorCluster(parent, + new BucketSplitting.Builder().build( + parent, new ModelElement(producerSpec)), gc, hasIndexedDocumentType); + } + } + + private DistributorCluster(ContentCluster parent, BucketSplitting bucketSplitting, + GcOptions gc, boolean hasIndexedDocumentType) + { + super(parent, "distributor"); + this.parent = parent; + this.bucketSplitting = bucketSplitting; + this.gc = gc; + this.hasIndexedDocumentType = hasIndexedDocumentType; + } + + @Override + public void getConfig(StorDistributormanagerConfig.Builder builder) { + if (gc.selection != null) { + builder.garbagecollection(new StorDistributormanagerConfig.Garbagecollection.Builder() + .selectiontoremove("not (" + gc.selection + ")") + .interval(gc.interval)); + } + builder.enable_revert(parent.getPersistence().supportRevert()); + builder.disable_bucket_activation(hasIndexedDocumentType == false); + + bucketSplitting.getConfig(builder); + } + + @Override + public void getConfig(MetricsmanagerConfig.Builder builder) { + ContentCluster.getMetricBuilder("log", builder). + addedmetrics("vds.distributor.docsstored"). + addedmetrics("vds.distributor.bytesstored"). + addedmetrics("vds.idealstate.delete_bucket.done_ok"). + addedmetrics("vds.idealstate.merge_bucket.done_ok"). + addedmetrics("vds.idealstate.split_bucket.done_ok"). + addedmetrics("vds.idealstate.join_bucket.done_ok"). + addedmetrics("vds.idealstate.buckets_rechecking"); + } + + @Override + public void getConfig(StorServerConfig.Builder builder) { + builder.root_folder(""); + builder.cluster_name(parent.getName()); + builder.is_distributor(true); + } + + public String getClusterName() { + return parent.getName(); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DocumentTypeVisitor.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DocumentTypeVisitor.java new file mode 100644 index 00000000000..585ae691773 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DocumentTypeVisitor.java @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.document.select.Visitor; +import com.yahoo.document.select.rule.*; + +public abstract class DocumentTypeVisitor implements Visitor { + @Override + public void visit(ArithmeticNode arithmeticNode) { + for (ArithmeticNode.NodeItem item : arithmeticNode.getItems()) { + item.getNode().accept(this); + } + } + + @Override + public void visit(AttributeNode attributeNode) { + attributeNode.getValue().accept(this); + } + + @Override + public void visit(ComparisonNode comparisonNode) { + comparisonNode.getLHS().accept(this); + comparisonNode.getRHS().accept(this); + } + + @Override + public void visit(EmbracedNode embracedNode) { + embracedNode.getNode().accept(this); + } + + @Override + public void visit(IdNode idNode) { + } + + @Override + public void visit(LiteralNode literalNode) { + } + + @Override + public void visit(LogicNode logicNode) { + for (LogicNode.NodeItem item : logicNode.getItems()) { + item.getNode().accept(this); + } + } + + @Override + public void visit(NegationNode negationNode) { + negationNode.getNode().accept(this); + } + + @Override + public void visit(NowNode nowNode) { + } + + @Override + public void visit(SearchColumnNode searchColumnNode) { + } + + @Override + public void visit(VariableNode variableNode) { + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java new file mode 100644 index 00000000000..861088a80f5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionValidator.java @@ -0,0 +1,117 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * Class used to validate that hierarchic distribution is correctly setup when having an indexed content cluster. + * + * Note that this class does not implement the com.yahoo.vespa.model.application.validation.Validator interface, + * but is instead used in the context of com.yahoo.vespa.model.ConfigProducer.validate() such that it can be unit tested + * without having to build the complete vespa model. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class IndexedHierarchicDistributionValidator { + + private final String clusterName; + private final StorageGroup rootGroup; + private final Redundancy redundancy; + private final TuningDispatch.DispatchPolicy dispatchPolicy; + + public IndexedHierarchicDistributionValidator(String clusterName, + StorageGroup rootGroup, + Redundancy redundancy, + TuningDispatch.DispatchPolicy dispatchPolicy) { + this.clusterName = clusterName; + this.rootGroup = rootGroup; + this.redundancy = redundancy; + this.dispatchPolicy = dispatchPolicy; + } + + public void validate() throws Exception { + validateThatWeHaveOneGroupLevel(); + validateThatLeafGroupsHasEqualNumberOfNodes(); + validateThatLeafGroupsCountIsAFactorOfRedundancy(); + validateThatRedundancyPerGroupIsEqual(); + validateThatReadyCopiesIsCompatibleWithRedundancy(rootGroup.getSubgroups().size()); + } + + private void validateThatWeHaveOneGroupLevel() { + for (StorageGroup group : rootGroup.getSubgroups()) { + if (group.getSubgroups().size() > 0) { + throw new IllegalArgumentException(getErrorMsgPrefix() + "Expected all groups under root group '" + + rootGroup.getName() + "' to be leaf groups only containing nodes, but sub group '" + group.getName() + "' contains " + + group.getSubgroups().size() + " sub groups."); + } + } + } + + private void validateThatLeafGroupsHasEqualNumberOfNodes() { + if (dispatchPolicy != TuningDispatch.DispatchPolicy.ROUNDROBIN) return; + + StorageGroup previousGroup = null; + for (StorageGroup group : rootGroup.getSubgroups()) { + if (previousGroup == null) { // first group + previousGroup = group; + continue; + } + + if (group.getNodes().size() != previousGroup.getNodes().size()) + throw new IllegalArgumentException(getErrorMsgPrefix() + "Expected leaf groups to contain an equal number of nodes, but leaf group '" + + previousGroup.getName() + "' contains " + previousGroup.getNodes().size() + " node(s) while leaf group '" + + group.getName() + "' contains " + group.getNodes().size() + " node(s)."); + previousGroup = group; + } + } + + private void validateThatLeafGroupsCountIsAFactorOfRedundancy() { + if (redundancy.effectiveFinalRedundancy() % rootGroup.getSubgroups().size() != 0) { + throw new IllegalArgumentException(getErrorMsgPrefix() + "Expected number of leaf groups (" + + rootGroup.getSubgroups().size() + ") to be a factor of redundancy (" + + redundancy.effectiveFinalRedundancy() + "), but it is not."); + } + } + + private void validateThatRedundancyPerGroupIsEqual() { + int redundancyPerGroup = redundancy.effectiveFinalRedundancy() / rootGroup.getSubgroups().size(); + String expPartitions = createDistributionPartitions(redundancyPerGroup, rootGroup.getSubgroups().size()); + if (!rootGroup.getPartitions().get().equals(expPartitions)) { + throw new IllegalArgumentException(getErrorMsgPrefix() + "Expected redundancy per leaf group to be " + + redundancyPerGroup + ", but it is not according to distribution partitions '" + + rootGroup.getPartitions().get() + "'. Expected distribution partitions should be '" + expPartitions + "'."); + } + } + + private List<StorageNode> nonRetired(List<StorageNode> nodes) { + return nodes.stream().filter((node) -> { return !node.isRetired(); } ).collect(Collectors.toList()); + } + + private String createDistributionPartitions(int redundancyPerGroup, int numGroups) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < numGroups - 1; ++i) { + sb.append(redundancyPerGroup); + sb.append("|"); + } + sb.append("*"); + return sb.toString(); + } + + private void validateThatReadyCopiesIsCompatibleWithRedundancy(int groupCount) throws Exception { + if (redundancy.effectiveFinalRedundancy() % groupCount != 0) { + throw new Exception(getErrorMsgPrefix() + "Expected equal redundancy per group."); + } + if (redundancy.effectiveReadyCopies() % groupCount != 0) { + throw new Exception(getErrorMsgPrefix() + "Expected equal amount of ready copies per group, but " + + redundancy.effectiveReadyCopies() + " ready copies is specified with " + groupCount + " groups"); + } + if (redundancy.effectiveReadyCopies() == 0) { + System.err.println(getErrorMsgPrefix() + "Warning. No ready copies configured. At least one is recommended."); + } + } + + private String getErrorMsgPrefix() { + return "In indexed content cluster '" + clusterName + "' using hierarchic distribution: "; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/PriorityMapping.java b/config-model/src/main/java/com/yahoo/vespa/model/content/PriorityMapping.java new file mode 100644 index 00000000000..597949ec9f2 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/PriorityMapping.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; + +import java.util.HashMap; +import java.util.Map; + +/** + * Created with IntelliJ IDEA. + * User: thomasg + * Date: 5/7/12 + * Time: 2:00 PM + * To change this template use File | Settings | File Templates. + */ +public class PriorityMapping { + ModelElement clusterXml; + Map<DocumentProtocol.Priority, Integer> priorityMappings = new HashMap<>(); + + public PriorityMapping(ModelElement clusterXml) { + this.clusterXml = clusterXml; + + int val = 50; + for (DocumentProtocol.Priority p : DocumentProtocol.Priority.values()) { + priorityMappings.put(p, val); + val += 10; + } + priorityMappings.put(DocumentProtocol.Priority.HIGHEST, 0); + priorityMappings.put(DocumentProtocol.Priority.LOWEST, 255); + } + + public int getPriorityMapping(String priorityName) { + return priorityMappings.get(Enum.valueOf(DocumentProtocol.Priority.class, priorityName)); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java new file mode 100644 index 00000000000..262c985e733 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Redundancy.java @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.vespa.config.content.StorDistributionConfig; + +/** + * Configuration of the redundancy of a content cluster. + * + * @author bratseth + */ +public class Redundancy implements StorDistributionConfig.Producer { + + private final int initialRedundancy ; + private final int finalRedundancy; + private final int readyCopies; + + private int implicitGroups = 1; + + /** The total number of nodes available in this cluster (assigned when this becomes known) */ + private int totalNodes = 0; + + public Redundancy(int initialRedundancy, int finalRedundancy, int readyCopies) { + this.initialRedundancy = initialRedundancy; + this.finalRedundancy = finalRedundancy; + this.readyCopies = readyCopies; + } + + /** + * Set the total number of nodes available in this cluster. + * This impacts the effective redundancy in the case where there are fewer nodes available than + * the requested redundancy. + */ + public void setTotalNodes(int totalNodes) { this.totalNodes = totalNodes; } + + /** + * Sets the number of groups resulting from implicit setup (groups attribute) + * in this cluster. With implicit groups the redundancy settings are taken to be + * <i>per group</i> and are multiplied by this number to get the effective <i>total</i> + * values returned in the config. + */ + public void setImplicitGroups(int implicitGroups) { this.implicitGroups = implicitGroups; } + + public int initialRedundancy() { return initialRedundancy; } + public int finalRedundancy() { return finalRedundancy; } + public int readyCopies() { return readyCopies; } + + public int effectiveInitialRedundancy() { return Math.min(totalNodes, initialRedundancy * implicitGroups); } + public int effectiveFinalRedundancy() { return Math.min(totalNodes, finalRedundancy * implicitGroups); } + public int effectiveReadyCopies() { return Math.min(totalNodes, readyCopies * implicitGroups); } + + @Override + public void getConfig(StorDistributionConfig.Builder builder) { + builder.initial_redundancy(effectiveInitialRedundancy()); + builder.redundancy(effectiveFinalRedundancy()); + builder.ready_copies(effectiveReadyCopies()); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java new file mode 100644 index 00000000000..bb16d542c20 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java @@ -0,0 +1,55 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.defaults.Defaults; + +import java.util.Optional; + +/** + * Class tracking resource limits for a content cluster with engine proton. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class ResourceLimits implements ProtonConfig.Producer { + + private final Optional<Double> diskLimit; + private final Optional<Double> memoryLimit; + + private ResourceLimits(Builder builder) { + this.diskLimit = builder.diskLimit; + this.memoryLimit = builder.memoryLimit; + } + + @Override + public void getConfig(ProtonConfig.Builder builder) { + ProtonConfig.Writefilter.Builder writeFilterBuilder = new ProtonConfig.Writefilter.Builder(); + if (diskLimit.isPresent()) { + writeFilterBuilder.disklimit(diskLimit.get()); + } + if (memoryLimit.isPresent()) { + writeFilterBuilder.memorylimit(memoryLimit.get()); + } + builder.writefilter(writeFilterBuilder); + } + + public static class Builder { + + private Optional<Double> diskLimit = Optional.empty(); + private Optional<Double> memoryLimit = Optional.empty(); + + public ResourceLimits build() { + return new ResourceLimits(this); + } + + public Builder setDiskLimit(double diskLimit) { + this.diskLimit = Optional.of(diskLimit); + return this; + } + + public Builder setMemoryLimit(double memoryLimit) { + this.memoryLimit = Optional.of(memoryLimit); + return this; + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/SearchCoverage.java b/config-model/src/main/java/com/yahoo/vespa/model/content/SearchCoverage.java new file mode 100644 index 00000000000..4c93711ebd3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/SearchCoverage.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.google.common.base.Preconditions; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class SearchCoverage { + + private final Double minimum; + private final Double minWaitAfterCoverageFactor; + private final Double maxWaitAfterCoverageFactor; + + private SearchCoverage(Builder builder) { + minimum = builder.minimum; + minWaitAfterCoverageFactor = builder.minWaitAfterCoverageFactor; + maxWaitAfterCoverageFactor = builder.maxWaitAfterCoverageFactor; + } + + public Double getMinimum() { + return minimum; + } + + public Double getMinWaitAfterCoverageFactor() { + return minWaitAfterCoverageFactor; + } + + public Double getMaxWaitAfterCoverageFactor() { + return maxWaitAfterCoverageFactor; + } + + public static class Builder { + + private Double minimum; + private Double minWaitAfterCoverageFactor; + private Double maxWaitAfterCoverageFactor; + + public SearchCoverage build() { + return new SearchCoverage(this); + } + + public Builder setMinimum(Double value) { + Preconditions.checkArgument(value == null || (value >= 0 && value <= 1), + "Expected value in range [0, 1], got " + value + "."); + minimum = value; + return this; + } + + public Builder setMinWaitAfterCoverageFactor(Double value) { + Preconditions.checkArgument(value == null || (value >= 0 && value <= 1), + "Expected value in range [0, 1], got " + value + "."); + Preconditions.checkArgument(value == null || maxWaitAfterCoverageFactor == null || + value <= maxWaitAfterCoverageFactor, + "Minimum wait (got %s) must be no larger than maximum wait (was %s).", + value, maxWaitAfterCoverageFactor); + minWaitAfterCoverageFactor = value; + return this; + } + + public Builder setMaxWaitAfterCoverageFactor(Double value) { + Preconditions.checkArgument(value == null || (value >= 0 && value <= 1), + "Expected value in range [0, 1], got " + value + "."); + Preconditions.checkArgument(value == null || minWaitAfterCoverageFactor == null || + value >= minWaitAfterCoverageFactor, + "Maximum wait (got %s) must be no smaller than minimum wait (was %s).", + value, minWaitAfterCoverageFactor); + maxWaitAfterCoverageFactor = value; + return this; + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java new file mode 100644 index 00000000000..78255ce8f64 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -0,0 +1,420 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.content.StorDistributionConfig; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.builder.xml.dom.NodesSpecification; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.content.engines.PersistenceEngine; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * A group of storage nodes/distributors. + * + * @author unknown, probably thomasg + * @author bratseth has done things here recently + */ +public class StorageGroup { + + private final boolean useCpuSocketAffinity; + private final String index; + private Optional<String> partitions; + String name; + private final ContentCluster owner; + private final Optional<Long> mmapNoCoreLimit; + + private final List<StorageGroup> subgroups = new ArrayList<>(); + private final List<StorageNode> nodes = new ArrayList<>(); + + /** + * Creates a storage group + * + * @param owner the cluster this group belongs to + * @param name the name of this group + * @param index the distribution-key index og this group + * @param partitions the distribution strategy to use to distribute content to subgroups or empty + * (meaning that the "*" distribution will be used) only if this is a leaf group + * (having nodes, not subgroups as children). + * @param useCpuSocketAffinity whether processes should be started with socket affinity + */ + private StorageGroup(ContentCluster owner, String name, String index, Optional<String> partitions, + boolean useCpuSocketAffinity, Optional<Long> mmapNoCoreLimit) + { + this.owner = owner; + this.index = index; + this.name = name; + this.partitions = partitions; + this.useCpuSocketAffinity = useCpuSocketAffinity; + this.mmapNoCoreLimit = mmapNoCoreLimit; + } + + /** Returns the name of this group, or null if it is the root group */ + public String getName() { return name; } + + /** Returns the subgroups of this, or an empty list if it is a leaf group */ + public List<StorageGroup> getSubgroups() { return subgroups; } + + /** Returns the nodes of this, or an empty list of it is not a leaf group */ + public List<StorageNode> getNodes() { return nodes; } + + public ContentCluster getOwner() { return owner; } + + /** Returns the index of this group, or null if it is the root group */ + public String getIndex() { return index; } + + public Optional<String> getPartitions() { return partitions; } + public boolean useCpuSocketAffinity() { return useCpuSocketAffinity; } + public Optional<Long> getMmapNoCoreLimit() { return mmapNoCoreLimit; } + + /** Returns all the nodes below this group */ + public List<StorageNode> recursiveGetNodes() { + if ( ! nodes.isEmpty()) return nodes; + List<StorageNode> nodes = new ArrayList<>(); + for (StorageGroup subgroup : subgroups) + nodes.addAll(subgroup.recursiveGetNodes()); + return nodes; + } + + public Collection<StorDistributionConfig.Group.Builder> getGroupStructureConfig() { + List<StorDistributionConfig.Group.Builder> groups = new ArrayList<>(); + + StorDistributionConfig.Group.Builder myGroup = new StorDistributionConfig.Group.Builder(); + getConfig(myGroup); + groups.add(myGroup); + + for (StorageGroup g : subgroups) { + groups.addAll(g.getGroupStructureConfig()); + } + + return groups; + } + + public void getConfig(StorDistributionConfig.Group.Builder builder) { + builder.index(index == null ? "invalid" : index); + builder.name(name == null ? "invalid" : name); + if (partitions.isPresent()) + builder.partitions(partitions.get()); + for (StorageNode node : nodes) { + StorDistributionConfig.Group.Nodes.Builder nb = new StorDistributionConfig.Group.Nodes.Builder(); + nb.index(node.getDistributionKey()); + nb.retired(node.isRetired()); + builder.nodes.add(nb); + } + builder.capacity(getCapacity()); + } + + public int getNumberOfLeafGroups() { + int count = subgroups.isEmpty() ? 1 : 0; + for (StorageGroup g : subgroups) { + count += g.getNumberOfLeafGroups(); + } + return count; + } + + public double getCapacity() { + double capacity = 0; + for (StorageNode node : nodes) { + capacity += node.getCapacity(); + } + for (StorageGroup group : subgroups) { + capacity += group.getCapacity(); + } + return capacity; + } + + /** Returns the total number of nodes below this group */ + public int countNodes() { + int nodeCount = nodes.size(); + for (StorageGroup group : subgroups) + nodeCount += group.countNodes(); + return nodeCount; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof StorageGroup) { + StorageGroup rhs = (StorageGroup)obj; + return this.index.equals(rhs.index) && + this.name.equals(rhs.name) && + this.partitions.equals(rhs.partitions); + } + return false; + } + + public static class Builder { + + private final ModelElement clusterElement; + private final ContentCluster owner; + private final DeployLogger deployLogger; + + public Builder(ModelElement clusterElement, ContentCluster owner, DeployLogger deployLogger) { + this.clusterElement = clusterElement; + this.owner = owner; + this.deployLogger = deployLogger; + } + + public StorageGroup buildRootGroup() { + Optional<ModelElement> group = Optional.ofNullable(clusterElement.getChild("group")); + Optional<ModelElement> nodes = getNodes(clusterElement); + + if (group.isPresent() && nodes.isPresent()) + throw new IllegalStateException("Both group and nodes exists, only one of these tags is legal"); + if (group.isPresent() && (group.get().getStringAttribute("name") != null || group.get().getIntegerAttribute("distribution-key") != null)) + owner.deployLogger().log(LogLevel.INFO, "'distribution-key' attribute on a content cluster's root group is ignored"); + + GroupBuilder groupBuilder = collectGroup(group, nodes, null, null); + if (owner.isHostedVespa()) { + return groupBuilder.buildHosted(owner, Optional.empty()); + } else { + return groupBuilder.buildNonHosted(owner, Optional.empty()); + } + } + + /** + * Represents a storage group and can build storage nodes in both hosted and non-hosted environments. + */ + private static class GroupBuilder { + + private final StorageGroup storageGroup; + + /* The explicitly defined subgroups of this */ + private final List<GroupBuilder> subGroups; + private final List<XmlNodeBuilder> nodeBuilders; + + /** The nodes explicitly specified as a nodes tag in this group, or empty if none */ + private final Optional<NodesSpecification> nodeRequirement; + + private final DeployLogger deployLogger; + + private GroupBuilder(StorageGroup storageGroup, List<GroupBuilder> subGroups, List<XmlNodeBuilder> nodeBuilders, + Optional<NodesSpecification> nodeRequirement, DeployLogger deployLogger) { + this.storageGroup = storageGroup; + this.subGroups = subGroups; + this.nodeBuilders = nodeBuilders; + this.nodeRequirement = nodeRequirement; + this.deployLogger = deployLogger; + } + + /** + * Builds a storage group for a nonhosted environment + * + * @param owner the cluster owning this + * @param parent the parent storage group, or empty if this is the root group + * @return the storage group build by this + */ + public StorageGroup buildNonHosted(ContentCluster owner, Optional<GroupBuilder> parent) { + for (GroupBuilder subGroup : subGroups) { + storageGroup.subgroups.add(subGroup.buildNonHosted(owner, Optional.of(this))); + } + for (XmlNodeBuilder nodeBuilder : nodeBuilders) { + storageGroup.nodes.add(nodeBuilder.build(owner, storageGroup)); + } + + if ( ! parent.isPresent()) + owner.redundancy().setTotalNodes(storageGroup.countNodes()); + + return storageGroup; + } + + /** + * Builds a storage group for a hosted environment + * + * @param owner the cluster owning this + * @param parent the parent storage group, or empty if this is the root group + * @return the storage group build by this + */ + public StorageGroup buildHosted(ContentCluster owner, Optional<GroupBuilder> parent) { + Map<HostResource, ClusterMembership> hostMapping = nodeRequirement.isPresent() ? allocateHosts(owner) : Collections.emptyMap(); + + Map<Optional<ClusterSpec.Group>, Map<HostResource, ClusterMembership>> hostGroups = collectAllocatedSubgroups(hostMapping); + if (hostGroups.size() > 1) { + if (parent.isPresent()) + throw new IllegalArgumentException("Cannot specify groups using the groups attribute in nested content groups"); + owner.redundancy().setTotalNodes(hostMapping.size()); + + // Switch redundancy settings to meaning "per group" + owner.redundancy().setImplicitGroups(hostGroups.size()); + + // Compute partitions expression + int redundancyPerGroup = (int)Math.floor(owner.redundancy().effectiveFinalRedundancy() / hostGroups.size()); + storageGroup.partitions = Optional.of(computePartitions(redundancyPerGroup, hostGroups.size())); + + // create subgroups as returned from allocation + for (Map.Entry<Optional<ClusterSpec.Group>, Map<HostResource, ClusterMembership>> hostGroup : hostGroups.entrySet()) { + String groupIndex = hostGroup.getKey().get().value(); + StorageGroup subgroup = new StorageGroup(owner, groupIndex, groupIndex, Optional.empty(), false, Optional.empty()); + for (Map.Entry<HostResource, ClusterMembership> host : hostGroup.getValue().entrySet()) { + subgroup.nodes.add(createStorageNode(owner, host.getKey(), subgroup, host.getValue())); + } + storageGroup.subgroups.add(subgroup); + } + } + else { // or otherwise just create the nodes directly on this group, or the explicitly enumerated subgroups + for (Map.Entry<HostResource, ClusterMembership> host : hostMapping.entrySet()) { + storageGroup.nodes.add(createStorageNode(owner, host.getKey(), storageGroup, host.getValue())); + } + for (GroupBuilder subGroup : subGroups) { + storageGroup.subgroups.add(subGroup.buildHosted(owner, Optional.of(this))); + } + if ( ! parent.isPresent()) + owner.redundancy().setTotalNodes(storageGroup.countNodes()); + } + return storageGroup; + } + + /** This returns a partition string which specifies equal distribution between all groups */ + // TODO: Make a partitions object + private String computePartitions(int redundancyPerGroup, int numGroups) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < numGroups - 1; ++i) { + sb.append(redundancyPerGroup); + sb.append("|"); + } + sb.append("*"); + return sb.toString(); + } + + private Map<HostResource, ClusterMembership> allocateHosts(ContentCluster parent) { + ClusterSpec.Id clusterId = ClusterSpec.Id.from(parent.getStorageNodes().getClusterName()); + Optional<ClusterSpec.Group> groupId = storageGroup.getIndex() == null ? Optional.empty() : Optional.of(ClusterSpec.Group.from(storageGroup.getIndex())); + return nodeRequirement.get().provision(parent.getRoot().getHostSystem(), ClusterSpec.Type.content, clusterId, groupId, deployLogger); + } + + /** Collect hosts per group */ + private Map<Optional<ClusterSpec.Group>, Map<HostResource, ClusterMembership>> collectAllocatedSubgroups(Map<HostResource, ClusterMembership> hostMapping) { + Map<Optional<ClusterSpec.Group>, Map<HostResource, ClusterMembership>> hostsPerGroup = new LinkedHashMap<>(); + for (Map.Entry<HostResource, ClusterMembership> entry : hostMapping.entrySet()) { + Optional<ClusterSpec.Group> group = entry.getValue().cluster().group(); + Map<HostResource, ClusterMembership> hostsInGroup = hostsPerGroup.get(group); + if (hostsInGroup == null) { + hostsInGroup = new LinkedHashMap<>(); + hostsPerGroup.put(group, hostsInGroup); + } + hostsInGroup.put(entry.getKey(), entry.getValue()); + } + return hostsPerGroup; + } + + } + + private static class XmlNodeBuilder { + private final ModelElement clusterElement; + private final ModelElement element; + + private XmlNodeBuilder(ModelElement clusterElement, ModelElement element) { + this.clusterElement = clusterElement; + this.element = element; + } + + public StorageNode build(ContentCluster parent, StorageGroup storageGroup) { + StorageNode sNode = new StorageNode.Builder().build(parent.getStorageNodes(), element.getXml()); + PersistenceEngine provider = parent.getPersistence().create(sNode, storageGroup, element); + new Distributor.Builder(clusterElement, provider).build(parent.getDistributorNodes(), element.getXml()); + return sNode; + } + } + + /** + * Creates a content group builder from a group and/or nodes element. + * These are the possibilities: + * <ul> + * <li>group and nodes is present: This is a leaf group specifying a set of nodes</li> + * <li>only group is present: This is a nonleaf group</li> + * <li>only nodes is present: This is the implicitly specified toplevel leaf group, or a set of groups + * specified using a group count attrbute. + * </ul> + */ + private GroupBuilder collectGroup(Optional<ModelElement> groupElement, Optional<ModelElement> nodesElement, String name, String index) { + StorageGroup group = new StorageGroup(owner, name, index, + childAsString(groupElement, "distribution.partitions"), + booleanAttributeOr(groupElement, VespaDomBuilder.CPU_SOCKET_AFFINITY_ATTRIB_NAME, false), + childAsLong(groupElement, VespaDomBuilder.MMAP_NOCORE_LIMIT)); + + List<GroupBuilder> subGroups = groupElement.isPresent() ? collectSubGroups(group, groupElement.get()) : Collections.emptyList(); + + List<XmlNodeBuilder> explicitNodes = new ArrayList<>(); + explicitNodes.addAll(collectExplicitNodes(groupElement)); + explicitNodes.addAll(collectExplicitNodes(nodesElement)); + + if (subGroups.size() > 0 && explicitNodes.size() > 0) + throw new IllegalArgumentException("A group can contain either nodes or groups, but not both."); + + Optional<NodesSpecification> nodeRequirement = + nodesElement.isPresent() && nodesElement.get().getStringAttribute("count") != null ? Optional.of(NodesSpecification.from(nodesElement.get())) : Optional.empty(); + if (nodeRequirement.isPresent() && subGroups.size() > 0) + throw new IllegalArgumentException("A group can contain either explicit subgroups or a nodes specification, but not both."); + return new GroupBuilder(group, subGroups, explicitNodes, nodeRequirement, deployLogger); + } + + private Optional<String> childAsString(Optional<ModelElement> element, String childTagName) { + if ( ! element.isPresent()) return Optional.empty(); + return Optional.ofNullable(element.get().childAsString(childTagName)); + } + private Optional<Long> childAsLong(Optional<ModelElement> element, String childTagName) { + if ( ! element.isPresent()) return Optional.empty(); + return Optional.ofNullable(element.get().childAsLong(childTagName)); + } + + private boolean booleanAttributeOr(Optional<ModelElement> element, String attributeName, boolean defaultValue) { + if ( ! element.isPresent()) return defaultValue; + return element.get().getBooleanAttribute(attributeName, defaultValue); + } + + private Optional<ModelElement> getNodes(ModelElement groupOrNodesElement) { + if (groupOrNodesElement.getXml().getTagName().equals("nodes")) return Optional.of(groupOrNodesElement); + return Optional.ofNullable(groupOrNodesElement.getChild("nodes")); + } + + private List<XmlNodeBuilder> collectExplicitNodes(Optional<ModelElement> groupOrNodesElement) { + if ( ! groupOrNodesElement.isPresent()) return Collections.emptyList(); + List<XmlNodeBuilder> nodes = new ArrayList<>(); + for (ModelElement n : groupOrNodesElement.get().subElements("node")) + nodes.add(new XmlNodeBuilder(clusterElement, n)); + return nodes; + } + + private List<GroupBuilder> collectSubGroups(StorageGroup parentGroup, ModelElement parentGroupElement) { + List<ModelElement> subGroupElements = parentGroupElement.subElements("group"); + if (subGroupElements.size() > 1 && ! parentGroup.getPartitions().isPresent()) + throw new IllegalArgumentException("'distribution' attribute is required with multiple subgroups"); + + List<GroupBuilder> subGroups = new ArrayList<>(); + String indexPrefix = ""; + if (parentGroup.index != null) { + indexPrefix = parentGroup.index + "."; + } + for (ModelElement g : subGroupElements) { + subGroups.add(collectGroup(Optional.of(g), Optional.ofNullable(g.getChild("nodes")), g.getStringAttribute("name"), + indexPrefix + g.getIntegerAttribute("distribution-key"))); + } + return subGroups; + } + + private static StorageNode createStorageNode(ContentCluster parent, HostResource hostResource, StorageGroup parentGroup, ClusterMembership clusterMembership) { + StorageNode sNode = new StorageNode(parent.getStorageNodes(), null, clusterMembership.index(), clusterMembership.retired()); + sNode.setHostResource(hostResource); + sNode.initService(); + + // TODO: Supplying null as XML is not very nice + PersistenceEngine provider = parent.getPersistence().create(sNode, parentGroup, null); + Distributor d = new Distributor(parent.getDistributorNodes(), clusterMembership.index(), null, provider); + d.setHostResource(sNode.getHostResource()); + d.initService(); + return sNode; + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java new file mode 100644 index 00000000000..750a66a0dda --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageNode.java @@ -0,0 +1,116 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.vespa.config.content.StorFilestorConfig; +import com.yahoo.vespa.config.content.core.StorBucketmoverConfig; +import com.yahoo.vespa.config.storage.StorDevicesConfig; +import com.yahoo.vespa.config.content.core.StorServerConfig; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.config.storage.StorMemfilepersistenceConfig; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.application.validation.RestartConfigs; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.content.engines.PersistenceEngine; +import com.yahoo.vespa.model.content.engines.ProtonProvider; +import com.yahoo.vespa.model.content.storagecluster.StorageCluster; +import org.w3c.dom.Element; + +import java.util.Arrays; + +/** + * Class to provide config related to a specific storage node. + */ +@RestartConfigs({StorDevicesConfig.class, StorFilestorConfig.class, + StorMemfilepersistenceConfig.class, StorBucketmoverConfig.class}) +public class StorageNode extends ContentNode implements StorServerConfig.Producer, StorDevicesConfig.Producer { + + static final String rootFolder = Defaults.getDefaults().vespaHome() + "var/db/vespa/vds/"; + + private final Double capacity; + private final boolean retired; + private final boolean isHostedVespa; + private boolean usesVdsEngine = false; + + public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<StorageNode> { + @Override + protected StorageNode doBuild(AbstractConfigProducer ancestor, Element producerSpec) { + ModelElement e = new ModelElement(producerSpec); + return new StorageNode((StorageCluster)ancestor, e.getDoubleAttribute("capacity"), e.getIntegerAttribute("distribution-key"), false); + } + } + + StorageNode(StorageCluster cluster, Double capacity, int distributionKey, boolean retired) { + super(cluster, + cluster.getClusterName(), + rootFolder + cluster.getClusterName() + "/storage/" + distributionKey, + distributionKey); + this.retired = retired; + this.capacity = capacity; + this.isHostedVespa = cluster.getRoot().getDeployState().getProperties().hostedVespa(); + } + + @Override + public String getStartupCommand() { + return isProviderProton() + ? null + : "exec sbin/storaged -c $VESPA_CONFIG_ID"; + } + + @Override + public void getConfig(StorDevicesConfig.Builder builder) { + String root_folder = getRootDirectory(); + builder.root_folder(root_folder); + + // For VDS in hosted Vespa, we default to using the root_folder as the disk to store the data in. + // Setting disk_path will then + if (isHostedVespa && usesVdsEngine) { + // VDS looks up the first disk at the directory path root_folder/disks/d0. + builder.disk_path(Arrays.asList(root_folder + "/disks/d0")); + } + } + + // 2015-08-11: Needed because of the following circular dependency: + // 1. StorageNode is created. + // 2. A particular persistence engine is picked depending on things (like the presence of engine/vds element) + // that are hidden from the code creating the StorageNode in (1). + // 3. The persistence engine depends on the StorageNode, e.g. it's a parent node. + // + // If the VDSEngine is picked in (2), we would like to know this in StorageNode::getConfig(). Hence this setter. + public void useVdsEngine() { + usesVdsEngine = true; + } + + + public double getCapacity() { + if (capacity != null) { + return capacity; + } else { + return 1.0; + } + } + + /** Whether this node is configured as retired, which means all content should migrate off the node */ + public boolean isRetired() { return retired; } + + private boolean isProviderProton() { + for (AbstractConfigProducer producer : getChildren().values()) { + if (producer instanceof ProtonProvider) { + return true; + } + } + return false; + } + + @Override + public void getConfig(StorServerConfig.Builder builder) { + super.getConfig(builder); + + builder.node_capacity(getCapacity()); + + for (AbstractConfigProducer producer : getChildren().values()) { + ((PersistenceEngine)producer).getConfig(builder); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/TuningDispatch.java b/config-model/src/main/java/com/yahoo/vespa/model/content/TuningDispatch.java new file mode 100644 index 00000000000..614be479d38 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/TuningDispatch.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class TuningDispatch { + + private final Integer maxHitsPerPartition; + public static enum DispatchPolicy { ROUNDROBIN, RANDOM}; + private final DispatchPolicy dispatchPolicy; + private final Boolean useLocalNode; + private final Double minGroupCoverage; + private final Double minActiveDocsCoverage; + + private TuningDispatch(Builder builder) + { + maxHitsPerPartition = builder.maxHitsPerPartition; + dispatchPolicy = builder.dispatchPolicy; + useLocalNode = builder.useLocalNode; + minGroupCoverage = builder.minGroupCoverage; + minActiveDocsCoverage = builder.minActiveDocsCoverage; + } + + public Integer getMaxHitsPerPartition() { + return maxHitsPerPartition; + } + public DispatchPolicy getDispatchPolicy() { return dispatchPolicy; } + public Boolean getUseLocalNode() { return useLocalNode; } + public Double getMinGroupCoverage() { return minGroupCoverage; } + public Double getMinActiveDocsCoverage() { return minActiveDocsCoverage; } + + public static class Builder { + + private Integer maxHitsPerPartition; + private DispatchPolicy dispatchPolicy = DispatchPolicy.ROUNDROBIN; + private Boolean useLocalNode; + private Double minGroupCoverage; + private Double minActiveDocsCoverage; + + public TuningDispatch build() { + return new TuningDispatch(this); + } + + public Builder setMaxHitsPerPartition(Integer maxHitsPerPartition) { + this.maxHitsPerPartition = maxHitsPerPartition; + return this; + } + public Builder setDispatchPolicy(String policy) { + if (policy == null) { + } else if ("round-robin".equals(policy.toLowerCase())) { + dispatchPolicy = DispatchPolicy.ROUNDROBIN; + } else { + dispatchPolicy = DispatchPolicy.valueOf(policy.toUpperCase()); + } + return this; + } + + public Builder setUseLocalNode(Boolean useLocalNode) { + this.useLocalNode = useLocalNode; + return this; + } + public Builder setMinGroupCoverage(Double minGroupCoverage) { + this.minGroupCoverage = minGroupCoverage; + return this; + } + public Builder setMinActiveDocsCoverage(Double minCoverage) { + this.minActiveDocsCoverage = minCoverage; + return this; + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java new file mode 100644 index 00000000000..08685659d35 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -0,0 +1,604 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.cluster; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Sets; +import com.yahoo.config.model.ConfigModelUtils; +import com.yahoo.config.model.producer.AbstractConfigProducerRoot; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.vespa.config.content.MessagetyperouteselectorpolicyConfig; +import com.yahoo.vespa.config.content.FleetcontrollerConfig; +import com.yahoo.vespa.config.content.StorDistributionConfig; +import com.yahoo.vespa.config.content.core.StorDistributormanagerConfig; +import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.metrics.MetricsmanagerConfig; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.admin.Admin; +import com.yahoo.vespa.model.admin.Metric; +import com.yahoo.vespa.model.admin.MetricsConsumer; +import com.yahoo.vespa.model.admin.MonitoringSystem; +import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerCluster; +import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerComponent; +import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerConfigurer; +import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.builder.xml.dom.NodesSpecification; +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; +import com.yahoo.vespa.model.content.*; +import com.yahoo.vespa.model.content.engines.PersistenceEngine; +import com.yahoo.vespa.model.content.engines.ProtonEngine; +import com.yahoo.vespa.model.content.engines.VDSEngine; +import com.yahoo.vespa.model.content.storagecluster.StorageCluster; +import com.yahoo.vespa.model.search.IndexedSearchCluster; +import com.yahoo.vespa.model.search.MultilevelDispatchValidator; +import com.yahoo.vespa.model.search.Tuning; +import org.w3c.dom.Element; + +import java.util.*; +import java.util.logging.Level; +import java.util.stream.Collectors; + +/** + * A content cluster. + * + * @author mostly somebody unknown + * @author bratseth + */ +public class ContentCluster extends AbstractConfigProducer implements StorDistributionConfig.Producer, + StorDistributormanagerConfig.Producer, + FleetcontrollerConfig.Producer, + MetricsmanagerConfig.Producer, + MessagetyperouteselectorpolicyConfig.Producer { + + // TODO: Make private + private String documentSelection; + ContentSearchCluster search; + final Map<String, NewDocumentType> documentDefinitions; + com.yahoo.vespa.model.content.StorageGroup rootGroup; + StorageCluster storageNodes; + DistributorCluster distributorNodes; + private Redundancy redundancy; + ClusterControllerConfig clusterControllerConfig; + PersistenceEngine.PersistenceFactory persistenceFactory; + String clusterName; + Integer maxNodesPerMerge; + + /** + * If multitenant or a cluster controller was explicitly configured in this cluster: + * The cluster controller cluster of this particular content cluster. + * + * Otherwise: null - the cluster controller is shared by all content clusters and part of Admin. + */ + private ContainerCluster clusterControllers; + + public enum DistributionMode { LEGACY, STRICT, LOOSE } + DistributionMode distributionMode; + + public static class Builder { + + /** The admin model of this system or null if none (which only happens in tests) */ + private final Admin admin; + private final DeployLogger deployLogger; + + public Builder(Admin admin, DeployLogger deployLogger) { + this.admin = admin; + this.deployLogger = deployLogger; + } + + public ContentCluster build(AbstractConfigProducer ancestor, Element w3cContentElement) { + ModelElement contentElement = new ModelElement(w3cContentElement); + + ModelElement documentsElement = contentElement.getChild("documents"); + Map<String, NewDocumentType> documentDefinitions = + new SearchDefinitionBuilder().build(ancestor.getRoot().getDeployState().getDocumentModel().getDocumentManager(), documentsElement); + + String routingSelection = new DocumentSelectionBuilder().build(contentElement.getChild("documents")); + Redundancy redundancy = new RedundancyBuilder().build(contentElement); + + ContentCluster c = new ContentCluster(ancestor, getClusterName(contentElement), documentDefinitions, + routingSelection, redundancy); + c.clusterControllerConfig = new ClusterControllerConfig.Builder(getClusterName(contentElement), contentElement).build(c, contentElement.getXml()); + c.search = new ContentSearchCluster.Builder(documentDefinitions).build(c, contentElement.getXml()); + c.persistenceFactory = new EngineFactoryBuilder().build(contentElement, c); + c.storageNodes = new StorageCluster.Builder().build(c, w3cContentElement); + c.distributorNodes = new DistributorCluster.Builder(c).build(c, w3cContentElement); + c.rootGroup = new StorageGroup.Builder(contentElement, c, deployLogger).buildRootGroup(); + validateThatGroupSiblingsAreUnique(c.clusterName, c.rootGroup); + c.search.handleRedundancy(redundancy); + + IndexedSearchCluster index = c.search.getIndexed(); + if (index != null) { + setupIndexedCluster(index, contentElement); + } + + if (c.search.hasIndexedCluster() && !(c.persistenceFactory instanceof ProtonEngine.Factory) ) { + throw new RuntimeException("If you have indexed search you need to have proton as engine"); + } + + if (documentsElement != null) { + ModelElement e = documentsElement.getChild("document-processing"); + if (e != null) { + setupDocumentProcessing(c, e); + } + } else if (c.persistenceFactory != null) { + throw new IllegalArgumentException("The specified content engine requires the <documents> element to be specified."); + } + + ModelElement tuning = contentElement.getChild("tuning"); + if (tuning != null) { + setupTuning(c, tuning); + } + + AbstractConfigProducerRoot root = ancestor.getRoot(); + if (root == null) return c; + + addClusterControllers(root, c.rootGroup, contentElement, c.clusterName, c); + return c; + } + + private void setupIndexedCluster(IndexedSearchCluster index, ModelElement element) { + ContentSearch search = DomContentSearchBuilder.build(element); + Double queryTimeout = search.getQueryTimeout(); + if (queryTimeout != null) { + Preconditions.checkState(index.getQueryTimeout() == null, + "You may not specify query-timeout in both proton and content."); + index.setQueryTimeout(queryTimeout); + } + Double visibilityDelay = search.getVisibilityDelay(); + if (visibilityDelay != null) { + index.setVisibilityDelay(visibilityDelay); + } + index.setSearchCoverage(DomSearchCoverageBuilder.build(element)); + index.setDispatchSpec(DomDispatchBuilder.build(element)); + if (index.useMultilevelDispatchSetup()) { + // We must validate this before we add tlds and setup the dispatch groups. + // This must therefore happen before the regular validate() step. + new MultilevelDispatchValidator(index.getClusterName(), index.getDispatchSpec(), index.getSearchNodes()).validate(); + } + + // TODO: This should be cleaned up to avoid having to change code in 100 places + // every time we add a dispatch option. + TuningDispatch tuningDispatch = DomTuningDispatchBuilder.build(element); + Integer maxHitsPerPartition = tuningDispatch.getMaxHitsPerPartition(); + Boolean useLocalNode = tuningDispatch.getUseLocalNode(); + + if (index.getTuning() == null) { + index.setTuning(new Tuning(index)); + } + if (index.getTuning().dispatch == null) { + index.getTuning().dispatch = new Tuning.Dispatch(); + } + if (maxHitsPerPartition != null) { + index.getTuning().dispatch.maxHitsPerPartition = maxHitsPerPartition; + } + if (useLocalNode != null) { + index.getTuning().dispatch.useLocalNode = useLocalNode; + } + index.getTuning().dispatch.minGroupCoverage = tuningDispatch.getMinGroupCoverage(); + index.getTuning().dispatch.minActiveDocsCoverage = tuningDispatch.getMinActiveDocsCoverage(); + index.getTuning().dispatch.policy = tuningDispatch.getDispatchPolicy(); + } + + private void setupDocumentProcessing(ContentCluster c, ModelElement e) { + String docprocCluster = e.getStringAttribute("cluster"); + if (docprocCluster != null) { + docprocCluster = docprocCluster.trim(); + } + if (c.getSearch().hasIndexedCluster()) { + if (docprocCluster != null && !docprocCluster.isEmpty()) { + c.getSearch().getIndexed().setIndexingClusterName(docprocCluster); + } + } + + String docprocChain = e.getStringAttribute("chain"); + if (docprocChain != null) { + docprocChain = docprocChain.trim(); + } + if (c.getSearch().hasIndexedCluster()) { + if (docprocChain != null && !docprocChain.isEmpty()) { + c.getSearch().getIndexed().setIndexingChainName(docprocChain); + } + } + } + + private void setupTuning(ContentCluster c, ModelElement tuning) { + ModelElement distribution = tuning.getChild("distribution"); + if (distribution != null) { + String attr = distribution.getStringAttribute("type"); + if (attr != null) { + if (attr.toLowerCase().equals("strict")) { + c.distributionMode = DistributionMode.STRICT; + } else if (attr.toLowerCase().equals("loose")) { + c.distributionMode = DistributionMode.LOOSE; + } else if (attr.toLowerCase().equals("legacy")) { + c.distributionMode = DistributionMode.LEGACY; + } else { + throw new IllegalStateException("Distribution type " + attr + " not supported."); + } + } + } + ModelElement merges = tuning.getChild("merges"); + if (merges != null) { + Integer attr = merges.getIntegerAttribute("max-nodes-per-merge"); + if (attr != null) { + c.maxNodesPerMerge = attr; + } + } + } + + private void validateGroupSiblings(String cluster, StorageGroup group) { + HashSet<String> siblings = new HashSet<>(); + for (StorageGroup g : group.getSubgroups()) { + String name = g.getName(); + if (siblings.contains(name)) { + throw new IllegalArgumentException("Cluster '" + cluster + "' has multiple groups " + + "with name '" + name + "' in the same subgroup. Group sibling names must be unique."); + } + siblings.add(name); + } + } + + private void validateThatGroupSiblingsAreUnique(String cluster, StorageGroup group) { + if (group == null) { + return; // Unit testing case + } + validateGroupSiblings(cluster, group); + for (StorageGroup g : group.getSubgroups()) { + validateThatGroupSiblingsAreUnique(cluster, g); + } + } + + private void addClusterControllers(AbstractConfigProducerRoot root, StorageGroup rootGroup, ModelElement contentElement, String contentClusterName, ContentCluster contentCluster) { + if (admin == null) return; // only in tests + if (contentCluster.getPersistence() == null) return; + + ContainerCluster clusterControllers; + + ContentCluster overlappingCluster = findOverlappingCluster(root, contentCluster); + if (overlappingCluster != null && overlappingCluster.getClusterControllers() != null) { + // Borrow the cluster controllers of the other cluster in this case. + // This condition only obtains on non-hosted systems with a shared config server, + // a combination which only exists in system tests + clusterControllers = overlappingCluster.getClusterControllers(); + } + else if (admin.multitenant()) { + String clusterName = contentClusterName + "-controllers"; + NodesSpecification nodesSpecification = + NodesSpecification.optionalDedicatedFromParent(contentElement.getChild("controllers")).orElse(NodesSpecification.nonDedicated(3)); + Collection<HostResource> hosts = nodesSpecification.isDedicated() ? + getControllerHosts(nodesSpecification, admin, clusterName) : + drawContentHosts(nodesSpecification.count(), rootGroup); + + clusterControllers = createClusterControllers(new ClusterControllerCluster(contentCluster, "standalone"), hosts, clusterName, true); + contentCluster.clusterControllers = clusterControllers; + } + else { + clusterControllers = admin.getClusterControllers(); + if (clusterControllers == null) { + List<HostResource> hosts = admin.getClusterControllerHosts(); + if (hosts.size() > 1) { + admin.deployLogger().log(Level.INFO, "When having content cluster(s) and more than 1 config server it is recommended to configure cluster controllers explicitly." + + " See " + ConfigModelUtils.createDocLink("reference/services-admin.html#cluster-controller")); + } + clusterControllers = createClusterControllers(admin, hosts, "cluster-controllers", false); + admin.setClusterControllers(clusterControllers); + } + } + + addClusterControllerComponentsForThisCluster(clusterControllers, contentCluster); + } + + /** Returns any other content cluster which shares nodes with this, or null if none are built */ + private ContentCluster findOverlappingCluster(AbstractConfigProducerRoot root, ContentCluster contentCluster) { + for (ContentCluster otherContentCluster : root.getChildrenByTypeRecursive(ContentCluster.class)) { + if (otherContentCluster != contentCluster && overlaps(contentCluster, otherContentCluster)) + return otherContentCluster; + } + return null; + } + + private boolean overlaps(ContentCluster c1, ContentCluster c2) { + Set<HostResource> c1Hosts = c1.getRootGroup().recursiveGetNodes().stream().map(StorageNode::getHostResource).collect(Collectors.toSet()); + Set<HostResource> c2Hosts = c2.getRootGroup().recursiveGetNodes().stream().map(StorageNode::getHostResource).collect(Collectors.toSet()); + return ! Sets.intersection(c1Hosts, c2Hosts).isEmpty(); + } + + private Collection<HostResource> getControllerHosts(NodesSpecification nodesSpecification, Admin admin, String clusterName) { + return nodesSpecification.provision(admin.getHostSystem(), ClusterSpec.Type.admin, ClusterSpec.Id.from(clusterName), Optional.empty(), deployLogger).keySet(); + } + + private List<HostResource> drawContentHosts(int count, StorageGroup rootGroup) { + List<HostResource> hosts = drawContentHostsRecursively(count, rootGroup); + if (hosts.size() % 2 == 0) // ZK clusters of even sizes are less available (even in the size=2 case) + hosts = hosts.subList(0, hosts.size()-1); + return hosts; + } + + /** + * Draw <code>count</code> nodes from as many different content groups below this as possible. + * This will only achieve maximum spread in the case where the groups are balanced and never on the same + * physical node. It will not achieve maximum spread over all levels in a multilevel group hierarchy. + */ + // Note: This method cannot be changed to draw different nodes without ensuring that it will draw nodes + // which overlaps with previously drawn nodes as this will prevent rolling upgrade + private List<HostResource> drawContentHostsRecursively(int count, StorageGroup group) { + Set<HostResource> hosts = new HashSet<>(); + if (group.getNodes().isEmpty()) { + int hostsPerSubgroup = (int)Math.ceil((double)count / group.getSubgroups().size()); + for (StorageGroup subgroup : group.getSubgroups()) + hosts.addAll(drawContentHostsRecursively(hostsPerSubgroup, subgroup)); + } + else { + hosts.addAll(group.getNodes().stream() + .filter(node -> ! node.isRetired()) // Avoid retired controllers to avoid surprises on expiry + .map(StorageNode::getHostResource).collect(Collectors.toList())); + } + List<HostResource> sortedHosts = new ArrayList<>(hosts); + Collections.sort(sortedHosts); + sortedHosts = sortedHosts.subList(0, Math.min(count, hosts.size())); + return sortedHosts; + } + + private ContainerCluster createClusterControllers(AbstractConfigProducer parent, Collection<HostResource> hosts, String name, boolean multitenant) { + ContainerCluster clusterControllers = new ContainerCluster(parent, name, name); + List<Container> containers = new ArrayList<>(); + // Add a cluster controller on each config server (there is always at least one). + if (clusterControllers.getContainers().isEmpty()) { + int index = 0; + for (HostResource host : hosts) { + ClusterControllerContainer clusterControllerContainer = new ClusterControllerContainer(clusterControllers, index, multitenant); + clusterControllerContainer.setHostResource(host); + clusterControllerContainer.initService(); + clusterControllerContainer.setProp("clustertype", "admin") + .setProp("clustername", clusterControllers.getName()) + .setProp("index", String.valueOf(index)); + containers.add(clusterControllerContainer); + ++index; + } + } + clusterControllers.addContainers(containers); + ContainerModelBuilder.addDefaultHandler_legacyBuilder(clusterControllers); + return clusterControllers; + } + + private void addClusterControllerComponentsForThisCluster(ContainerCluster clusterControllers, ContentCluster contentCluster) { + int index = 0; + for (Container container : clusterControllers.getContainers()) { + if ( ! hasClusterControllerComponent(container)) + container.addComponent(new ClusterControllerComponent()); + container.addComponent(new ClusterControllerConfigurer(contentCluster, index++, clusterControllers.getContainers().size())); + } + + } + + private boolean hasClusterControllerComponent(Container container) { + for (Object o : container.getComponents().getComponents()) + if (o instanceof ClusterControllerComponent) return true; + return false; + } + + } + + private ContentCluster(AbstractConfigProducer parent, + String clusterName, + Map<String, NewDocumentType> documentDefinitions, + String routingSelection, + Redundancy redundancy) { + super(parent, clusterName); + this.clusterName = clusterName; + this.documentDefinitions = documentDefinitions; + this.documentSelection = routingSelection; + this.redundancy = redundancy; + } + + public void prepare() { + search.prepare(); + + if (clusterControllers != null) { + clusterControllers.prepare(); + } + } + + /** Returns cluster controllers if this is multitenant, null otherwise */ + public ContainerCluster getClusterControllers() { return clusterControllers; } + + public DistributionMode getDistributionMode() { + if (distributionMode != null) return distributionMode; + return getPersistence().getDefaultDistributionMode(); + } + + public boolean isMemfilePersistence() { + return persistenceFactory instanceof VDSEngine.Factory; + } + + public static String getClusterName(ModelElement clusterElem) { + String clusterName = clusterElem.getStringAttribute("id"); + if (clusterName == null) { + clusterName = "content"; + } + + return clusterName; + } + + public String getName() { return clusterName; } + + public String getRoutingSelector() { return documentSelection; } + + public DistributorCluster getDistributorNodes() { return distributorNodes; } + + public StorageCluster getStorageNodes() { return storageNodes; } + + public ClusterControllerConfig getClusterControllerConfig() { return clusterControllerConfig; } + + public PersistenceEngine.PersistenceFactory getPersistence() { return persistenceFactory; } + + /** + * The list of documentdefinitions declared at the cluster level. + * @return the set of documenttype names + */ + public Map<String, NewDocumentType> getDocumentDefinitions() { return documentDefinitions; } + + public final ContentSearchCluster getSearch() { return search; } + + public Redundancy redundancy() { return redundancy; } + + @Override + public void getConfig(MessagetyperouteselectorpolicyConfig.Builder builder) { + if (!getSearch().hasIndexedCluster()) return; + builder. + defaultroute(com.yahoo.vespa.model.routing.DocumentProtocol.getDirectRouteName(getConfigId())). + route(new MessagetyperouteselectorpolicyConfig.Route.Builder(). + messagetype(DocumentProtocol.MESSAGE_PUTDOCUMENT). + name(com.yahoo.vespa.model.routing.DocumentProtocol.getIndexedRouteName(getConfigId()))). + route(new MessagetyperouteselectorpolicyConfig.Route.Builder(). + messagetype(DocumentProtocol.MESSAGE_REMOVEDOCUMENT). + name(com.yahoo.vespa.model.routing.DocumentProtocol.getIndexedRouteName(getConfigId()))). + route(new MessagetyperouteselectorpolicyConfig.Route.Builder(). + messagetype(DocumentProtocol.MESSAGE_UPDATEDOCUMENT). + name(com.yahoo.vespa.model.routing.DocumentProtocol.getIndexedRouteName(getConfigId()))); + } + + public com.yahoo.vespa.model.content.StorageGroup getRootGroup() { + return rootGroup; + } + + @Override + public void getConfig(StorDistributionConfig.Builder builder) { + if (rootGroup != null) { + builder.group.addAll(rootGroup.getGroupStructureConfig()); + } + + if (redundancy != null) { + redundancy.getConfig(builder); + } + + if (search.usesHierarchicDistribution()) { + builder.active_per_leaf_group(true); + } + } + + int getNodeCount() { + return storageNodes.getChildren().size(); + } + + int getNodeCountPerGroup() { + return rootGroup != null ? getNodeCount() / rootGroup.getNumberOfLeafGroups() : getNodeCount(); + } + + @Override + public void getConfig(FleetcontrollerConfig.Builder builder) { + builder.ideal_distribution_bits(distributionBits()); + if (getNodeCount() < 5) { + builder.min_storage_up_count(1); + builder.min_distributor_up_ratio(0); + builder.min_storage_up_ratio(0); + } + } + + @Override + public void getConfig(StorDistributormanagerConfig.Builder builder) { + builder.minsplitcount(distributionBits()); + if (maxNodesPerMerge != null) { + builder.maximum_nodes_per_merge(maxNodesPerMerge); + } + } + + /** + * Returns the distribution bits this cluster should use. + * OnHosted Vespa this is hardcoded not computed from the nodes because reducing the number of nodes is a common + * operation while reducing the number of distribution bits can lead to consistency problems. + * This hardcoded value should work fine from 1-200 nodes. Those who have more will need to set this value + * in config and not remove it again if they reduce the node count. + */ + public int distributionBits() { + // if (hostedVespa) return 16; TODO: Re-enable this later (Nov 2015, ref VESPA-1702) + return DistributionBitCalculator.getDistributionBits(getNodeCountPerGroup(), getDistributionMode()); + } + + @Override + public void validate() throws Exception { + super.validate(); + if (search.usesHierarchicDistribution() && ! isHostedVespa()) { + // validate manually configured groups + new IndexedHierarchicDistributionValidator(search.getClusterName(), rootGroup, redundancy, search.getIndexed().getTuning().dispatch.policy).validate(); + if (search.getIndexed().useMultilevelDispatchSetup()) { + throw new IllegalArgumentException("In indexed content cluster '" + search.getClusterName() + "': Using multi-level dispatch setup is not supported when using hierarchical distribution."); + } + } + } + + public static Map<String, Integer> METRIC_INDEX_MAP = new TreeMap<>(); + static { + METRIC_INDEX_MAP.put("status", 0); + METRIC_INDEX_MAP.put("log", 1); + METRIC_INDEX_MAP.put("yamas", 2); + METRIC_INDEX_MAP.put("health", 3); + METRIC_INDEX_MAP.put("fleetcontroller", 4); + METRIC_INDEX_MAP.put("statereporter", 5); + } + + public static MetricsmanagerConfig.Consumer.Builder getMetricBuilder(String name, MetricsmanagerConfig.Builder builder) { + Integer index = METRIC_INDEX_MAP.get(name); + if (index != null) { + return builder.consumer.get(index); + } + + MetricsmanagerConfig.Consumer.Builder retVal = new MetricsmanagerConfig.Consumer.Builder(); + retVal.name(name); + builder.consumer(retVal); + return retVal; + } + + @Override + public void getConfig(MetricsmanagerConfig.Builder builder) { + MonitoringSystem monitoringSystem = getMonitoringService(); + if (monitoringSystem != null) { + builder.snapshot(new MetricsmanagerConfig.Snapshot.Builder(). + periods(monitoringSystem.getIntervalSeconds()).periods(300)); + } + builder.consumer( + new MetricsmanagerConfig.Consumer.Builder(). + name("status"). + addedmetrics("*"). + removedtags("partofsum")); + + builder.consumer( + new MetricsmanagerConfig.Consumer.Builder(). + name("log"). + tags("logdefault"). + removedtags("loadtype")); + builder.consumer( + new MetricsmanagerConfig.Consumer.Builder(). + name("yamas"). + tags("yamasdefault"). + removedtags("loadtype")); + builder.consumer( + new MetricsmanagerConfig.Consumer.Builder(). + name("health")); + builder.consumer( + new MetricsmanagerConfig.Consumer.Builder(). + name("fleetcontroller")); + builder.consumer( + new MetricsmanagerConfig.Consumer.Builder(). + name("statereporter"). + addedmetrics("*"). + removedtags("thread"). + tags("disk")); + + Map<String, MetricsConsumer> consumers = getRoot().getAdmin().getUserMetricsConsumers(); + if (consumers != null) { + for (Map.Entry<String, MetricsConsumer> e : consumers.entrySet()) { + MetricsmanagerConfig.Consumer.Builder b = getMetricBuilder(e.getKey(), builder); + for (Metric m : e.getValue().getMetrics().values()) { + b.addedmetrics(m.getName()); + } + } + } + + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DocumentSelectionBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DocumentSelectionBuilder.java new file mode 100644 index 00000000000..b145637b6c5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DocumentSelectionBuilder.java @@ -0,0 +1,80 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.cluster; + +import com.yahoo.document.select.DocumentSelector; +import com.yahoo.document.select.parser.ParseException; +import com.yahoo.document.select.rule.DocumentNode; +import com.yahoo.vespa.model.content.DocumentTypeVisitor; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; + +/** + * @author thomasg + */ +public class DocumentSelectionBuilder { + + private static class AllowedDocumentTypesChecker extends DocumentTypeVisitor { + String allowedType; + + private AllowedDocumentTypesChecker(String allowedType) { + this.allowedType = allowedType; + } + + @Override + public void visit(DocumentNode documentNode) { + if (!documentNode.getType().equals(this.allowedType)) { + if (this.allowedType == null) { + throw new IllegalArgumentException("Document type references are not allowed " + + "in global <documents> tag selection attribute (found reference to type '" + + documentNode.getType() + "')"); + } else { + throw new IllegalArgumentException("Selection for document type '" + + this.allowedType + "' can not contain references to other " + + "document types (found reference to type '" + documentNode.getType() + "')"); + } + } + } + } + + private void validateSelectionExpression(String sel, String allowedType) { + DocumentSelector s; + try { + s = new DocumentSelector(sel); + } catch (ParseException e) { + throw new IllegalArgumentException("Could not parse document routing selection: " + sel, e); + } + AllowedDocumentTypesChecker checker = new AllowedDocumentTypesChecker(allowedType); + s.visit(checker); + } + + public String build(ModelElement elem) { + StringBuilder sb = new StringBuilder(); + if (elem != null) { + for (ModelElement e : elem.subElements("document")) { + if (sb.length() > 0) { + sb.append(" OR "); + } + sb.append('('); + String type = e.getStringAttribute("type"); + sb.append(type); + String selection = e.getStringAttribute("selection"); + if (selection != null) { + validateSelectionExpression(selection, type); + sb.append(" AND ("); + sb.append(selection); + sb.append(')'); + } + sb.append(')'); + } + + String globalSelection = elem.getStringAttribute("selection"); + if (globalSelection != null) { + validateSelectionExpression(globalSelection, null); + StringBuilder global = new StringBuilder(); + global.append('(').append(globalSelection).append(") AND (") + .append(sb.toString()).append(')'); + return global.toString(); + } + } + return sb.toString(); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomContentSearchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomContentSearchBuilder.java new file mode 100644 index 00000000000..b43b856b38a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomContentSearchBuilder.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.cluster; + +import com.yahoo.vespa.model.content.ContentSearch; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class DomContentSearchBuilder { + + public static ContentSearch build(ModelElement contentXml) { + ContentSearch.Builder builder = new ContentSearch.Builder(); + ModelElement searchElement = contentXml.getChild("search"); + if (searchElement == null) { + return builder.build(); + } + builder.setQueryTimeout(searchElement.childAsDouble("query-timeout")); + builder.setVisibilityDelay(searchElement.childAsDouble("visibility-delay")); + return builder.build(); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomDispatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomDispatchBuilder.java new file mode 100644 index 00000000000..4796c8b1382 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomDispatchBuilder.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.cluster; + + +import com.yahoo.vespa.model.content.DispatchSpec; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; + +import java.util.ArrayList; +import java.util.List; + +/** + * Builder for the dispatch setup for a content cluster. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class DomDispatchBuilder { + + public static DispatchSpec build(ModelElement contentXml) { + DispatchSpec.Builder builder = new DispatchSpec.Builder(); + ModelElement dispatchElement = contentXml.getChild("dispatch"); + if (dispatchElement == null) { + return builder.build(); + } + builder.setNumDispatchGroups(dispatchElement.childAsInteger("num-dispatch-groups")); + + List<ModelElement> groupsElement = dispatchElement.subElements("group"); + if (groupsElement != null) { + builder.setGroups(buildGroups(groupsElement)); + } + return builder.build(); + } + + private static List<DispatchSpec.Group> buildGroups(List<ModelElement> groupsElement) { + List<DispatchSpec.Group> groups = new ArrayList<>(); + for (ModelElement groupElement : groupsElement) { + groups.add(buildGroup(groupElement)); + } + return groups; + } + + private static DispatchSpec.Group buildGroup(ModelElement groupElement) { + List<ModelElement> nodes = groupElement.subElements("node"); + DispatchSpec.Group group = new DispatchSpec.Group(); + for (ModelElement nodeElement : nodes) { + group.addNode(buildNode(nodeElement)); + } + return group; + } + + private static DispatchSpec.Node buildNode(ModelElement nodeElement) { + return new DispatchSpec.Node(nodeElement.getIntegerAttribute("distribution-key")); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java new file mode 100644 index 00000000000..8c83957eb26 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.cluster; + +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.content.ResourceLimits; + +/** + * Builder for resource limits for a content cluster with engine proton. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class DomResourceLimitsBuilder { + + public static ResourceLimits build(ModelElement contentXml) { + ResourceLimits.Builder builder = new ResourceLimits.Builder(); + ModelElement resourceLimits = contentXml.getChild("resource-limits"); + if (resourceLimits == null) { + return builder.build(); + } + builder.setDiskLimit(resourceLimits.childAsDouble("disk")); + builder.setMemoryLimit(resourceLimits.childAsDouble("memory")); + return builder.build(); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomSearchCoverageBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomSearchCoverageBuilder.java new file mode 100644 index 00000000000..83602c3003c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomSearchCoverageBuilder.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.cluster; + +import com.yahoo.vespa.model.content.SearchCoverage; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class DomSearchCoverageBuilder { + + public static SearchCoverage build(ModelElement contentXml) { + SearchCoverage.Builder builder = new SearchCoverage.Builder(); + ModelElement searchElement = contentXml.getChild("search"); + if (searchElement == null) { + return builder.build(); + } + ModelElement coverageElement = searchElement.getChild("coverage"); + if (coverageElement == null) { + return builder.build(); + } + builder.setMinimum(coverageElement.childAsDouble("minimum")); + builder.setMinWaitAfterCoverageFactor(coverageElement.childAsDouble("min-wait-after-coverage-factor")); + builder.setMaxWaitAfterCoverageFactor(coverageElement.childAsDouble("max-wait-after-coverage-factor")); + return builder.build(); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java new file mode 100644 index 00000000000..1d2bf938758 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomTuningDispatchBuilder.java @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.cluster; + +import com.yahoo.vespa.model.content.TuningDispatch; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class DomTuningDispatchBuilder { + + public static TuningDispatch build(ModelElement contentXml) { + TuningDispatch.Builder builder = new TuningDispatch.Builder(); + ModelElement tuningElement = contentXml.getChild("tuning"); + if (tuningElement == null) { + return builder.build(); + } + ModelElement dispatchElement = tuningElement.getChild("dispatch"); + if (dispatchElement == null) { + return builder.build(); + } + builder.setMaxHitsPerPartition(dispatchElement.childAsInteger("max-hits-per-partition")); + builder.setDispatchPolicy(dispatchElement.childAsString("dispatch-policy")); + builder.setUseLocalNode(dispatchElement.childAsBoolean("use-local-node")); + builder.setMinGroupCoverage(dispatchElement.childAsDouble("min-group-coverage")); + builder.setMinActiveDocsCoverage(dispatchElement.childAsDouble("min-active-docs-coverage")); + + return builder.build(); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java new file mode 100644 index 00000000000..374f970739d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/EngineFactoryBuilder.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.cluster; + +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.content.engines.*; + +/** + * Creates the correct engine factory from XML. + */ +public class EngineFactoryBuilder { + public PersistenceEngine.PersistenceFactory build(ModelElement clusterElem, ContentCluster c) { + ModelElement persistence = clusterElem.getChild("engine"); + if (persistence != null) { + if (c.getSearch().hasIndexedCluster() && persistence.getChild("proton") == null) { + throw new IllegalArgumentException("Persistence engine does not allow for indexed search. Please use <proton> as your engine."); + } + + ModelElement e; + if ((e = persistence.getChild("vds")) != null) { + return new VDSEngine.Factory(e); + } else if (persistence.getChild("proton") != null) { + return new ProtonEngine.Factory(c.getSearch()); + } else if (persistence.getChild("dummy") != null) { + return new com.yahoo.vespa.model.content.engines.DummyPersistence.Factory(); + } else if (persistence.getChild("rpc") != null) { + return new RPCEngine.Factory(); + } + } + + return new ProtonEngine.Factory(c.getSearch()); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java new file mode 100644 index 00000000000..d80f7029320 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/RedundancyBuilder.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.cluster; + +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.content.Redundancy; + +/** + * Builds redundancy config for a content cluster. + */ +public class RedundancyBuilder { + Redundancy build(ModelElement clusterXml) { + Integer initialRedundancy = 2; + Integer finalRedundancy = 3; + Integer readyCopies = 2; + + ModelElement redundancyElement = clusterXml.getChild("redundancy"); + if (redundancyElement != null) { + initialRedundancy = redundancyElement.getIntegerAttribute("reply-after"); + finalRedundancy = (int)redundancyElement.asLong(); + + if (initialRedundancy == null) { + initialRedundancy = finalRedundancy; + } else { + if (finalRedundancy < initialRedundancy) { + throw new IllegalArgumentException("Final redundancy must be higher than or equal to initial redundancy"); + } + } + + readyCopies = clusterXml.childAsInteger("engine.proton.searchable-copies"); + if (readyCopies == null) { + readyCopies = Math.min(finalRedundancy, 2); + } + if (readyCopies > finalRedundancy) { + throw new IllegalArgumentException("Number of searchable copies can not be higher than final redundancy"); + } + } + + return new Redundancy(initialRedundancy, finalRedundancy, readyCopies); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java new file mode 100644 index 00000000000..3f43dbe4491 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/SearchDefinitionBuilder.java @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.cluster; + +import com.yahoo.documentmodel.DocumentTypeRepo; +import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; + +import java.util.Map; +import java.util.TreeMap; + +/** +* Created with IntelliJ IDEA. +* User: thomasg +* Date: 9/28/12 +* Time: 1:20 PM +* To change this template use File | Settings | File Templates. +*/ +public class SearchDefinitionBuilder { + public Map<String, NewDocumentType> build(DocumentTypeRepo repo, ModelElement elem) { + Map<String, NewDocumentType> docTypes = new TreeMap<>(); + + if (elem != null) { + for (ModelElement e : elem.subElements("document")) { + String name = e.getStringAttribute("type"); // Schema-guaranteed presence + NewDocumentType documentType = repo.getDocumentType(name); + if (documentType != null) { + docTypes.put(documentType.getName(), documentType); + } else { + throw new RuntimeException("Document type '" + name + "' not found in application package"); + } + } + } + + return docTypes; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/package-info.java new file mode 100644 index 00000000000..47504d5d0cd --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.model.content.cluster; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/engines/DummyPersistence.java b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/DummyPersistence.java new file mode 100644 index 00000000000..b52b3e0d248 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/DummyPersistence.java @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.engines; + +import com.yahoo.vespa.config.content.core.StorServerConfig; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.content.StorageGroup; +import com.yahoo.vespa.model.content.StorageNode; +import com.yahoo.vespa.model.content.cluster.ContentCluster; + +public class DummyPersistence extends PersistenceEngine { + public DummyPersistence(StorageNode parent) { + super(parent, "provider"); + } + + @Override + public void getConfig(StorServerConfig.Builder builder) { + builder.persistence_provider(new StorServerConfig.Persistence_provider.Builder().type(StorServerConfig.Persistence_provider.Type.Enum.DUMMY)); + } + + public static class Factory implements PersistenceFactory { + + @Override + public PersistenceEngine create(StorageNode storageNode, StorageGroup parentGroup, ModelElement storageNodeElement) { + return new DummyPersistence(storageNode); + } + + @Override + public boolean supportRevert() { + return true; + } + + @Override + public boolean enableMultiLevelSplitting() { + return true; + } + + @Override + public ContentCluster.DistributionMode getDefaultDistributionMode() { + return ContentCluster.DistributionMode.LOOSE; + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/engines/PersistenceEngine.java b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/PersistenceEngine.java new file mode 100644 index 00000000000..29ed02ab2b6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/PersistenceEngine.java @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.engines; + +import com.yahoo.vespa.config.content.core.StorServerConfig; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.content.StorageGroup; +import com.yahoo.vespa.model.content.StorageNode; +import com.yahoo.vespa.model.content.cluster.ContentCluster; + +public abstract class PersistenceEngine extends AbstractConfigProducer implements StorServerConfig.Producer { + + public PersistenceEngine(AbstractConfigProducer parent, String name) { + super(parent, name); + } + + /** + * Creates a config producer for the engines provider at a given node. + */ + public static interface PersistenceFactory { + public PersistenceEngine create(StorageNode storageNode, StorageGroup parentGroup, ModelElement storageNodeElement); + + /** + * If a write request succeeds on some nodes and fails on others, causing request to + * fail to client, the content layer will revert the operation where it succeeded if + * reverts are supported. (Typically require backend to keep multiple entries of the + * same document identifier persisted at the same time) + */ + public boolean supportRevert(); + + /** + * Multi level splitting can increase split performance a lot where documents have been + * co-localized, for backends where retrieving document identifiers contained in bucket + * is cheap. Backends where split is cheaper than fetching document identifiers will + * not want to enable multi level splitting. + */ + public boolean enableMultiLevelSplitting(); + + public ContentCluster.DistributionMode getDefaultDistributionMode(); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/engines/ProtonEngine.java b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/ProtonEngine.java new file mode 100644 index 00000000000..595da4da2a0 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/ProtonEngine.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.engines; + +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.content.ContentSearchCluster; +import com.yahoo.vespa.model.content.StorageGroup; +import com.yahoo.vespa.model.content.StorageNode; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.search.SearchNode; + +/** + * Initializes the engines engine on each storage node. May include creating other + * nodes. + */ +public class ProtonEngine { + public static class Factory implements PersistenceEngine.PersistenceFactory { + ContentSearchCluster search; + + public Factory(ContentSearchCluster search) { + this.search = search; + } + + @Override + public PersistenceEngine create(StorageNode storageNode, StorageGroup parentGroup, ModelElement storageNodeElement) { + SearchNode searchNode = search.addSearchNode(storageNode, parentGroup, storageNodeElement); + return new ProtonProvider(storageNode, searchNode); + } + + @Override + public boolean supportRevert() { + return false; + } + + @Override + public boolean enableMultiLevelSplitting() { + return false; + } + + @Override + public ContentCluster.DistributionMode getDefaultDistributionMode() { + return ContentCluster.DistributionMode.LOOSE; + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/engines/ProtonProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/ProtonProvider.java new file mode 100644 index 00000000000..d665df3d53c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/ProtonProvider.java @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.engines; + +import com.yahoo.vespa.model.content.StorageNode; +import com.yahoo.vespa.model.search.SearchNode; + +/** + * Created with IntelliJ IDEA. + * User: balder + * Date: 20.11.12 + * Time: 20:04 + * To change this template use File | Settings | File Templates. + */ +public class ProtonProvider extends RPCEngine { + public ProtonProvider(StorageNode parent, SearchNode searchNode) { + super(parent, searchNode); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/engines/RPCEngine.java b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/RPCEngine.java new file mode 100644 index 00000000000..ec0d7dc35f4 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/RPCEngine.java @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.engines; + +import com.yahoo.vespa.config.content.core.StorServerConfig; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.content.StorageGroup; +import com.yahoo.vespa.model.content.StorageNode; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.search.SearchNode; + +public class RPCEngine extends PersistenceEngine { + + private SearchNode searchNode; + public RPCEngine(StorageNode parent) { + super(parent, "provider"); + } + + public RPCEngine(StorageNode parent, SearchNode searchNode) { + super(parent, "provider"); + this.searchNode = searchNode; + } + + @Override + public void getConfig(StorServerConfig.Builder builder) { + StorServerConfig.Persistence_provider.Builder provider = + new StorServerConfig.Persistence_provider.Builder(); + provider.type(StorServerConfig.Persistence_provider.Type.Enum.RPC); + + if (searchNode != null) { + provider.rpc(new StorServerConfig.Persistence_provider.Rpc.Builder().connectspec("tcp/localhost:" + searchNode.getPersistenceProviderRpcPort())); + } + + builder.persistence_provider(provider); + } + + public static class Factory implements PersistenceFactory { + @Override + public PersistenceEngine create(StorageNode storageNode, StorageGroup parentGroup, ModelElement storageNodeElement) { + return new RPCEngine(storageNode); + } + + @Override + public boolean supportRevert() { + return false; + } + + @Override + public boolean enableMultiLevelSplitting() { + return false; + } + + @Override + public ContentCluster.DistributionMode getDefaultDistributionMode() { + return ContentCluster.DistributionMode.LOOSE; + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/engines/VDSEngine.java b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/VDSEngine.java new file mode 100644 index 00000000000..5e407938159 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/VDSEngine.java @@ -0,0 +1,83 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.engines; + +import com.yahoo.vespa.config.storage.StorMemfilepersistenceConfig; +import com.yahoo.vespa.config.content.core.StorServerConfig; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.content.StorageGroup; +import com.yahoo.vespa.model.content.StorageNode; +import com.yahoo.vespa.model.content.cluster.ContentCluster; + +/** + * Configuration class to generate config for the memfile engines provider. + */ +public class VDSEngine extends PersistenceEngine + implements StorMemfilepersistenceConfig.Producer +{ + ModelElement tuning; + + public VDSEngine(StorageNode parent, ModelElement vdsConfig) { + super(parent, "provider"); + + if (vdsConfig != null) { + this.tuning = vdsConfig.getChild("tuning"); + } + + if (parent != null) { + parent.useVdsEngine(); + } + } + + @Override + public void getConfig(StorMemfilepersistenceConfig.Builder builder) { + if (tuning == null) { + return; + } + + ModelElement diskFullRatio = tuning.getChild("disk-full-ratio"); + if (diskFullRatio != null) { + builder.disk_full_factor(diskFullRatio.asDouble()); + } + + ModelElement cacheSize = tuning.getChild("cache-size"); + if (cacheSize != null) { + builder.cache_size(cacheSize.asLong()); + } + } + + @Override + public void getConfig(StorServerConfig.Builder builder) { + builder.persistence_provider( + new StorServerConfig.Persistence_provider.Builder().type( + StorServerConfig.Persistence_provider.Type.Enum.STORAGE) + ); + } + + public static class Factory implements PersistenceFactory { + ModelElement vdsConfig; + + public Factory(ModelElement vdsConfig) { + this.vdsConfig = vdsConfig; + } + + @Override + public PersistenceEngine create(StorageNode storageNode, StorageGroup parentGroup, ModelElement storageNodeElement) { + return new VDSEngine(storageNode, vdsConfig); + } + + @Override + public boolean supportRevert() { + return true; + } + + @Override + public boolean enableMultiLevelSplitting() { + return true; + } + + @Override + public ContentCluster.DistributionMode getDefaultDistributionMode() { + return ContentCluster.DistributionMode.STRICT; + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/engines/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/package-info.java new file mode 100644 index 00000000000..28652c82996 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.model.content.engines; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/content/package-info.java new file mode 100644 index 00000000000..a0af4d2a5fd --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.model.content; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java new file mode 100644 index 00000000000..6a5f14899ee --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java @@ -0,0 +1,73 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.storagecluster; + +import com.yahoo.vespa.config.content.StorFilestorConfig; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.content.PriorityMapping; +import com.yahoo.vespa.model.content.cluster.ContentCluster; + +import java.util.ArrayList; +import java.util.List; + +/** + * Serves stor-filestor for storage clusters. + */ +public class FileStorProducer implements StorFilestorConfig.Producer { + public static class Builder { + protected FileStorProducer build(ContentCluster parent, ModelElement clusterElem) { + return new FileStorProducer(parent, getThreads(clusterElem)); + } + + private List<StorFilestorConfig.Threads.Builder> getThreads(ModelElement clusterElem) { + ModelElement tuning = clusterElem.getChild("tuning"); + if (tuning == null) { + return null; + } + ModelElement threads = tuning.getChild("persistence-threads"); + if (threads == null) { + return null; + } + + List<StorFilestorConfig.Threads.Builder> retVal = new ArrayList<>(); + + PriorityMapping mapping = new PriorityMapping(clusterElem); + + for (ModelElement thread : threads.subElements("thread")) { + String priorityName = thread.getStringAttribute("lowest-priority"); + if (priorityName == null) { + priorityName = "LOWEST"; + } + + Integer count = thread.getIntegerAttribute("count"); + if (count == null) { + count = 1; + } + + for (int i = 0; i < count; ++i) { + retVal.add(new StorFilestorConfig.Threads.Builder().lowestpri(mapping.getPriorityMapping(priorityName))); + } + } + + return retVal; + } + } + + private List<StorFilestorConfig.Threads.Builder> threads; + private ContentCluster cluster; + + public FileStorProducer(ContentCluster parent, List<StorFilestorConfig.Threads.Builder> threads) { + this.threads = threads; + this.cluster = parent; + } + + @Override + public void getConfig(StorFilestorConfig.Builder builder) { + if (threads != null) { + for (StorFilestorConfig.Threads.Builder t : threads) { + builder.threads.add(t); + } + } + builder.enable_multibit_split_optimalization(cluster.getPersistence().enableMultiLevelSplitting()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/IntegrityCheckerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/IntegrityCheckerProducer.java new file mode 100644 index 00000000000..69bf29ca307 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/IntegrityCheckerProducer.java @@ -0,0 +1,80 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.storagecluster; + +import com.yahoo.vespa.config.content.core.StorIntegritycheckerConfig; +import com.yahoo.config.model.ConfigModelUtils; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; + +/** + * Serves stor-integritychecker config for storage clusters. + */ +public class IntegrityCheckerProducer implements StorIntegritycheckerConfig.Producer { + + public static class Builder { + protected IntegrityCheckerProducer build(ModelElement clusterElem) { + ModelElement tuning = clusterElem.getChild("tuning"); + + if (tuning == null) { + return new IntegrityCheckerProducer(); + } + + ModelElement maintenance = tuning.getChild("maintenance"); + if (maintenance == null) { + return new IntegrityCheckerProducer(); + } + + Integer startTime = null; + Integer stopTime = null; + String weeklyCycle = null; + + String start = maintenance.getStringAttribute("start"); + if (start != null) { + startTime = ConfigModelUtils.getTimeOfDay(start); + } + + String stop = maintenance.getStringAttribute("stop"); + if (stop != null) { + stopTime = ConfigModelUtils.getTimeOfDay(stop); + } + + String high = maintenance.getStringAttribute("high"); + + if (high != null) { + int weekday = ConfigModelUtils.getDayOfWeek(high); + char[] weeklycycle = "rrrrrrr".toCharArray(); + weeklycycle[weekday] = 'R'; + weeklyCycle = String.valueOf(weeklycycle); + } + + return new IntegrityCheckerProducer(startTime, stopTime, weeklyCycle); + } + } + + private Integer startTime; + private Integer stopTime; + private String weeklyCycle; + + IntegrityCheckerProducer() { + } + + IntegrityCheckerProducer(Integer startTime, Integer stopTime, String weeklyCycle) { + this.startTime = startTime; + this.stopTime = stopTime; + this.weeklyCycle = weeklyCycle; + } + + @Override + public void getConfig(StorIntegritycheckerConfig.Builder builder) { + if (startTime != null) { + builder.dailycyclestart(startTime); + } + + if (stopTime != null) { + builder.dailycyclestop(stopTime); + } + + if (weeklyCycle != null) { + builder.weeklycycle(weeklyCycle); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/PersistenceProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/PersistenceProducer.java new file mode 100644 index 00000000000..d632329192b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/PersistenceProducer.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.storagecluster; + +import com.yahoo.vespa.config.content.PersistenceConfig; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.utils.Duration; + +/** + * Serves engines config for storage clusters. + */ +public class PersistenceProducer implements PersistenceConfig.Producer { + + public static class Builder { + public PersistenceProducer build(ModelElement element) { + ModelElement persistence = element.getChild("engine"); + if (persistence == null) { + return new PersistenceProducer(); + } + + return new PersistenceProducer( + persistence.childAsBoolean("fail-partition-on-error"), + persistence.childAsDuration("recovery-time"), + persistence.childAsDuration("revert-time")); + } + } + + Boolean failOnError; + Duration recoveryPeriod; + Duration revertTimePeriod; + + public PersistenceProducer() {} + + public PersistenceProducer(Boolean failOnError, Duration recoveryPeriod, Duration revertTimePeriod) { + this.failOnError = failOnError; + this.recoveryPeriod = recoveryPeriod; + this.revertTimePeriod = revertTimePeriod; + } + + @Override + public void getConfig(PersistenceConfig.Builder builder) { + if (failOnError != null) { + builder.fail_partition_on_error(failOnError); + } + if (recoveryPeriod != null) { + builder.keep_remove_time_period((int)recoveryPeriod.getSeconds()); + } + if (revertTimePeriod != null) { + builder.revert_time_period((int)revertTimePeriod.getSeconds()); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java new file mode 100644 index 00000000000..e6f767e881c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.storagecluster; + +import com.yahoo.vespa.config.content.core.StorServerConfig; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; + +/** + * Serves config for stor-server for storage clusters (clusters of storage nodes). + */ +public class StorServerProducer implements StorServerConfig.Producer { + public static class Builder { + StorServerProducer build(ModelElement element) { + ModelElement tuning = element.getChild("tuning"); + + if (tuning == null) { + return new StorServerProducer(ContentCluster.getClusterName(element), null, null); + } + + ModelElement merges = tuning.getChild("merges"); + if (merges == null) { + return new StorServerProducer(ContentCluster.getClusterName(element), null, null); + } + + return new StorServerProducer(ContentCluster.getClusterName(element), + merges.getIntegerAttribute("max-per-node"), + merges.getIntegerAttribute("max-queue-size")); + } + } + + private String clusterName; + private Integer maxMergesPerNode; + private Integer queueSize; + + public StorServerProducer(String clusterName, Integer maxMergesPerNode, Integer queueSize) { + this.clusterName = clusterName; + this.maxMergesPerNode = maxMergesPerNode; + this.queueSize = queueSize; + } + + @Override + public void getConfig(StorServerConfig.Builder builder) { + builder.root_folder(""); + builder.is_distributor(false); + + if (clusterName != null) { + builder.cluster_name(clusterName); + } + if (maxMergesPerNode != null) { + builder.max_merges_per_node(maxMergesPerNode); + } + if (queueSize != null) { + builder.max_merge_queue_size(queueSize); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorVisitorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorVisitorProducer.java new file mode 100644 index 00000000000..b04ef0d15a4 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorVisitorProducer.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.storagecluster; + +import com.yahoo.vespa.config.content.core.StorVisitorConfig; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; + +/** + * Serves stor-visitor config for storage clusters. + */ +public class StorVisitorProducer implements StorVisitorConfig.Producer { + public static class Builder { + public StorVisitorProducer build(ModelElement element) { + ModelElement tuning = element.getChild("tuning"); + if (tuning == null) { + return new StorVisitorProducer(); + } + + ModelElement visitors = tuning.getChild("visitors"); + if (visitors == null) { + return new StorVisitorProducer(); + } + + return new StorVisitorProducer(visitors.getIntegerAttribute("thread-count"), + visitors.getIntegerAttribute("max-queue-size"), + visitors.childAsInteger("max-concurrent.fixed"), + visitors.childAsInteger("max-concurrent.variable")); + } + } + + Integer threadCount; + Integer maxQueueSize; + Integer maxConcurrentFixed; + Integer maxConcurrentVariable; + + public StorVisitorProducer() {} + + StorVisitorProducer(Integer threadCount, Integer maxQueueSize, Integer maxConcurrentFixed, Integer maxConcurrentVariable) { + this.threadCount = threadCount; + this.maxQueueSize = maxQueueSize; + this.maxConcurrentFixed = maxConcurrentFixed; + this.maxConcurrentVariable = maxConcurrentVariable; + } + + @Override + public void getConfig(StorVisitorConfig.Builder builder) { + if (threadCount != null) { + builder.visitorthreads(threadCount); + } + if (maxQueueSize != null) { + builder.maxvisitorqueuesize(maxQueueSize); + } + if (maxConcurrentFixed != null) { + builder.maxconcurrentvisitors_fixed(maxConcurrentFixed); + } + if (maxConcurrentVariable != null) { + builder.maxconcurrentvisitors_variable(maxConcurrentVariable); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java new file mode 100644 index 00000000000..6c9cbadd21a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java @@ -0,0 +1,149 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content.storagecluster; + +import com.yahoo.vespa.config.content.core.StorIntegritycheckerConfig; +import com.yahoo.vespa.config.storage.StorMemfilepersistenceConfig; +import com.yahoo.vespa.config.content.core.StorBucketmoverConfig; +import com.yahoo.vespa.config.content.core.StorVisitorConfig; +import com.yahoo.vespa.config.content.StorFilestorConfig; +import com.yahoo.vespa.config.content.core.StorServerConfig; +import com.yahoo.vespa.config.content.PersistenceConfig; +import com.yahoo.metrics.MetricsmanagerConfig; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.content.StorageNode; +import org.w3c.dom.Element; + +/** + * Represents configuration that is common to all storage nodes. + */ +public class StorageCluster extends AbstractConfigProducer<StorageNode> + implements StorServerConfig.Producer, + StorBucketmoverConfig.Producer, + StorMemfilepersistenceConfig.Producer, + StorIntegritycheckerConfig.Producer, + StorFilestorConfig.Producer, + StorVisitorConfig.Producer, + PersistenceConfig.Producer, + MetricsmanagerConfig.Producer +{ + public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<StorageCluster> { + @Override + protected StorageCluster doBuild(AbstractConfigProducer ancestor, Element producerSpec) { + ModelElement clusterElem = new ModelElement(producerSpec); + + return new StorageCluster(ancestor, + ContentCluster.getClusterName(clusterElem), + new FileStorProducer.Builder().build(((ContentCluster)ancestor), clusterElem), + new IntegrityCheckerProducer.Builder().build(clusterElem), + new StorServerProducer.Builder().build(clusterElem), + new StorVisitorProducer.Builder().build(clusterElem), + new PersistenceProducer.Builder().build(clusterElem)); + } + } + + private Integer bucketMoverMaxFillAboveAverage = null; + private Long cacheSize = null; + private Double diskFullPercentage = null; + + private String clusterName; + private FileStorProducer fileStorProducer; + private IntegrityCheckerProducer integrityCheckerProducer; + private StorServerProducer storServerProducer; + private StorVisitorProducer storVisitorProducer; + private PersistenceProducer persistenceProducer; + + StorageCluster(AbstractConfigProducer parent, + String clusterName, + FileStorProducer fileStorProducer, + IntegrityCheckerProducer integrityCheckerProducer, + StorServerProducer storServerProducer, + StorVisitorProducer storVisitorProducer, + PersistenceProducer persistenceProducer) { + super(parent, "storage"); + this.clusterName = clusterName; + this.fileStorProducer = fileStorProducer; + this.integrityCheckerProducer = integrityCheckerProducer; + this.storServerProducer = storServerProducer; + this.storVisitorProducer = storVisitorProducer; + this.persistenceProducer = persistenceProducer; + } + + @Override + public void getConfig(StorBucketmoverConfig.Builder builder) { + if (bucketMoverMaxFillAboveAverage != null) { + builder.max_target_fill_rate_above_average(bucketMoverMaxFillAboveAverage); + } + } + + @Override + public void getConfig(StorMemfilepersistenceConfig.Builder builder) { + if (cacheSize != null) { + builder.cache_size(cacheSize); + } + + if (diskFullPercentage != null) { + builder.disk_full_factor(diskFullPercentage / 100.0); + builder.disk_full_factor_move(diskFullPercentage / 100.0 * 0.9); + } + } + + @Override + public void getConfig(MetricsmanagerConfig.Builder builder) { + ContentCluster.getMetricBuilder("fleetcontroller", builder). + addedmetrics("vds.filestor.*.allthreads.put.sum"). + addedmetrics("vds.filestor.*.allthreads.get.sum"). + addedmetrics("vds.filestor.*.allthreads.multi.sum"). + addedmetrics("vds.filestor.*.allthreads.update.sum"). + addedmetrics("vds.filestor.*.allthreads.remove.sum"). + addedmetrics("vds.filestor.*.allthreads.operations"). + addedmetrics("vds.datastored.alldisks.docs"). + addedmetrics("vds.datastored.alldisks.bytes"). + addedmetrics("vds.datastored.alldisks.buckets"); + + ContentCluster.getMetricBuilder("log", builder). + addedmetrics("vds.filestor.alldisks.allthreads.put.sum"). + addedmetrics("vds.filestor.alldisks.allthreads.get.sum"). + addedmetrics("vds.filestor.alldisks.allthreads.remove.sum"). + addedmetrics("vds.filestor.alldisks.allthreads.update.sum"). + addedmetrics("vds.datastored.alldisks.docs"). + addedmetrics("vds.datastored.alldisks.bytes"). + addedmetrics("vds.filestor.alldisks.queuesize"). + addedmetrics("vds.filestor.alldisks.averagequeuewait.sum"). + addedmetrics("vds.visitor.cv_queuewaittime"). + addedmetrics("vds.visitor.allthreads.averagequeuewait"). + addedmetrics("vds.visitor.allthreads.averagevisitorlifetime"). + addedmetrics("vds.visitor.allthreads.created.sum"); + } + + public String getClusterName() { + return clusterName; + } + + @Override + public void getConfig(StorIntegritycheckerConfig.Builder builder) { + integrityCheckerProducer.getConfig(builder); + } + + @Override + public void getConfig(StorServerConfig.Builder builder) { + storServerProducer.getConfig(builder); + } + + @Override + public void getConfig(StorVisitorConfig.Builder builder) { + storVisitorProducer.getConfig(builder); + } + + @Override + public void getConfig(PersistenceConfig.Builder builder) { + persistenceProducer.getConfig(builder); + } + + @Override + public void getConfig(StorFilestorConfig.Builder builder) { + fileStorProducer.getConfig(builder); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/package-info.java new file mode 100644 index 00000000000..6b3ca6a06b7 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.model.content.storagecluster; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java new file mode 100644 index 00000000000..c3fcdddb273 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java @@ -0,0 +1,59 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.filedistribution; + +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.Host; +import com.yahoo.vespa.model.admin.FileDistributionOptions; + +import java.util.IdentityHashMap; +import java.util.Map; + +/** + * @author tonytv + */ +public class FileDistributionConfigProducer extends AbstractConfigProducer { + private final Map<Host, FileDistributorService> fileDistributorServices = new IdentityHashMap<>(); + private final FileDistributor fileDistributor; + private final FileDistributionOptions options; + + private FileDistributionConfigProducer(AbstractConfigProducer parent, FileDistributor fileDistributor, FileDistributionOptions options) { + super(parent, "filedistribution"); + this.fileDistributor = fileDistributor; + this.options = options; + } + + public String getFileDistributionServiceConfigId(Host host) { + FileDistributorService service = fileDistributorServices.get(host); + if (service == null) { + throw new IllegalStateException("No file distribution service for host " + host); + } + return service.getConfigId(); + } + + public FileDistributor getFileDistributor() { + return fileDistributor; + } + + public FileDistributionOptions getOptions() { + return options; + } + + public void addFileDistributionService(Host host, FileDistributorService fds) { + fileDistributorServices.put(host, fds); + } + + public static class Builder { + + private final FileDistributionOptions options; + + public Builder(FileDistributionOptions fileDistributionOptions) { + this.options = fileDistributionOptions; + } + + public FileDistributionConfigProducer build(AbstractConfigProducer ancestor, FileRegistry fileRegistry) { + FileDistributor fileDistributor = new FileDistributor(fileRegistry); + return new FileDistributionConfigProducer(ancestor, fileDistributor, options); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java new file mode 100644 index 00000000000..4dc24618a61 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java @@ -0,0 +1,120 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.filedistribution; + +import com.yahoo.config.FileReference; +import com.yahoo.config.model.api.FileDistribution; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.vespa.model.Host; + +import java.util.*; +import java.util.stream.Collectors; + +import static java.util.Arrays.asList; + + +/** + * Responsible for directing distribution of files to hosts. + * + * @author tonytv + */ +public class FileDistributor { + + private final FileRegistry fileRegistry; + + /** A map from files to the hosts to which that file should be distributed */ + private final Map<FileReference, Set<Host>> filesToHosts = new LinkedHashMap<>(); + + /** + * Adds the given file to the associated application packages' registry of file and marks the file + * for distribution to the given hosts. + * <b>Note: This class receives ownership of the given collection.</b> + * + * @return the reference to the file, created by the application package + */ + public FileReference sendFileToHosts(String relativePath, Collection<Host> hosts) { + FileReference reference = fileRegistry.addFile(relativePath); + addToFilesToDistribute(reference, hosts); + + return reference; + } + + /** Same as sendFileToHost(relativePath,Collections.singletonList(host) */ + public FileReference sendFileToHost(String relativePath, Host host) { + return sendFileToHosts(relativePath, Arrays.asList(host)); + } + + private void addToFilesToDistribute(FileReference reference, Collection<Host> hosts) { + Set<Host> oldHosts = getHosts(reference); + oldHosts.addAll(hosts); + } + + private Set<Host> getHosts(FileReference reference) { + Set<Host> hosts = filesToHosts.get(reference); + if (hosts == null) { + hosts = new HashSet<>(); + filesToHosts.put(reference, hosts); + } + return hosts; + } + + public FileDistributor(FileRegistry fileRegistry) { + this.fileRegistry = fileRegistry; + } + + /** Returns the files which has been marked for distribution to the given host */ + public Set<FileReference> filesToSendToHost(Host host) { + Set<FileReference> files = new HashSet<>(); + + for (Map.Entry<FileReference,Set<Host>> e : filesToHosts.entrySet()) { + if (e.getValue().contains(host)) { + files.add(e.getKey()); + } + } + return files; + } + + public Set<Host> getTargetHosts() { + Set<Host> hosts = new HashSet<>(); + for (Set<Host> hostSubset: filesToHosts.values()) + hosts.addAll(hostSubset); + return hosts; + } + + public Set<String> getTargetHostnames() { + return getTargetHosts().stream().map(Host::getHostName).collect(Collectors.toSet()); + } + + /** Returns the host which is the source of the files */ + public String fileSourceHost() { + return fileRegistry.fileSourceHost(); + } + + public Set<FileReference> allFilesToSend() { + return filesToHosts.keySet(); + } + + // should only be called during deploy + public void sendDeployedFiles(FileDistribution dbHandler) { + String fileSourceHost = fileSourceHost(); + for (Host host : getTargetHosts()) { + if ( ! host.getHostName().equals(fileSourceHost)) { + dbHandler.sendDeployedFiles(host.getHostName(), filesToSendToHost(host)); + } + } + dbHandler.sendDeployedFiles(fileSourceHost, allFilesToSend()); + dbHandler.limitSendingOfDeployedFilesTo(union(getTargetHostnames(), fileSourceHost)); + dbHandler.removeDeploymentsThatHaveDifferentApplicationId(getTargetHostnames()); + } + + // should only be called during deploy, and only once, since it leads to file distributor + // rescanning all files, which is very expensive ATM (April 2016) + public void reloadDeployFileDistributor(FileDistribution dbHandler) { + dbHandler.reloadDeployFileDistributor(); + } + + private Set<String> union(Set<String> hosts, String... additionalHosts) { + Set<String> result = new HashSet<>(hosts); + result.addAll(asList(additionalHosts)); + return result; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java new file mode 100644 index 00000000000..bfa32c003e0 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributorService.java @@ -0,0 +1,98 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.filedistribution; + +import com.yahoo.config.FileReference; +import com.yahoo.cloud.config.filedistribution.*; +import com.yahoo.config.model.api.FileDistribution; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.admin.FileDistributionOptions; + +import java.io.File; +import java.util.Collection; + +/** + * @author tonytv + */ +public class FileDistributorService extends AbstractService implements + FiledistributorConfig.Producer, + FiledistributorrpcConfig.Producer, + FilereferencesConfig.Producer { + private final static int BASEPORT = 19092; + + private final FileDistributor fileDistributor; + private final FileDistributionOptions fileDistributionOptions; + private final boolean sendAllFiles; + + private Collection<FileReference> getFileReferences() { + if (sendAllFiles) { + return fileDistributor.allFilesToSend(); + } else { + return fileDistributor.filesToSendToHost(getHost()); + } + } + + public FileDistributorService(AbstractConfigProducer parent, + String name, + FileDistributor fileDistributor, + FileDistributionOptions fileDistributionOptions, + boolean sendAllFiles) { + super(parent, name); + portsMeta.on(0).tag("rpc"); + portsMeta.on(1).tag("torrent"); + portsMeta.on(2).tag("http").tag("state"); + setProp("clustertype", "filedistribution"); + setProp("clustername", "admin"); + + this.fileDistributor = fileDistributor; + this.fileDistributionOptions = fileDistributionOptions; + this.sendAllFiles = sendAllFiles; + monitorService(); + } + + @Override + public String getStartupCommand() { + return "exec $ROOT/sbin/filedistributor" + + " --configid " + getConfigId(); + } + + @Override + public boolean getAutostartFlag() { + return true; + } + + @Override + public boolean getAutorestartFlag() { + return true; + } + + public int getPortCount() { + return 3; + } + + @Override + public int getWantedPort() { + return BASEPORT; + } + + @Override + public void getConfig(FiledistributorConfig.Builder builder) { + fileDistributionOptions.getConfig(builder); + builder.torrentport(getRelativePort(1)); + builder.stateport(getRelativePort(2)); + builder.hostname(getHostName()); + builder.filedbpath(FileDistribution.getDefaultFileDBPath().toString()); + } + + @Override + public void getConfig(FiledistributorrpcConfig.Builder builder) { + builder.connectionspec("tcp/" + getHostName() + ":" + getRelativePort(0)); + } + + @Override + public void getConfig(FilereferencesConfig.Builder builder) { + for (FileReference reference : getFileReferences()) { + builder.filereferences(reference.value()); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/package-info.java new file mode 100644 index 00000000000..bc53f9222de --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.model.filedistribution; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/GenericServicesBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/GenericServicesBuilder.java new file mode 100644 index 00000000000..7bbe9d14bc5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/generic/GenericServicesBuilder.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.generic; + +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.config.model.builder.xml.ConfigModelId; +import com.yahoo.vespa.model.generic.builder.DomServiceClusterBuilder; +import org.w3c.dom.Element; + +import java.util.Arrays; +import java.util.List; + +/** + * @author lulf + * @since 5.1 + */ +public class GenericServicesBuilder extends ConfigModelBuilder<GenericServicesModel> { + + public GenericServicesBuilder() { + super(GenericServicesModel.class); + } + + @Override + public List<ConfigModelId> handlesElements() { + return Arrays.asList(ConfigModelId.fromName("service")); + } + + @Override + public void doBuild(GenericServicesModel model, Element spec, ConfigModelContext modelContext) { + String name = spec.getAttribute("name"); + model.addCluster(new DomServiceClusterBuilder(name).build(modelContext.getParentProducer(), spec)); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/GenericServicesModel.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/GenericServicesModel.java new file mode 100644 index 00000000000..c6b2e305658 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/generic/GenericServicesModel.java @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.generic; + +import com.yahoo.config.model.ConfigModel; +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.vespa.model.generic.service.ServiceCluster; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author lulf + * @since 5.1 + */ +public class GenericServicesModel extends ConfigModel { + private final List<ServiceCluster> clusters = new ArrayList<>(); + public GenericServicesModel(ConfigModelContext modelContext) { + super(modelContext); + } + + public void addCluster(ServiceCluster cluster) { + clusters.add(cluster); + } + + public List<ServiceCluster> serviceClusters() { + return clusters; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomModuleBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomModuleBuilder.java new file mode 100644 index 00000000000..f4470f0dc9b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomModuleBuilder.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.generic.builder; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.generic.service.Module; +import org.w3c.dom.Element; + +/** + * Produces sub services for generic services. + */ +public class DomModuleBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Module> { + private final String name; + + public DomModuleBuilder(String name) { + this.name = name; + } + + private void addChildren(Module s, Element subServiceSpec) { + for (Element nodeSpec : XML.getChildren(subServiceSpec, "module")) { + new DomModuleBuilder(nodeSpec.getAttribute("name")).build(s, nodeSpec); + } + } + + @Override + protected Module doBuild(AbstractConfigProducer ancestor, Element subServiceSpec) { + Module s = new Module(ancestor, name); + addChildren(s, subServiceSpec); + return s; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomServiceBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomServiceBuilder.java new file mode 100644 index 00000000000..3cfd85c956b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomServiceBuilder.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.generic.builder; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.generic.service.Service; +import com.yahoo.vespa.model.generic.service.ServiceCluster; +import org.w3c.dom.Element; + +/** +* @author lulf +* @since 5.1 +*/ +public class DomServiceBuilder extends VespaDomBuilder.DomConfigProducerBuilder<Service> { + private final int i; + + public DomServiceBuilder(int i) { + this.i = i; + } + + @Override + protected com.yahoo.vespa.model.generic.service.Service doBuild(AbstractConfigProducer parent, + Element serviceSpec) { + ServiceCluster sc = (ServiceCluster) parent; + com.yahoo.vespa.model.generic.service.Service service = new com.yahoo.vespa.model.generic.service.Service(sc, i + ""); + for (Element subServiceSpec : XML.getChildren(serviceSpec, "module")) { + new DomModuleBuilder(subServiceSpec.getAttribute("name")).build(service, subServiceSpec); + } + return service; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomServiceClusterBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomServiceClusterBuilder.java new file mode 100644 index 00000000000..d537afdd30e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/DomServiceClusterBuilder.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.generic.builder; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.generic.service.ServiceCluster; +import org.w3c.dom.Element; +import java.util.Map; + +/** +* @author lulf +* @since 5.1 +*/ +public class DomServiceClusterBuilder extends VespaDomBuilder.DomConfigProducerBuilder<ServiceCluster> { + + private String name; + + public DomServiceClusterBuilder(String name) { + this.name = name; + } + + @Override + protected ServiceCluster doBuild(AbstractConfigProducer ancestor, Element spec) { + ServiceCluster cluster = new ServiceCluster(ancestor, name, spec.getAttribute("command")); + int nodeIndex = 0; + for (Element nodeSpec : XML.getChildren(spec, "node")) { + com.yahoo.vespa.model.generic.service.Service service = new DomServiceBuilder(nodeIndex).build(cluster, nodeSpec); + + // TODO: Currently creates the config for each service. Should instead build module tree first + // and store them in ServiceCluster. Then have some way of referencing them from each service. + for (Element subServiceSpec : XML.getChildren(spec, "module")) { + String subServiceName = subServiceSpec.getAttribute("name"); + Map<String, AbstractConfigProducer<?>> map = service.getChildren(); + // Add only non-conflicting modules. Does not merge unspecified configs that are specified in root though. + if (!map.containsKey(subServiceName)) + new DomModuleBuilder(subServiceName).build(service, subServiceSpec); + } + nodeIndex++; + } + return cluster; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/package-info.java new file mode 100644 index 00000000000..77679e00d08 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/generic/builder/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.model.generic.builder; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/package-info.java new file mode 100644 index 00000000000..6922e6138f5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/generic/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.model.generic; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/service/Module.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/service/Module.java new file mode 100644 index 00000000000..82190e3344e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/generic/service/Module.java @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.generic.service; + +import com.yahoo.config.model.producer.AbstractConfigProducer; + +/** + * A simple sub service that is essentially just to have a node with a nice name + * in the tree. Could might as well have used an AbstractConfigProducer as well, + * but that makes the code very confusing to read. + * + * @author lulf + */ +public class Module extends AbstractConfigProducer { + + public Module(AbstractConfigProducer parent, String subId) { + super(parent, subId); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/service/Service.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/service/Service.java new file mode 100644 index 00000000000..77ebde12634 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/generic/service/Service.java @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.generic.service; + +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.HostResource; + +/** + * An application specific generic service + * @author vegardh + * + */ +public class Service extends AbstractService { + private static final long serialVersionUID = 1L; + + public Service(ServiceCluster parent, String id) { + super(parent, id); + setProp("clustertype", parent.getName()); + setProp("clustername", parent.getName()); + } + + @Override + public int getPortCount() { + return 0; + } + + @Override + public String getStartupCommand() { + return ((ServiceCluster) getParent()).getCommand(); + } + + private String getClusterName() { + return ((ServiceCluster) getParent()).getName(); + } + + /** + * Different services are represented using same class here, so we must take service name into account too + * + * @param host a host + * @return the index of the host + */ + protected int getIndex(HostResource host) { + int i = 0; + for (com.yahoo.vespa.model.Service s : host.getServices()) { + if (!s.getClass().equals(getClass())) continue; + Service other = (Service)s; + if (s!=this && other.getClusterName().equals(getClusterName())) { + i++; + } + } + return i + 1; + } + + @Override + public String getServiceType() { + return getClusterName(); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/service/ServiceCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/service/ServiceCluster.java new file mode 100644 index 00000000000..b81841442de --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/generic/service/ServiceCluster.java @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.generic.service; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.HostSystem; + +import java.util.ArrayList; +import java.util.Collection; + +/** + * A cluster of nodes running one application specific generic service. These are defined on the top level in the Vespa config + * @author vegardh + * + */ +public class ServiceCluster extends AbstractConfigProducer { + + private static final long serialVersionUID = 1L; + private String command; + private String name; + private HostSystem hostSystem; // A generic cluster can resolve hosts for its nodes + + public ServiceCluster(AbstractConfigProducer parent, String name, String command) { + super(parent, name); + this.command=command; + this.name=name; + } + + public String getName() { + return name; + } + + String getCommand() { + return command; + } + + public Collection<Service> services() { + Collection<Service> ret = new ArrayList<>(); + for (Object child : getChildren().values()) { + if (child instanceof Service) ret.add((Service) child); + } + return ret; + } + + @Override + public HostSystem getHostSystem() { + if (hostSystem!=null) return hostSystem; + return super.getHostSystem(); + } + + /** + * Sets the host system for this. + * @param hostSystem a {@link com.yahoo.vespa.model.HostSystem} + */ + public void setHostSystem(HostSystem hostSystem) { + this.hostSystem = hostSystem; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/generic/service/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/generic/service/package-info.java new file mode 100644 index 00000000000..49683ab3d97 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/generic/service/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.model.generic.service; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/package-info.java new file mode 100644 index 00000000000..4d185d0bb19 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/package-info.java @@ -0,0 +1,188 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + Provides the classes for the Vespa config model framework. + + <p>The {@link com.yahoo.vespa.model.VespaModel VespaModel} class + is the natural starting point. It reads the user-defined + application specification, handles <a + href="#plugin_loading">plugin loading</a> and currently + instantiates one {@link com.yahoo.config.model.ApplicationConfigProducerRoot Vespa} + object. VespaModel is the root node in a tree of {@link +com.yahoo.config.model.producer.AbstractConfigProducer + AbstractConfigProducers} that is built from the structure of the + user's specification. In a future version, the VespaModel can + contain multiple Vespa instances, each built from a separate user + specification (currently called 'services.xml'). + </p> + + <p>Each AbstractConfigProducer in the tree represents an actual + service or another logical unit in the Vespa system. An example of + a logical unit is a cluster that holds a set of services. Each + child class of {@link com.yahoo.config.model.producer.AbstractConfigProducer + AbstractConfigProducer} can contain hard-wired config that should + be delivered to the Vespa unit it represents, and its children. It + can also keep track of the status of the unit. + </p> + + <p>A service that runs on a hardware host is always represented by + an {@link com.yahoo.vespa.model.AbstractService AbstractService} + object, containing the command that will be used to start the + service, which host it is running on, and the ports that it + uses. Each hardware host in the Vespa system is represented by a + {@link com.yahoo.vespa.model.Host Host} object, and the set of + hosts is handled by the {@link com.yahoo.vespa.model.HostSystem + HostSystem}. Each Host is responsible for avoiding port collisions + between services, see <a href="#port_allocation">port + allocation</a>. + </p> + + <h3>Config Generation</h3> + + <p>The method {@link + com.yahoo.vespa.model.VespaModel#getConfig(com.yahoo.config.ConfigInstance.Builder, String) + VespaModel.getConfig} looks up the ConfigProducer with the config + ID that config is requested for. The composition of the actual + config starts from the root node of the ConfigProducer tree, which + is always an instance of the {@link com.yahoo.config.model.ApplicationConfigProducerRoot + Vespa} class, and traverses each level of the tree back down to + the ConfigProducer that got the first call from the root node. + This is handled in such a way + that config from the root node gets the lowest priority, and the + ConfigProducer itself has the highest priority when the same + parameter is given different values in the path down the tree.</p> + + <p>User defined configuration can be embedded in the service setup + file in the application specification. Currently this is done by adding + <config> tags at the desired position in the file named + 'services.xml', where each position corresponds to a + ConfigProducer. These config values have a higher priority than + the default config returned from the same method. However, it can be overridden by the config + from a ConfigProducer at a lower level, both by its getConfig + method and by user defined config. + </p> + + <h4>Example:</h4> + <p> + Say we have a config named 'sample' with an integer parameter + named 'v'. If the VespaModel root node's {@link + com.yahoo.vespa.model.VespaModel#getConfig(com.yahoo.config.ConfigInstance.Builder,String) + getConfig(builder, configid)} method returns a hardcoded value of + 'v=2' for that parameter, this becomes the default value for all + ConfigProducers when asking for the 'sample' config. Now, let's + assume that we need the 'sample' config for a ConfigProducer of + class 'Grandchild', which has a configId + 'grandchild_0'. grandchild_0's parent in the ConfigProducer tree + is a ConfigProducer of class 'Child' and configId 'child_0' which + is a direct child of the Vespa root node: + </p> + + <p>The initial step when retrieving a config is always a call to + {@link + com.yahoo.vespa.model.VespaModel#getConfig(com.yahoo.config.ConfigInstance.Builder,String) + VespaModel.getConfig(builder, configId}. Here, the call + could look like this: + VespaModel.getConfig(builder, "grandchild_0"). + This triggers a call to the {@link + com.yahoo.config.model.producer.AbstractConfigProducer#cascadeConfig(com.yahoo.config.ConfigInstance.Builder)}) AbstractConfigProducer.cascadeConfig} method for + grandchild_0 which calls the same method in child_0, and finally + in the VespaModel root node, where the {@link + com.yahoo.vespa.model.VespaModel#getConfig(com.yahoo.config.ConfigInstance.Builder,String) + getConfig (name, namespace)} method returns the value 'v=2' as + previously mentioned. This value might be overridden on the + traversal back down in the tree, first in child_0, which could + return the value 'v=1'. Now, if the user specification for child_0 + contains the value 'v=0', this overrides the previous values. The + same happens for grandchild_0: if there is a value returned from + the getConfig() method, this overrides the value from child_0, and + if there is a value from the user specification for grandchild_0, + that will always become the final result. + </p> + + + <h3><a name="plugin_loading">Plugin Loading</a></h3> + + <p>Each highest-level node in the setup file from the user's + application specification corresponds to a {@link +com.yahoo.config.model.builder.xml.ConfigModelBuilder ConfigModelBuilder}. The + builders are loaded when the system is started. Each builder produce + a {@link com.yahoo.config.model.ConfigModel ConfigModel}. The model can depend + on other models by having them injected in its constructor. This ensures + that the builders are invoked in the correct order as well. + In its build method, the builder is responsible for building all its + ConfigProducers, and linking them to the parent ConfigProducer + given as input argument. + </p> + + <p>The built models are given to other models that depends on it. + </p> + + <h4>Important notes for plugin developers:</h4> + <ul> + + <li>The constructors of all child classes of {@link +com.yahoo.config.model.producer.AbstractConfigProducer + AbstractConfigProducer} should throw a new 'RuntimeException' upon + errors in xml or other initialization problems. This allows the + exception to be nested upwards, adding valuable information from + each level in the ConfigProducer tree to the error message output + to the user. The exception should contain detailed information + about the error that occurred. + </li> + + <li>The plugins are not allowed to put any constraints on the + contents of the hosts specification file (currently named + 'hosts.xml'), such as demanding special hostnames for + different service types. This file belongs solely to the + vespamodel framework. + </li> + </ul> + + + <h3><a name="port_allocation">Port Allocation</a></h3> + + <p>Each {@link com.yahoo.vespa.model.Host Host} has an available + dynamic port range running from {@link + com.yahoo.vespa.model.HostResource#BASE_PORT BASE_PORT} (currently 19100) + with {@link com.yahoo.vespa.model.HostResource#MAX_PORTS MAX_PORTS} + (currently 799) ports upwards. When an instance of a subclass of + {@link com.yahoo.vespa.model.AbstractService AbstractService} is + assigned to a host, it is given the lowest available base port in + this range. The service owns a continuous port range of {@link + com.yahoo.vespa.model.Service#getPortCount Service.getPortCount} + ports upwards from the base port. + </p> + + <p>The base port for a specific service instance on a host is + decided by {@link + com.yahoo.vespa.model.AbstractService #getInstanceWantedPort + AbstractService.getInstanceWantedPort}. The most important aspects + are described below: + </p> + + <p>It is not possible to reserve a certain port inside the dynamic + range, but a service can specify that it wants a base port outside + the range by overriding the {@link + com.yahoo.vespa.model.Service #getWantedPort Service.getWantedPort} + method. If the service type is required to run with the specified + base port, it must also override the {@link + com.yahoo.vespa.model.Service #requiresWantedPort + Service.requiresWantedPort}. The user specified port number + returned from {@link com.yahoo.vespa.model.Service #getWantedPort + getWantedPort} applies to the first instance of that specific + subclass on each host, and the next instance on the same host + <em>must</em> have its baseport specified by the 'baseport' attribute + in 'services.xml' + </p> + + <p>The user-defined application specification can also give a + required base port for each individual service. Currently this is + done by adding a 'baseport' attribute to the service's tag in the + file named 'hosts.xml'. If the port is not available, an + exception will be thrown. + </p> + +*/ +@ExportPackage +package com.yahoo.vespa.model; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java new file mode 100644 index 00000000000..344c5e16d29 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java @@ -0,0 +1,385 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.routing; + +import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; +import com.yahoo.document.select.DocumentSelector; +import com.yahoo.messagebus.routing.ApplicationSpec; +import com.yahoo.messagebus.routing.HopSpec; +import com.yahoo.messagebus.routing.RouteSpec; +import com.yahoo.messagebus.routing.RoutingTableSpec; +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.ContainerModel; +import com.yahoo.vespa.model.container.docproc.ContainerDocproc; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.content.Content; +import com.yahoo.vespa.model.container.docproc.DocprocChain; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +/** + * This class is responsible for generating all hops and routes for the Document protocol running on message bus. All + * the code within could really be part of {@link Routing}, but it has been partitioned out to allow better readability + * and also more easily maintainable as the number of protocols increase. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public final class DocumentProtocol implements Protocol, DocumentrouteselectorpolicyConfig.Producer { + + private static final String NAME = "document"; + private ApplicationSpec application; + private RoutingTableSpec routingTable; + ConfigModelRepo repo; + + public static String getIndexedRouteName(String configId) { + return configId + "-index"; + } + + public static String getDirectRouteName(String configId) { + return configId + "-direct"; + } + + /** + * Constructs a new document protocol based on the content of the given plugins. + * + * @param plugins The plugins to reflect on. + */ + public DocumentProtocol(ConfigModelRepo plugins) { + application = createApplicationSpec(plugins); + routingTable = createRoutingTable(plugins); + this.repo = plugins; + } + + /** + * Creates a service index based on the plugins loaded. This means to fill the index with all services known by this + * protocol by traversing the plugins. + * + * @param plugins All initialized plugins of the Vespa model. + * @return The index of all known services. + */ + private static ApplicationSpec createApplicationSpec(ConfigModelRepo plugins) { + ApplicationSpec ret = new ApplicationSpec(); + + for (ContentCluster cluster : Content.getContentClusters(plugins)) { + for (com.yahoo.vespa.model.content.Distributor node : cluster.getDistributorNodes().getChildren().values()) { + ret.addService(NAME, node.getConfigId() + "/default"); + } + } + + for (ContainerCluster containerCluster: ContainerModel.containerClusters(plugins)) { + ContainerDocproc containerDocproc = containerCluster.getDocproc(); + if (containerDocproc != null) { + createDocprocChainSpec(ret, + containerDocproc.getChains().allChains().allComponents(), + containerCluster.getContainers()); + } + } + + return ret; + } + + private static void createDocprocChainSpec(ApplicationSpec spec, + List<DocprocChain> docprocChains, + List<Container> containerNodes) { + for (DocprocChain chain: docprocChains) { + for (Container node: containerNodes) + spec.addService(NAME, node.getConfigId() + "/chain." + chain.getComponentId().stringValue()); + } + } + + @Override + public void getConfig(DocumentrouteselectorpolicyConfig.Builder builder) { + for (ContentCluster cluster : Content.getContentClusters(repo)) { + addRoute(cluster.getConfigId(), cluster.getRoutingSelector(), builder); + } + } + + private static void addRoute(String clusterConfigId, String selector, DocumentrouteselectorpolicyConfig.Builder builder) { + try { + new DocumentSelector(selector); + } catch (com.yahoo.document.select.parser.ParseException e) { + throw new IllegalArgumentException("Failed to parse selector '" + selector + "' for route '" + clusterConfigId + + "' in policy 'DocumentRouteSelector'."); + } + DocumentrouteselectorpolicyConfig.Route.Builder routeBuilder = new DocumentrouteselectorpolicyConfig.Route.Builder(); + routeBuilder.name(clusterConfigId); + routeBuilder.selector(selector); + builder.route(routeBuilder); + } + + /** + * This function extrapolates any routes for the document protocol that it can from the vespa model. + * + * @param plugins All initialized plugins of the vespa model. + * @return Routing table for the document protocol. + */ + private static RoutingTableSpec createRoutingTable(ConfigModelRepo plugins) { + // Build simple hops and routes. + List<ContentCluster> content = Content.getContentClusters(plugins); + Collection<ContainerCluster> containerClusters = ContainerModel.containerClusters(plugins); + + RoutingTableSpec table = new RoutingTableSpec(NAME); + addContainerClusterDocprocHops(containerClusters, table); + addContentRouting(content, table); + + // Build the indexing hop if it is possible to derive. + addIndexingHop(content, table); + + // Build the default route if is is possible to derive. + addDefaultRoute(content, containerClusters, table); + + // Return the complete routing table. + simplifyRouteNames(table); + return table; + } + + private static void addContainerClusterDocprocHops(Collection<ContainerCluster> containerClusters, + RoutingTableSpec table) { + + for (ContainerCluster cluster: containerClusters) { + ContainerDocproc docproc = cluster.getDocproc(); + + if (docproc != null) { + String policy = policy(docproc); + + for (DocprocChain chain : docproc.getChains().allChains().allComponents()) { + addChainHop(table, cluster.getConfigId(), policy, chain); + } + } + } + } + + private static void addChainHop(RoutingTableSpec table, String configId, String policy, DocprocChain chain) { + final String selector; + if (policy != null) { + selector = configId + "/" + policy + "/" + chain.getSessionName(); + } else { + selector = "[LoadBalancer:cluster=" + configId + + ";session=" + chain.getSessionName() + "]"; + } + table.addHop(new HopSpec(chain.getServiceName(), selector)); + } + + private static String policy(ContainerDocproc docproc) { + if (docproc.getNumNodesPerClient() > 0) { + return "[SubsetService:" + docproc.getNumNodesPerClient() + "]"; + } else if (docproc.isPreferLocalNode()) { + return "[LocalService]"; + } else { + return null; + } + } + + /** + * Create hops to all configured storage nodes for the Document protocol. The "Distributor" policy resolves its + * recipients using slobrok lookups, so it requires no configured recipients. + * + * @param content The storage model from {@link com.yahoo.vespa.model.VespaModel}. + * @param table The routing table to add to. + */ + private static void addContentRouting(List<ContentCluster> content, RoutingTableSpec table) { + + for (ContentCluster cluster : content) { + RouteSpec spec = new RouteSpec(cluster.getConfigId()); + + if (cluster.getSearch().hasIndexedCluster()) { + table.addRoute(spec.addHop("[MessageType:" + cluster.getConfigId() + "]")); + table.addRoute(new RouteSpec(getIndexedRouteName(cluster.getConfigId())) + .addHop(cluster.getSearch().getIndexed().getIndexingServiceName()) + .addHop("[Content:cluster=" + cluster.getName() + "]")); + table.addRoute(new RouteSpec(getDirectRouteName(cluster.getConfigId())) + .addHop("[Content:cluster=" + cluster.getName() + "]")); + } else { + table.addRoute(spec.addHop("[Content:cluster=" + cluster.getName() + "]")); + } + table.addRoute(new RouteSpec("storage/cluster." + cluster.getName()) + .addHop("route:" + cluster.getConfigId())); + } + } + + /** + * Create the "indexing" hop. This hop contains all non-streaming search clusters as recipients, and the routing + * policy "SearchCluster" will decide which cluster(s) are to receive every document passed through it based on a + * document select string derived from services.xml. + * + * @param table The routing table to add to. + */ + private static void addIndexingHop(List<ContentCluster> content, RoutingTableSpec table) { + if (content.isEmpty()) { + return; + } + HopSpec hop = new HopSpec("indexing", "[DocumentRouteSelector]"); + for (ContentCluster cluster : content) { + hop.addRecipient(cluster.getConfigId()); + } + if (hop.hasRecipients()) { + table.addHop(hop); + } + } + + /** + * Create the "default" route for the Document protocol. This route will be either a route to storage or a route to + * search. Since we will be supporting recovery from storage, storage takes precedence over search when deciding on + * the final target of the default route. If there is an unambigous docproc cluster in the application, the default + * route will pass through this. + * + * @param content The content model from {@link com.yahoo.vespa.model.VespaModel}. + * @param containerClusters a collection of {@link com.yahoo.vespa.model.container.ContainerCluster}s + * @param table The routing table to add to. + */ + private static void addDefaultRoute(List<ContentCluster> content, + Collection<ContainerCluster> containerClusters, + RoutingTableSpec table) { + List<String> hops = new ArrayList<>(); + if (!content.isEmpty()) { + boolean found = false; + for (int i = 0, len = table.getNumHops(); i < len; ++i) { + if (table.getHop(i).getName().equals("indexing")) { + found = true; + break; + } + } + if (found) { + hops.add("indexing"); + } + } + if (!hops.isEmpty()) { + RouteSpec route = new RouteSpec("default"); + String hop = getContainerClustersDocprocHop(containerClusters); + if (hop != null) { + route.addHop(hop); + } + int numHops = hops.size(); + if (numHops == 1) { + route.addHop(hops.get(0)); + } else { + StringBuilder str = new StringBuilder(); + for (int i = 0; i < numHops; ++i) { + str.append(hops.get(i)).append(i < numHops - 1 ? " " : ""); + } + route.addHop("[AND:" + str.toString() + "]"); + } + table.addRoute(route); + } + } + + private static String getContainerClustersDocprocHop(Collection<ContainerCluster> containerClusters) { + DocprocChain result = null; + + for (ContainerCluster containerCluster: containerClusters) { + DocprocChain defaultChain = getDefaultChain(containerCluster.getDocproc()); + if (defaultChain != null) { + if (result != null) + throw new RuntimeException("Only a single default docproc chain is allowed across all container clusters"); + + result = defaultChain; + } + } + + return result == null ? + null: + result.getServiceName(); + } + + private static DocprocChain getDefaultChain(ContainerDocproc docproc) { + return docproc == null ? + null: + docproc.getChains().allChains().getComponent("default"); + } + + /** + * Attempts to simplify all route names by removing prefixing plugin name and whatever comes before the dot (.) in + * the second naming element. This can only be done to those routes that do not share primary name elements with + * other routes (e.g. a search clusters with the same name as a storage cluster). + * + * @param table The routing table whose route names are to be simplified. + */ + private static void simplifyRouteNames(RoutingTableSpec table) { + if (table == null || !table.hasRoutes()) { + return; + } + + // Pass 1: Determine which simplifications are in conflict. + Map<String, Set<String>> simple = new TreeMap<>(); + List<String> broken = new ArrayList<>(); + for (int i = 0, len = table.getNumRoutes(); i < len; ++i) { + String before = table.getRoute(i).getName(); + String after = simplifyRouteName(before); + if (simple.containsKey(after)) { + Set<String> l = simple.get(after); + l.add(before); + if (!(l.contains("content/" + after) && l.contains("storage/cluster." + after) && (l.size() == 2))) { + broken.add(after); + } + } else { + Set<String> l = new HashSet<>(); + l.add(before); + simple.put(after, l); + } + } + + // Pass 2: Simplify all non-conflicting route names by alias. + Set<RouteSpec> alias = new HashSet<>(); + Set<String> unique = new HashSet<>(); + for (int i = 0; i < table.getNumRoutes(); ) { + RouteSpec route = table.getRoute(i); + String before = route.getName(); + String after = simplifyRouteName(before); + if (!before.equals(after)) { + if (!broken.contains(after)) { + if (route.getNumHops() == 1 && route.getHop(0).equals(route.getName())) { + alias.add(new RouteSpec(after).addHop(route.getHop(0))); // full route name is redundant + unique.add(after); + table.removeRoute(i); + continue; // do not increment i + } else { + if (!unique.contains(after)) { + alias.add(new RouteSpec(after).addHop("route:" + before)); + unique.add(after); + } + } + } + } + ++i; + } + for (RouteSpec rs : alias) { + table.addRoute(rs); + } + } + + /** + * Returns a simplified version of the given route name. This method will remove the first component of the name as + * separated by a forward slash, and then remove the first component of the remaining name as separated by a dot. + * + * @param name The route name to simplify. + * @return The simplified route name. + */ + private static String simplifyRouteName(String name) { + String[] foo = name.split("/", 2); + if (foo.length < 2) { + return name; + } + String[] bar = foo[1].split("\\.", 2); + if (bar.length < 2) { + return foo[1]; + } + return bar[1]; + } + + @Override + public ApplicationSpec getApplicationSpec() { + return application; + } + + @Override + public RoutingTableSpec getRoutingTableSpec() { + return routingTable; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/routing/Protocol.java b/config-model/src/main/java/com/yahoo/vespa/model/routing/Protocol.java new file mode 100644 index 00000000000..20b503aac96 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/routing/Protocol.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.routing; + +import com.yahoo.messagebus.routing.ApplicationSpec; +import com.yahoo.messagebus.routing.RoutingTableSpec; + +/** + * This interface defines the necessary api for {@link Routing} to prepare and combine routing tables for all available + * protocols. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public interface Protocol { + + /** + * Returns the specification for the routing table of this protocol. + * + * @return The routing table spec. + */ + public RoutingTableSpec getRoutingTableSpec(); + + /** + * Returns the specification of the application as seen by this protocol. + * + * @return The application spec. + */ + public ApplicationSpec getApplicationSpec(); + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/routing/Routing.java b/config-model/src/main/java/com/yahoo/vespa/model/routing/Routing.java new file mode 100644 index 00000000000..d5a987869e9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/routing/Routing.java @@ -0,0 +1,201 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.routing; + +import com.yahoo.config.model.ConfigModel; +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.messagebus.routing.*; +import com.yahoo.messagebus.MessagebusConfig; +import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; +import java.util.*; + +/** + * This is the routing plugin of the Vespa model. This class is responsible for parsing all routing information given + * explicitly by the user in the optional <routing> element. If there is no such element, only default routes and + * hops will be available. + * + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class Routing extends ConfigModel { + + private final List<String> errors = new ArrayList<>(); + private ApplicationSpec explicitApplication = null; + private RoutingSpec explicitRouting = null; + private List<Protocol> protocols = new ArrayList<>(); + private RoutingSpec derivedRouting; + + public Routing(ConfigModelContext modelContext) { + super(modelContext); + } + + /** + * Sets the application specification to include when verifying the complete routing config. This needs to be + * invoked before {@link #deriveCommonSettings(com.yahoo.config.model.ConfigModelRepo)} to be included. + * + * @param app The application specification to include. + */ + public void setExplicitApplicationSpec(ApplicationSpec app) { + explicitApplication = app; + } + + /** + * Sets the routing specification to include in the derived routing config. This needs to be invoked before + * {@link #deriveCommonSettings(com.yahoo.config.model.ConfigModelRepo)} to be included. + * + * @param routing The routing specification to include. + */ + public void setExplicitRoutingSpec(RoutingSpec routing) { + explicitRouting = routing; + } + + public final List<Protocol> getProtocols() { return protocols; } + + /** + * Derives all routing settings that can be found by inspecting the given plugin container. + * + * @param plugins All initialized plugins of the vespa model. + */ + public void deriveCommonSettings(ConfigModelRepo plugins) { + // Combine explicit routing with protocol derived routing. + ApplicationSpec app = explicitApplication != null ? new ApplicationSpec(explicitApplication) : new ApplicationSpec(); + RoutingSpec routing = explicitRouting != null ? new RoutingSpec(explicitRouting) : new RoutingSpec(); + protocols.clear(); + protocols.add(new DocumentProtocol(plugins)); + for (Protocol protocol : protocols) { + app.add(protocol.getApplicationSpec()); + addRoutingTable(routing, protocol.getRoutingTableSpec()); + } + + // Add default routes where appropriate, and sort content. + for (int i = 0, len = routing.getNumTables(); i < len; ++i) { + RoutingTableSpec table = routing.getTable(i); + if ( ! table.hasRoute("default") && table.getNumRoutes() == 1) { + table.addRoute(new RouteSpec("default").addHop("route:" + table.getRoute(0).getName())); + } + table.sort(); + } + + // Verify and export all produced configs. + errors.clear(); + if (routing.verify(app, errors)) { + this.derivedRouting=routing; + } + } + + public void getConfig(DocumentrouteselectorpolicyConfig.Builder builder) { + for (Protocol protocol : protocols) { + if (protocol instanceof DocumentProtocol) { + ((DocumentProtocol)protocol).getConfig(builder); + } + } + } + + public void getConfig(MessagebusConfig.Builder builder) { + if (derivedRouting==null) { + // The error list should be populated then + return; + } + if (derivedRouting.hasTables()) { + for (int tableIdx = 0, numTables = derivedRouting.getNumTables(); tableIdx < numTables; ++tableIdx) { + RoutingTableSpec table = derivedRouting.getTable(tableIdx); + MessagebusConfig.Routingtable.Builder tableBuilder = new MessagebusConfig.Routingtable.Builder(); + tableBuilder.protocol(table.getProtocol()); + if (table.hasHops()) { + for (int hopIdx = 0, numHops = table.getNumHops(); hopIdx < numHops; ++hopIdx) { + MessagebusConfig.Routingtable.Hop.Builder hopBuilder = new MessagebusConfig.Routingtable.Hop.Builder(); + HopSpec hop = table.getHop(hopIdx); + hopBuilder.name(hop.getName()); + hopBuilder.selector(hop.getSelector()); + if (hop.getIgnoreResult()) { + hopBuilder.ignoreresult(true); + } + if (hop.hasRecipients()) { + for (int recipientIdx = 0, numRecipients = hop.getNumRecipients(); + recipientIdx < numRecipients; ++recipientIdx) + { + hopBuilder.recipient(hop.getRecipient(recipientIdx)); + } + } + tableBuilder.hop(hopBuilder); + } + } + if (table.hasRoutes()) { + for (int routeIdx = 0, numRoutes = table.getNumRoutes(); routeIdx < numRoutes; ++routeIdx) { + MessagebusConfig.Routingtable.Route.Builder routeBuilder = new MessagebusConfig.Routingtable.Route.Builder(); + RouteSpec route = table.getRoute(routeIdx); + routeBuilder.name(route.getName()); + if (route.hasHops()) { + for (int hopIdx = 0, numHops = route.getNumHops(); hopIdx < numHops; ++hopIdx) { + routeBuilder.hop(route.getHop(hopIdx)); + } + } + tableBuilder.route(routeBuilder); + } + } + builder.routingtable(tableBuilder); + } + } + } + + /** + * Adds the given routing table to the given routing spec. This method will not copy hops or routes that are already + * defined in the target table. + * + * @param routing The routing spec to add to. + * @param from The table to copy content from. + */ + private static void addRoutingTable(RoutingSpec routing, RoutingTableSpec from) { + RoutingTableSpec to = getRoutingTable(routing, from.getProtocol()); + if (to != null) { + Set<String> names = new HashSet<>(); + for (int i = 0, len = to.getNumHops(); i < len; ++i) { + names.add(to.getHop(i).getName()); + } + for (int i = 0, len = from.getNumHops(); i < len; ++i) { + HopSpec hop = from.getHop(i); + if (!names.contains(hop.getName())) { + to.addHop(hop); + } + } + + names.clear(); + for (int i = 0, len = to.getNumRoutes(); i < len; ++i) { + names.add(to.getRoute(i).getName()); + } + for (int i = 0, len = from.getNumRoutes(); i < len; ++i) { + RouteSpec route = from.getRoute(i); + if (!names.contains(route.getName())) { + to.addRoute(route); + } + } + } else { + routing.addTable(from); + } + } + + /** + * Returns the routing table from the given routing spec that belongs to the named protocol. + * + * @param routing The routing whose tables to search through. + * @param protocol The name of the protocol whose table to return. + * @return The routing table found, or null. + */ + private static RoutingTableSpec getRoutingTable(RoutingSpec routing, String protocol) { + for (int i = 0, len = routing.getNumTables(); i < len; ++i) { + RoutingTableSpec table = routing.getTable(i); + if (protocol.equals(table.getProtocol())) { + return table; + } + } + return null; + } + + /** + * Returns a list of errors found when preparing the routing configuration. + * + * @return The error list. + */ + public List<String> getErrors() { + return Collections.unmodifiableList(errors); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java new file mode 100644 index 00000000000..a9dd764458c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java @@ -0,0 +1,129 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +import com.yahoo.config.model.producer.UserConfigRepo; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.vespa.config.search.DispatchConfig; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.vespa.configdefinition.IlscriptsConfig; +import java.util.*; + +/** + * Superclass for search clusters. + * + * @author <a href="mailto:boros@yahoo-inc.com">Peter Boros</a> + */ +public abstract class AbstractSearchCluster extends AbstractConfigProducer + implements + DocumentdbInfoConfig.Producer, + IndexInfoConfig.Producer, + IlscriptsConfig.Producer +{ + private Double queryTimeout; + protected String clusterName; + protected int index; + private Double visibilityDelay = 0.0; + private List<String> documentNames = new ArrayList<>(); + + protected List<SearchDefinitionSpec> localSDS = new LinkedList<>(); + + public static final class IndexingMode { + + public static final IndexingMode REALTIME = new IndexingMode("REALTIME"); + public static final IndexingMode STREAMING = new IndexingMode("STREAMING"); + + public static IndexingMode createIndexingMode(String ixm) { + if ("REALTIME".equalsIgnoreCase(ixm)) { + return REALTIME; + } else if ("STREAMING".equalsIgnoreCase(ixm)) { + return STREAMING; + } + return null; + } + + private String name; + + private IndexingMode(String name) { + this.name = name; + } + + public String getName() { return name; } + + public String toString() { + return "indexingmode: " + name; + } + } + + public static final class SearchDefinitionSpec { + private final SearchDefinition searchDefinition; + private final UserConfigRepo userConfigRepo; + + public SearchDefinitionSpec(SearchDefinition searchDefinition, + UserConfigRepo userConfigRepo) { + this.searchDefinition = searchDefinition; + this.userConfigRepo = userConfigRepo; + } + + public SearchDefinition getSearchDefinition() { + return searchDefinition; + } + + public UserConfigRepo getUserConfigs() { + return userConfigRepo; + } + } + + public AbstractSearchCluster(AbstractConfigProducer parent, + String clusterName, int index) { + super(parent, "cluster." + clusterName); + this.clusterName = clusterName; + this.index = index; + } + public void addDocumentNames(SearchDefinition searchDefinition) { + String dName = searchDefinition.getSearch().getDocument().getDocumentName().getName(); + documentNames.add(dName); + } + + /** + * Returns a List with document names used in this search cluster + * @return contained document names + */ + public List<String> getDocumentNames() { return documentNames; } + + public List<SearchDefinitionSpec> getLocalSDS() { + return localSDS; + } + + public String getClusterName() { return clusterName; } + public final String getIndexingModeName() { return getIndexingMode().getName(); } + public final boolean isRealtime() { return getIndexingMode() == IndexingMode.REALTIME; } + public final boolean isStreaming() { return getIndexingMode() == IndexingMode.STREAMING; } + public final AbstractSearchCluster setQueryTimeout(Double to) { + this.queryTimeout=to; + return this; + } + public final AbstractSearchCluster setVisibilityDelay(double delay) { + this.visibilityDelay=delay; + return this; + } + protected abstract IndexingMode getIndexingMode(); + public final Double getVisibilityDelay() { return visibilityDelay; } + public final Double getQueryTimeout() { return queryTimeout; } + public abstract int getRowBits(); + public final void setClusterIndex(int index) { this.index = index; } + public final int getClusterIndex() { return index; } + protected abstract void assureSdConsistent(); + + @Override + public abstract void getConfig(DocumentdbInfoConfig.Builder builder); + @Override + public abstract void getConfig(IndexInfoConfig.Builder builder); + @Override + public abstract void getConfig(IlscriptsConfig.Builder builder); + public abstract void getConfig(RankProfilesConfig.Builder builder); + public abstract void getConfig(AttributesConfig.Builder builder); +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java new file mode 100644 index 00000000000..ebe43bc07db --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Dispatch.java @@ -0,0 +1,226 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +import com.yahoo.vespa.config.search.core.FdispatchrcConfig; +import com.yahoo.vespa.config.search.core.PartitionsConfig; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.application.validation.RestartConfigs; +import com.yahoo.vespa.model.content.SearchCoverage; + +import java.util.ArrayList; +import java.util.List; + +/** + * Represents a dispatch (top-level (tld) or mid-level). + * There must be one or more tld instances in a search cluster. + * + * @author arnej27959 + */ +@RestartConfigs({FdispatchrcConfig.class, PartitionsConfig.class}) +public class Dispatch extends AbstractService implements SearchInterface, + FdispatchrcConfig.Producer, + PartitionsConfig.Producer { + + private static final String TLD_NAME = "tld"; + private static final String DISPATCH_NAME = "dispatch"; + + private static final long serialVersionUID = 1L; + private final DispatchGroup dispatchGroup; + private final NodeSpec nodeSpec; + private final int dispatchLevel; + private final boolean preferLocalRow; + private final boolean isTopLevel; + + private Dispatch(DispatchGroup dispatchGroup, AbstractConfigProducer parent, String subConfigId, + NodeSpec nodeSpec, int dispatchLevel, boolean preferLocalRow, boolean isTopLevel) { + super(parent, subConfigId); + this.dispatchGroup = dispatchGroup; + this.nodeSpec = nodeSpec; + this.dispatchLevel = dispatchLevel; + this.preferLocalRow = preferLocalRow; + this.isTopLevel = isTopLevel; + portsMeta.on(0).tag("rpc").tag("admin"); + portsMeta.on(1).tag("fs4"); + portsMeta.on(2).tag("http").tag("json").tag("health").tag("state"); + setProp("clustertype", "search") + .setProp("clustername", dispatchGroup.getClusterName()) + .setProp("index", nodeSpec.rowId()); + monitorService(); + } + + public static Dispatch createTld(DispatchGroup dispatchGroup, AbstractConfigProducer parent, int rowId) { + return createTld(dispatchGroup, parent, rowId, false); + } + + public static Dispatch createTld(DispatchGroup dispatchGroup, AbstractConfigProducer parent, int rowId, boolean preferLocalRow) { + String subConfigId = TLD_NAME + "." + rowId; + return new Dispatch(dispatchGroup, parent, subConfigId, new NodeSpec(rowId, 0), 0, preferLocalRow, true); + } + + public static Dispatch createTldWithContainerIdInName(DispatchGroup dispatchGroup, AbstractConfigProducer parent, String containerName, int containerIndex) { + String subConfigId = containerName + "." + containerIndex + "." + TLD_NAME + "." + containerIndex; + return new Dispatch(dispatchGroup, parent, subConfigId, new NodeSpec(containerIndex, 0), 0, false, true); + } + + public static Dispatch createDispatchWithStableConfigId(DispatchGroup dispatchGroup, AbstractConfigProducer parent, NodeSpec nodeSpec, int distributionKey, int dispatchLevel) { + String subConfigId = DISPATCH_NAME + "." + distributionKey; + return new Dispatch(dispatchGroup, parent, subConfigId, nodeSpec, dispatchLevel, false, false); + } + + /** + * Override the default service-type + * @return String "topleveldispatch" + */ + public String getServiceType() { + return "topleveldispatch"; + } + + /** + * @return the startup command + */ + public String getStartupCommand() { + return "exec sbin/fdispatch -c $VESPA_CONFIG_ID"; + } + + public int getFrtPort() { return getRelativePort(0); } + public int getDispatchPort() { return getRelativePort(1); } + public @Override int getHealthPort() { return getRelativePort(2); } + + /** + * Twice the default of the number of threads in the container. + * Could have been unbounded if it was not roundrobin, but stack based usage in dispatch. + * We are not putting to much magic into this one as this will disappear as soon as + * dispatch is implemented in Java. + */ + public int getMaxThreads() { return 500*2; } + + public String getHostname() { + return getHost().getHostName(); + } + + @Override + public NodeSpec getNodeSpec() { + return nodeSpec; + } + + public String getDispatcherConnectSpec() { + return "tcp/" + getHost().getHostName() + ":" + getDispatchPort(); + } + + public DispatchGroup getDispatchGroup() { + return dispatchGroup; + } + + @Override + public void getConfig(FdispatchrcConfig.Builder builder) { + builder.ptport(getDispatchPort()). + frtport(getFrtPort()). + healthport(getHealthPort()). + maxthreads(getMaxThreads()); + if (!isTopLevel) { + builder.partition(getNodeSpec().partitionId()); + builder.dispatchlevel(dispatchLevel); + } + } + + @Override + public void getConfig(PartitionsConfig.Builder builder) { + int rowbits = dispatchGroup.getRowBits(); + final PartitionsConfig.Dataset.Builder datasetBuilder = new PartitionsConfig.Dataset.Builder(). + id(0). + refcost(1). + rowbits(rowbits). + numparts(dispatchGroup.getNumPartitions()). + mpp(dispatchGroup.getMinNodesPerColumn()); + if (dispatchGroup.useFixedRowInDispatch()) { + datasetBuilder.querydistribution(PartitionsConfig.Dataset.Querydistribution.Enum.FIXEDROW); + datasetBuilder.maxnodesdownperfixedrow(dispatchGroup.getMaxNodesDownPerFixedRow()); + } + SearchCoverage coverage = dispatchGroup.getSearchCoverage(); + if (coverage != null) { + if (coverage.getMinimum() != null) { + datasetBuilder.minimal_searchcoverage(coverage.getMinimum() * 100); // as percentage + } + if (coverage.getMinWaitAfterCoverageFactor() != null) { + datasetBuilder.higher_coverage_minsearchwait(coverage.getMinWaitAfterCoverageFactor()); + } + if (coverage.getMaxWaitAfterCoverageFactor() != null) { + datasetBuilder.higher_coverage_maxsearchwait(coverage.getMaxWaitAfterCoverageFactor()); + } + } + + Tuning tuning = dispatchGroup.getTuning(); + boolean useLocalNode = false; + if (tuning != null && tuning.dispatch != null) { + useLocalNode = tuning.dispatch.useLocalNode; + } + final List<PartitionsConfig.Dataset.Engine.Builder> allEngines = new ArrayList<>(); + for (SearchInterface searchNode : dispatchGroup.getSearchersIterable()) { + final PartitionsConfig.Dataset.Engine.Builder engineBuilder = new PartitionsConfig.Dataset.Engine.Builder(). + name_and_port(searchNode.getDispatcherConnectSpec()). + rowid(searchNode.getNodeSpec().rowId()). + partid(searchNode.getNodeSpec().partitionId()); + allEngines.add(engineBuilder); + if (preferLocalRow) { + if (getHostname().equals(searchNode.getHostName())) { + engineBuilder.refcost(1); + } else { + engineBuilder.refcost(Integer.MAX_VALUE); + } + } + + if (!useLocalNode || getHostname().equals(searchNode.getHostName())) { + if (useLocalNode) { + engineBuilder.rowid(0); + } + datasetBuilder.engine.add(engineBuilder); + + } + } + //Do not create empty engine list for a dataset if no local search nodes found + if(datasetBuilder.engine.isEmpty() && useLocalNode) { + for(PartitionsConfig.Dataset.Engine.Builder engineBuilder: allEngines) { + datasetBuilder.engine.add(engineBuilder); + } + } + + builder.dataset.add(datasetBuilder); + + if (tuning != null) { + tuning.getConfig(builder); + scaleMaxHitsPerPartitions(builder, tuning); + } + } + + private int getNumLeafNodesInGroup() { + int numSearchers = 0; + for (SearchInterface search : dispatchGroup.getSearchersIterable()) { + if (search instanceof Dispatch) { + numSearchers += ((Dispatch) search).getNumLeafNodesInGroup(); + } else { + numSearchers++; + } + } + if (numSearchers > 0) { + // Divide by number of partitions, otherwise we would count the same leaf node partition number of times. + return numSearchers / dispatchGroup.getNumPartitions(); + } + return 0; + } + + private void scaleMaxHitsPerPartitions(PartitionsConfig.Builder builder, Tuning tuning) { + if (tuning == null || tuning.dispatch == null || tuning.dispatch.maxHitsPerPartition == null) { + return; + } + int numLeafNodes = getNumLeafNodesInGroup(); + for (PartitionsConfig.Dataset.Builder dataset : builder.dataset) { + dataset.maxhitspernode(tuning.dispatch.maxHitsPerPartition * numLeafNodes); + } + } + + /** + * @return the number of ports needed + */ + public int getPortCount() { return 3; } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java new file mode 100644 index 00000000000..e9d01f50fb6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroup.java @@ -0,0 +1,131 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +import com.yahoo.vespa.model.content.SearchCoverage; + +import java.util.*; + +/** + * Class representing a group of @link{SearchInterface} nodes and a set of @link{Dispatch} nodes. + * + * Each @link{Dispatch} has a reference to an instance of this class and use it when producing config. + * + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public class DispatchGroup { + + private final List<Dispatch> dispatchers = new ArrayList<>(); + private final Map<Integer, Map<Integer, SearchInterface> > searchers = new TreeMap<>(); + + final private IndexedSearchCluster sc; + + public DispatchGroup(IndexedSearchCluster sc) { + this.sc = sc; + } + + DispatchGroup addDispatcher(Dispatch dispatch) { + dispatchers.add(dispatch); + return this; + } + + DispatchGroup addSearcher(SearchInterface search) { + Map<Integer, SearchInterface> rows = searchers.get(search.getNodeSpec().partitionId()); + if (rows == null) { + rows = new TreeMap<>(); + rows.put(search.getNodeSpec().rowId(), search); + searchers.put(search.getNodeSpec().partitionId(), rows); + } else { + if (rows.containsKey(search.getNodeSpec().rowId())) { + throw new IllegalArgumentException("Already contains a search node with row id '" + search.getNodeSpec().rowId() + "'"); + } + rows.put(search.getNodeSpec().rowId(), search); + } + return this; + } + + DispatchGroup clearSearchers() { + searchers.clear(); + return this; + } + + List<Dispatch> getDispatchers() { + return Collections.unmodifiableList(dispatchers); + } + + public Iterable getSearchersIterable() { + return new Iterable(searchers); + } + + public int getRowBits() { + return sc.getRowBits(); + } + + public int getNumPartitions() { + return searchers.size(); + } + + public boolean useFixedRowInDispatch() { + return sc.useFixedRowInDispatch(); + } + + public int getMinNodesPerColumn() { + return sc.getMinNodesPerColumn(); + } + + public int getMaxNodesDownPerFixedRow() { + return sc.getMaxNodesDownPerFixedRow(); + } + + SearchCoverage getSearchCoverage() { + return sc.getSearchCoverage(); + } + + Tuning getTuning() { + return sc.getTuning(); + } + + String getClusterName() { + return sc.getClusterName(); + } + + static class Iterator implements java.util.Iterator<SearchInterface> { + private java.util.Iterator<Map<Integer, SearchInterface>> it1; + private java.util.Iterator<SearchInterface> it2; + Iterator(Map<Integer, Map<Integer, SearchInterface> > s) { + it1 = s.values().iterator(); + if (it1.hasNext()) { + it2 = it1.next().values().iterator(); + } + } + @Override + public boolean hasNext() { + if (it2 == null) { + return false; + } + while (!it2.hasNext() && it1.hasNext()) { + it2 = it1.next().values().iterator(); + } + return it2.hasNext(); + } + + @Override + public SearchInterface next() { + return it2.next(); + } + + @Override + public void remove() { + throw new IllegalStateException("'remove' not implemented"); + } + } + + public static class Iterable implements java.lang.Iterable<SearchInterface> { + final Map<Integer, Map<Integer, SearchInterface> > searchers; + Iterable(Map<Integer, Map<Integer, SearchInterface> > searchers) { this.searchers = searchers; } + @Override + public java.util.Iterator<SearchInterface> iterator() { + return new Iterator(searchers); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroupBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroupBuilder.java new file mode 100644 index 00000000000..9b1f63e813e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/DispatchGroupBuilder.java @@ -0,0 +1,110 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.SimpleConfigProducer; +import com.yahoo.vespa.model.content.DispatchSpec; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Class used to build the mid-level dispatch groups in an indexed content cluster. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class DispatchGroupBuilder { + + private final SimpleConfigProducer dispatchParent; + private final DispatchGroup rootDispatch; + private final IndexedSearchCluster searchCluster; + + public DispatchGroupBuilder(SimpleConfigProducer dispatchParent, + DispatchGroup rootDispatch, + IndexedSearchCluster searchCluster) { + this.dispatchParent = dispatchParent; + this.rootDispatch = rootDispatch; + this.searchCluster = searchCluster; + } + + public void build(List<DispatchSpec.Group> groupsSpec, + List<SearchNode> searchNodes) { + Map<Integer, SearchNode> searchNodeMap = buildSearchNodeMap(searchNodes); + for (int partId = 0; partId < groupsSpec.size(); ++partId) { + DispatchSpec.Group groupSpec = groupsSpec.get(partId); + DispatchGroup group = new DispatchGroup(searchCluster); + populateDispatchGroup(group, groupSpec.getNodes(), searchNodeMap, partId); + } + } + + private void populateDispatchGroup(DispatchGroup group, + List<DispatchSpec.Node> nodeList, + Map<Integer, SearchNode> searchNodesMap, + int partId) { + for (int rowId = 0; rowId < nodeList.size(); ++rowId) { + int distributionKey = nodeList.get(rowId).getDistributionKey(); + SearchNode searchNode = searchNodesMap.get(distributionKey); + Dispatch dispatch = buildDispatch(group, new NodeSpec(rowId, partId), distributionKey, searchNode.getHostResource()); + group.addDispatcher(dispatch); + rootDispatch.addSearcher(dispatch); + + // Note: the rowId in this context will be the partId for the underlying search node. + group.addSearcher(buildSearchInterface(searchNode, rowId)); + } + } + + /** + * Builds a mid-level dispatcher with a configId containing the same stable distribution-key as the search node it + * is located on. + * + * If this.dispatchParent has subConfigId 'dispatchers', the config ids of the mid-level + * dispatchers are '../dispatchers/dispatch.X' where X is the distribution-key of the search node. + * + * The dispatch group that will contain this mid-level dispatcher is no longer part of the config producer tree, + * but only contains information about the dispatchers and searchers in this group. + */ + private Dispatch buildDispatch(DispatchGroup group, NodeSpec nodeSpec, int distributionKey, HostResource hostResource) { + Dispatch dispatch = Dispatch.createDispatchWithStableConfigId(group, dispatchParent, nodeSpec, distributionKey, 1); + dispatch.setHostResource(hostResource); + dispatch.initService(); + return dispatch; + } + + private static SearchInterface buildSearchInterface(SearchNode searchNode, int partId) { + searchNode.updatePartition(partId); // ensure that search node uses the same partId as dispatch sees + return new SearchNodeWrapper(new NodeSpec(0, partId), searchNode); + } + + private static Map<Integer, SearchNode> buildSearchNodeMap(List<SearchNode> searchNodes) { + Map<Integer, SearchNode> retval = new LinkedHashMap<>(); + for (SearchNode node : searchNodes) { + retval.put(node.getDistributionKey(), node); + } + return retval; + } + + public static List<DispatchSpec.Group> createDispatchGroups(List<SearchNode> searchNodes, + int numDispatchGroups) { + if (numDispatchGroups > searchNodes.size()) + numDispatchGroups = searchNodes.size(); + + List<DispatchSpec.Group> groupsSpec = new ArrayList<>(); + int numNodesPerGroup = searchNodes.size() / numDispatchGroups; + if (searchNodes.size() % numDispatchGroups != 0) { + numNodesPerGroup += 1; + } + int searchNodeIdx = 0; + for (int i = 0; i < numDispatchGroups; ++i) { + DispatchSpec.Group groupSpec = new DispatchSpec.Group(); + for (int j = 0; j < numNodesPerGroup && searchNodeIdx < searchNodes.size(); ++j) { + groupSpec.addNode(new DispatchSpec.Node(searchNodes.get(searchNodeIdx++).getDistributionKey())); + } + groupsSpec.add(groupSpec); + } + assert(searchNodeIdx == searchNodes.size()); + return groupsSpec; + } +} + diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java new file mode 100644 index 00000000000..d44f81d571b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java @@ -0,0 +1,90 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.vespa.config.search.SummaryConfig; +import com.yahoo.vespa.config.search.IndexschemaConfig; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.SummarymapConfig; +import com.yahoo.vespa.config.search.summary.JuniperrcConfig; +import com.yahoo.vespa.configdefinition.IlscriptsConfig; + +/** + * Represents a document database and the backend configuration needed for this database. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class DocumentDatabase extends AbstractConfigProducer implements + IndexInfoConfig.Producer, + IlscriptsConfig.Producer, + AttributesConfig.Producer, + RankProfilesConfig.Producer, + IndexschemaConfig.Producer, + JuniperrcConfig.Producer, + SummarymapConfig.Producer, + SummaryConfig.Producer { + + private final String inputDocType; + private final DerivedConfiguration derivedCfg; + + public DocumentDatabase(AbstractConfigProducer parent, String inputDocType, DerivedConfiguration derivedCfg) { + super(parent, inputDocType); + this.inputDocType = inputDocType; + this.derivedCfg = derivedCfg; + } + + public String getName() { + return inputDocType; + } + + public String getInputDocType() { + return inputDocType; + } + + public DerivedConfiguration getDerivedConfiguration() { + return derivedCfg; + } + + @Override + public void getConfig(IndexInfoConfig.Builder builder) { + derivedCfg.getIndexInfo().getConfig(builder); + } + + @Override + public void getConfig(IlscriptsConfig.Builder builder) { + derivedCfg.getIndexingScript().getConfig(builder); + } + + @Override + public void getConfig(AttributesConfig.Builder builder) { + derivedCfg.getAttributeFields().getConfig(builder); + } + + @Override + public void getConfig(RankProfilesConfig.Builder builder) { + derivedCfg.getRankProfileList().getConfig(builder); + } + + @Override + public void getConfig(IndexschemaConfig.Builder builder) { + derivedCfg.getIndexSchema().getConfig(builder); + } + + @Override + public void getConfig(JuniperrcConfig.Builder builder) { + derivedCfg.getJuniperrc().getConfig(builder); + } + + @Override + public void getConfig(SummarymapConfig.Builder builder) { + derivedCfg.getSummaryMap().getConfig(builder); + } + + @Override + public void getConfig(SummaryConfig.Builder builder) { + derivedCfg.getSummaries().getConfig(builder); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentSelectionConverter.java b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentSelectionConverter.java new file mode 100644 index 00000000000..942b389fce9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentSelectionConverter.java @@ -0,0 +1,54 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +import com.yahoo.document.select.*; +import com.yahoo.document.select.convert.SelectionExpressionConverter; +import com.yahoo.document.select.parser.ParseException; +import java.util.Map; + + +/** + * @author <a href="mailto:lulf@yahoo-inc.com">Ulf Lilleengen</a> + */ +public class DocumentSelectionConverter { + + private final DocumentSelector selector; + private final Map<String, String> queryExpressionMap; + + public DocumentSelectionConverter(String selection) throws ParseException, UnsupportedOperationException, IllegalArgumentException { + this.selector = new DocumentSelector(selection); + NowCheckVisitor nowChecker = new NowCheckVisitor(); + selector.visit(nowChecker); + if (nowChecker.requiresConversion()) { + SelectionExpressionConverter converter = new SelectionExpressionConverter(); + selector.visit(converter); + this.queryExpressionMap = converter.getQueryMap(); + } else { + this.queryExpressionMap = null; + } + } + + /** + * Transforms the selection into a search query. + * @return A search query representing the selection. + */ + public String getQuery(String documentType) { + if (queryExpressionMap == null) + return null; + if (!queryExpressionMap.containsKey(documentType)) + return null; + return queryExpressionMap.get(documentType); + } + + /** + * Transforms the selection into an inverted search query. + * @return A search query representing the selection. + */ + public String getInvertedQuery(String documentType) { + String query = getQuery(documentType); + if (query == null) + return null; + return query.replaceAll(">", "<"); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedElasticSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedElasticSearchCluster.java new file mode 100644 index 00000000000..88ebef9da86 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedElasticSearchCluster.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +import com.yahoo.config.model.producer.AbstractConfigProducer; + +import java.util.ArrayList; +import java.util.Collections; + +/** + * @author balder + */ +public class IndexedElasticSearchCluster extends IndexedSearchCluster { + + public IndexedElasticSearchCluster(AbstractConfigProducer parent, String clusterName, int index) { + super(parent, clusterName, index); + } + + @Override + public boolean getAllowFeedingWhenNodesDown() { + return true; + } + + @Override + public int getMinNodesPerColumn() { return 0; } + + @Override + protected void assureSdConsistent() { } + + @Override + public int getRowBits() { return 8; } + + @Override + boolean useFixedRowInDispatch() { + for (SearchNode node : getSearchNodes()) { + if (node.getNodeSpec().rowId() > 0) { + return true; + } + } + return false; + } + + @Override + public boolean isElastic() { return true; } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java new file mode 100644 index 00000000000..aa664264ba6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -0,0 +1,390 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.vespa.config.search.DispatchConfig; +import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.searchdefinition.UnproperSearch; +import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.vespa.configdefinition.IlscriptsConfig; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.Service; +import com.yahoo.vespa.model.SimpleConfigProducer; +import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.docproc.DocprocChain; +import com.yahoo.vespa.model.content.DispatchSpec; +import com.yahoo.vespa.model.content.SearchCoverage; + +import java.io.File; +import java.io.IOException; +import java.util.*; +import java.util.logging.Logger; + +/** + * @author <a href="mailto:balder@yahoo-inc.com">Henning Baldersheim</a> + */ +public abstract class IndexedSearchCluster extends SearchCluster + implements + DocumentdbInfoConfig.Producer, + // TODO consider removing, these only produced by UnionConfiguration and DocumentDatabase? + IndexInfoConfig.Producer, + IlscriptsConfig.Producer, + DispatchConfig.Producer +{ + + /** + * Class used to retrieve combined configuration from multiple document databases. + * It is not a {@link com.yahoo.config.ConfigInstance.Producer} of those configs, + * that is handled (by delegating to this) by the {@link IndexedSearchCluster} + * which is the parent to this. This avoids building the config multiple times. + */ + public static class UnionConfiguration + extends AbstractConfigProducer + implements AttributesConfig.Producer { + private final List<DocumentDatabase> docDbs; + + public void getConfig(IndexInfoConfig.Builder builder) { + for (DocumentDatabase docDb : docDbs) { + docDb.getConfig(builder); + } + } + + public void getConfig(IlscriptsConfig.Builder builder) { + for (DocumentDatabase docDb : docDbs) { + docDb.getConfig(builder); + } + } + + @Override + public void getConfig(AttributesConfig.Builder builder) { + for (DocumentDatabase docDb : docDbs) { + docDb.getConfig(builder); + } + } + + public void getConfig(RankProfilesConfig.Builder builder) { + for (DocumentDatabase docDb : docDbs) { + docDb.getConfig(builder); + } + } + + public UnionConfiguration(AbstractConfigProducer parent, List<DocumentDatabase> docDbs) { + super(parent, "union"); + this.docDbs = docDbs; + } + } + + private static final Logger log = Logger.getLogger(IndexedSearchCluster.class.getName()); + + private String indexingClusterName = null; // The name of the docproc cluster to run indexing, by config. + private String indexingChainName = null; + + private DocprocChain indexingChain; // The actual docproc chain indexing for this. + + private Tuning tuning; + private SearchCoverage searchCoverage; + + // This is the document selector string as derived from the subscription tag. + private String routingSelector = null; + private DocumentSelectionConverter selectionConverter = null; + private List<DocumentDatabase> documentDbs = new LinkedList<>(); + private final UnionConfiguration unionCfg; + private int maxNodesDownPerFixedRow = 0; + + private final SimpleConfigProducer dispatchParent; + private final DispatchGroup rootDispatch; + private DispatchSpec dispatchSpec; + private List<SearchNode> searchNodes = new ArrayList<>(); + + /** + * Returns the document selector that is able to resolve what documents are to be routed to this search cluster. + * This string uses the document selector language as defined in the "document" module. + * + * @return The document selector. + */ + public String getRoutingSelector() { + return routingSelector; + } + + public IndexedSearchCluster(AbstractConfigProducer parent, String clusterName, int index) { + super(parent, clusterName, index); + unionCfg = new UnionConfiguration(this, documentDbs); + dispatchParent = new SimpleConfigProducer(this, "dispatchers"); + rootDispatch = new DispatchGroup(this); + } + + @Override + protected IndexingMode getIndexingMode() { return IndexingMode.REALTIME; } + + public final boolean hasExplicitIndexingCluster() { + return indexingClusterName != null; + } + + public final boolean hasExplicitIndexingChain() { + return indexingChainName != null; + } + + /** + * Returns the name of the docproc cluster running indexing for this search cluster. This is derived from the + * services file on initialization, this can NOT be used at runtime to determine indexing chain. When initialization + * is done, the {@link #getIndexingServiceName()} method holds the actual indexing docproc chain object. + * + * @return The name of the docproc cluster associated with this. + */ + public String getIndexingClusterName() { + return hasExplicitIndexingCluster() ? indexingClusterName : getClusterName() + ".indexing"; + } + + public String getIndexingChainName() { + return indexingChainName; + } + + public void setIndexingChainName(String indexingChainName) { + this.indexingChainName = indexingChainName; + } + + /** + * Sets the name of the docproc cluster running indexing for this search cluster. This is for initial configuration, + * and will not reflect the actual indexing chain. See {@link #getIndexingClusterName} for more detail. + * + * @param name The name of the docproc cluster associated with this. + */ + public void setIndexingClusterName(String name) { + indexingClusterName = name; + } + + public String getIndexingServiceName() { + return indexingChain.getServiceName(); + } + + /** + * Sets the docproc chain that will be running indexing for this search cluster. This is set by the + * {@link com.yahoo.vespa.model.content.Content} model during build. + * + * @param chain the chain that is to run indexing for this cluster. + * @return this, to allow chaining. + */ + public AbstractSearchCluster setIndexingChain(DocprocChain chain) { + indexingChain = chain; + return this; + } + + public Dispatch addTld(AbstractConfigProducer tldParent, HostResource hostResource) { + int index = rootDispatch.getDispatchers().size(); + Dispatch tld = Dispatch.createTld(rootDispatch, tldParent, index); + tld.setHostResource(hostResource); + tld.initService(); + rootDispatch.addDispatcher(tld); + return tld; + } + + /** + * Make sure to allocate tld with same id as container (i.e if container cluster name is 'foo', with containers + * with index 0,1,2 the tlds created will get names ../foo.0.tld.0, ../foo.1.tld.1, ../foo.2.tld.2, so that tld config id is + * stable no matter what changes are done to the number of containers in a container cluster + * @param tldParent the indexed search cluster the tlds to add should be connected to + * @param containerCluster the container cluster that should use the tlds created for searching the indexed search cluster above + */ + public void addTldsWithSameIdsAsContainers(AbstractConfigProducer tldParent, ContainerCluster containerCluster) { + for (Container container : containerCluster.getContainers()) { + final String containerSubId = container.getSubId(); + if (!containerSubId.contains(".")) { + throw new RuntimeException("Expected container sub id to be of the form string.number"); + } + int containerIndex = Integer.parseInt(containerSubId.split("\\.")[1]); + final String containerClusterName = containerCluster.getName(); + log.log(LogLevel.DEBUG, "Adding tld with index " + containerIndex + " for content cluster " + this.getClusterName() + + ", container cluster " + containerClusterName + " (container id " + containerSubId + ") on host " + container.getHostResource().getHostName()); + rootDispatch.addDispatcher(createTld(tldParent, container.getHostResource(), containerClusterName, containerIndex)); + } + } + + public Dispatch createTld(AbstractConfigProducer tldParent, HostResource hostResource, String containerClusterName, int containerIndex) { + Dispatch tld = Dispatch.createTldWithContainerIdInName(rootDispatch, tldParent, containerClusterName, containerIndex); + tld.setHostResource(hostResource); + tld.initService(); + return tld; + } + + public DispatchGroup getRootDispatch() { return rootDispatch; } + + public void addSearcher(SearchNode searcher) { + searchNodes.add(searcher); + rootDispatch.addSearcher(searcher); + } + + public List<Dispatch> getTLDs() { return rootDispatch.getDispatchers(); } + + public List<SearchNode> getSearchNodes() { return Collections.unmodifiableList(searchNodes); } + public int getSearchNodeCount() { return searchNodes.size(); } + public SearchNode getSearchNode(int index) { return searchNodes.get(index); } + public void setTuning(Tuning tuning) { + this.tuning = tuning; + } + public Tuning getTuning() { return tuning; } + + public void fillDocumentDBConfig(String documentType, ProtonConfig.Documentdb.Builder builder) { + for (DocumentDatabase sdoc : documentDbs) { + if (sdoc.getName().equals(documentType)) { + fillDocumentDBConfig(sdoc, builder); + return; + } + } + } + + protected void fillDocumentDBConfig(DocumentDatabase sdoc, ProtonConfig.Documentdb.Builder ddbB) { + ddbB.inputdoctypename(sdoc.getInputDocType()) + .configid(sdoc.getConfigId()) + .visibilitydelay(getVisibilityDelay()); + } + + @Override + public void getConfig(DocumentdbInfoConfig.Builder builder) { + for (DocumentDatabase db : documentDbs) { + DocumentdbInfoConfig.Documentdb.Builder docDb = new DocumentdbInfoConfig.Documentdb.Builder(); + docDb.name(db.getName()); + convertSummaryConfig(db, db, docDb); + RankProfilesConfig.Builder rpb = new RankProfilesConfig.Builder(); + db.getConfig(rpb); + addRankProfilesConfig(docDb, new RankProfilesConfig(rpb)); + builder.documentdb(docDb); + } + } + + public void setRoutingSelector(String sel) { + this.routingSelector=sel; + if (this.routingSelector != null) { + try { + this.selectionConverter = new DocumentSelectionConverter(this.routingSelector); + } catch (Exception e) { + throw new IllegalArgumentException("Invalid routing selector: " + e.getMessage()); + } + } + } + /** + * Create default config if not specified by user. + * Accept empty strings as user config - it means that all feeds/documents are accepted. + */ + public void defaultDocumentsConfig() { + if ((routingSelector == null) && !getDocumentNames().isEmpty()) { + Iterator<String> it = getDocumentNames().iterator(); + routingSelector = it.next(); + while (it.hasNext()) { + routingSelector += " or " + it.next(); + } + } + } + protected void deriveAllSearchDefinitions(List<SearchDefinitionSpec> localSearches, + List<com.yahoo.searchdefinition.Search> globalSearches) { + for (SearchDefinitionSpec spec : localSearches) { + com.yahoo.searchdefinition.Search search = spec.getSearchDefinition().getSearch(); + if (!(search instanceof UnproperSearch)) { + DocumentDatabase db = new DocumentDatabase(this, search.getName(), new DerivedConfiguration(search, globalSearches, deployLogger(), getRoot().getDeployState().rankProfileRegistry())); + // TODO: remove explicit adding of user configs when the complete content model is built using builders. + db.mergeUserConfigs(spec.getUserConfigs()); + documentDbs.add(db); + } + } + } + + public List<DocumentDatabase> getDocumentDbs() { + return documentDbs; + } + + public void setSearchCoverage(SearchCoverage searchCoverage) { + this.searchCoverage = searchCoverage; + } + + public SearchCoverage getSearchCoverage() { + return searchCoverage; + } + + @Override + public DerivedConfiguration getSdConfig() { return null; } + + @Override + public void getConfig(IndexInfoConfig.Builder builder) { + unionCfg.getConfig(builder); + } + + @Override + public void getConfig(IlscriptsConfig.Builder builder) { + unionCfg.getConfig(builder); + } + + @Override + public void getConfig(AttributesConfig.Builder builder) { + unionCfg.getConfig(builder); + } + + @Override + public void getConfig(RankProfilesConfig.Builder builder) { + unionCfg.getConfig(builder); + } + + @Override + protected void exportSdFiles(File toDir) throws IOException { } + + public abstract boolean getAllowFeedingWhenNodesDown(); + + public abstract int getMinNodesPerColumn(); + + boolean useFixedRowInDispatch() { + return false; + } + + public abstract boolean isElastic(); + + int getMaxNodesDownPerFixedRow() { + return maxNodesDownPerFixedRow; + } + + public void setMaxNodesDownPerFixedRow(int value) { + maxNodesDownPerFixedRow = value; + } + + public void setDispatchSpec(DispatchSpec dispatchSpec) { + if (dispatchSpec.getNumDispatchGroups() != null) { + this.dispatchSpec = new DispatchSpec.Builder().setGroups + (DispatchGroupBuilder.createDispatchGroups(getSearchNodes(), + dispatchSpec.getNumDispatchGroups())).build(); + } else { + this.dispatchSpec = dispatchSpec; + } + } + + public DispatchSpec getDispatchSpec() { + return dispatchSpec; + } + + public boolean useMultilevelDispatchSetup() { + return dispatchSpec != null && dispatchSpec.getGroups() != null && !dispatchSpec.getGroups().isEmpty(); + } + + public void setupDispatchGroups() { + if (!useMultilevelDispatchSetup()) { + return; + } + rootDispatch.clearSearchers(); + new DispatchGroupBuilder(dispatchParent, rootDispatch, this).build(dispatchSpec.getGroups(), getSearchNodes()); + } + + @Override + public void getConfig(DispatchConfig.Builder builder) { + for (SearchNode node : getSearchNodes()) { + DispatchConfig.Node.Builder nodeBuilder = new DispatchConfig.Node.Builder(); + nodeBuilder.key(node.getDistributionKey()); + nodeBuilder.host(node.getHostName()); + nodeBuilder.port(node.getRpcPort()); + builder.node(nodeBuilder); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocprocChain.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocprocChain.java new file mode 100644 index 00000000000..c6fb6ef4b71 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocprocChain.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +import com.yahoo.collections.Pair; +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.Phase; +import com.yahoo.component.chain.model.ChainSpecification; +import com.yahoo.vespa.configdefinition.SpecialtokensConfig; +import com.yahoo.vespa.model.container.docproc.DocprocChain; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class IndexingDocprocChain extends DocprocChain implements SpecialtokensConfig.Producer { + + public static final String NAME = "indexing"; + private static final List<Phase> phases = new ArrayList<>(2); + + static { + phases.add(new Phase("indexingStart", Collections.<String>emptySet(), Collections.<String>emptySet())); + phases.add(new Phase("indexingEnd", Collections.<String>emptySet(), Collections.<String>emptySet())); + } + + public IndexingDocprocChain() { + super(new ChainSpecification(new ComponentId(NAME), + new ChainSpecification.Inheritance(Collections.<ComponentSpecification>emptySet(), + Collections.<ComponentSpecification>emptySet()), + phases, + Collections.<ComponentSpecification>emptySet()), + new HashMap<>()); + addInnerComponent(new IndexingProcessor()); + } + + @Override + public void getConfig(SpecialtokensConfig.Builder builder) { + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingProcessor.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingProcessor.java new file mode 100644 index 00000000000..89f2ebbe5eb --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingProcessor.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +import com.yahoo.collections.Pair; +import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.component.chain.dependencies.Dependencies; +import com.yahoo.vespa.model.container.docproc.DocumentProcessor; +import com.yahoo.vespa.model.container.docproc.model.DocumentProcessorModel; + +import java.util.Collections; +import java.util.HashMap; + +/** + * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + */ +public class IndexingProcessor extends DocumentProcessor { + + public static final String docprocsBundleSpecification = "docprocs"; + + public IndexingProcessor() { + super(new DocumentProcessorModel(new BundleInstantiationSpecification(new ComponentId(DocumentProcessor.INDEXER), + new ComponentSpecification(DocumentProcessor.INDEXER), + new ComponentSpecification(docprocsBundleSpecification)), + new Dependencies(Collections.<String>emptyList(), + Collections.<String>emptyList(), + Collections.<String>emptyList()), + new HashMap<>())); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/MultilevelDispatchValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/search/MultilevelDispatchValidator.java new file mode 100644 index 00000000000..a068d173ad8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/MultilevelDispatchValidator.java @@ -0,0 +1,89 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +import com.yahoo.vespa.model.content.DispatchSpec; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Class used to validate that multilevel dispatch is correctly setup in an indexed content cluster. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class MultilevelDispatchValidator { + + private final String clusterName; + private final DispatchSpec dispatchSpec; + private final List<SearchNode> searchNodes; + + public MultilevelDispatchValidator(String clusterName, + DispatchSpec dispatchSpec, + List<SearchNode> searchNodes) { + this.clusterName = clusterName; + this.dispatchSpec = dispatchSpec; + this.searchNodes = searchNodes; + } + + public void validate() { + validateThatWeReferenceNodesOnlyOnce(); + validateThatWeReferenceAllNodes(); + validateThatWeUseValidNodeReferences(); + } + + private void validateThatWeReferenceNodesOnlyOnce() { + Set<Integer> distKeys = new HashSet<>(); + for (DispatchSpec.Group group : dispatchSpec.getGroups()) { + for (DispatchSpec.Node node : group.getNodes()) { + int distKey = node.getDistributionKey(); + if (distKeys.contains(distKey)) { + throw new IllegalArgumentException(getErrorMsgPrefix() + "Expected nodes to be referenced only once in dispatch groups, but node with distribution key '" + distKey + "' is referenced multiple times."); + } + distKeys.add(distKey); + } + } + } + + private void validateThatWeReferenceAllNodes() { + Set<Integer> distKeys = createDistributionKeysSet(); + for (DispatchSpec.Group group : dispatchSpec.getGroups()) { + for (DispatchSpec.Node node : group.getNodes()) { + distKeys.remove(node.getDistributionKey()); + } + } + if (!distKeys.isEmpty()) { + Object[] sorted = distKeys.toArray(); + Arrays.sort(sorted); + throw new IllegalArgumentException(getErrorMsgPrefix() + "Expected all nodes to be referenced in dispatch groups, but " + distKeys.size() + + " node(s) with distribution keys " + Arrays.toString(sorted) + " are not referenced."); + } + } + + private void validateThatWeUseValidNodeReferences() { + Set<Integer> distKeys = createDistributionKeysSet(); + for (DispatchSpec.Group group : dispatchSpec.getGroups()) { + for (DispatchSpec.Node node : group.getNodes()) { + int distKey = node.getDistributionKey(); + if (!distKeys.contains(distKey)) { + throw new IllegalArgumentException(getErrorMsgPrefix() + "Expected all node references in dispatch groups to reference existing nodes, " + + "but node with distribution key '" + distKey + "' does not exists."); + } + } + } + } + + private Set<Integer> createDistributionKeysSet() { + Set<Integer> distKeys = new HashSet<>(); + for (SearchNode node : searchNodes) { + distKeys.add(node.getDistributionKey()); + } + return distKeys; + } + + private String getErrorMsgPrefix() { + return "In indexed content cluster '" + clusterName + "': "; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeSpec.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeSpec.java new file mode 100644 index 00000000000..e5759d8c2e5 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeSpec.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +/** + * Represents the row id and partition id of a search interface node. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class NodeSpec { + + private final int rowId; + private final int partitionId; + + public NodeSpec(int rowId, int partitionId) { + if (rowId < 0) { + throw new IllegalArgumentException("RowId(" + rowId + ") can not be below 0"); + } + if (partitionId < 0) { + throw new IllegalArgumentException("PartId(" + partitionId + ") can not be below 0"); + } + this.rowId = rowId; + this.partitionId = partitionId; + } + + public int rowId() { + return rowId; + } + + public int partitionId() { + return partitionId; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java new file mode 100644 index 00000000000..c88b40c8c3d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -0,0 +1,153 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +import com.yahoo.searchdefinition.derived.RawRankProfile; +import com.yahoo.searchdefinition.derived.SummaryMap; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.SummaryConfig; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.vespa.config.search.SummarymapConfig; +import com.yahoo.search.config.IndexInfoConfig; +import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.vespa.configdefinition.IlscriptsConfig; +import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.config.model.producer.AbstractConfigProducer; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +/** + * Represents a search cluster. + * + * @author arnej27959 + */ +public abstract class SearchCluster extends AbstractSearchCluster + implements + DocumentdbInfoConfig.Producer, + IndexInfoConfig.Producer, + IlscriptsConfig.Producer { + + private static final long serialVersionUID = 1L; + + protected SearchCluster(AbstractConfigProducer parent, String clusterName, int index) { + super(parent, clusterName, index); + } + + public void writeFiles(File directory) throws java.io.IOException { + if (!directory.isDirectory() && !directory.mkdirs()) { + throw new java.io.IOException("Cannot create directory: "+ directory); + } + writeSdFiles(directory); + super.writeFiles(directory); + } + + /** + * Must be called after cluster is built, to derive SD configs + * Derives the search definitions from the application package.. + * Also stores the document names contained in the search + * definitions. + */ + public void deriveSearchDefinitions(List<com.yahoo.searchdefinition.Search> global) { + deriveAllSearchDefinitions(getLocalSDS(), global); + } + + @Override + public void getConfig(IndexInfoConfig.Builder builder) { + if (getSdConfig()!=null) getSdConfig().getIndexInfo().getConfig(builder); + } + + @Override + public void getConfig(IlscriptsConfig.Builder builder) { + if (getSdConfig()!=null) getSdConfig().getIndexingScript().getConfig(builder); + } + + @Override + public void getConfig(AttributesConfig.Builder builder) { + if (getSdConfig()!=null) getSdConfig().getAttributeFields().getConfig(builder); + } + + @Override + public void getConfig(RankProfilesConfig.Builder builder) { + if (getSdConfig()!=null) getSdConfig().getRankProfileList().getConfig(builder); + } + + /** + * Converst summary and summary map config to the appropriate information in documentdb + * + * @param summaryConfigProducer the summary config + * @param summarymapConfigProducer the summary map config, or null if none is available + * @param docDb the target document dm config + */ + protected void convertSummaryConfig(SummaryConfig.Producer summaryConfigProducer, + SummarymapConfig.Producer summarymapConfigProducer, + DocumentdbInfoConfig.Documentdb.Builder docDb) { + + SummaryConfig.Builder summaryConfigBuilder = new SummaryConfig.Builder(); + summaryConfigProducer.getConfig(summaryConfigBuilder); + SummaryConfig summaryConfig = new SummaryConfig(summaryConfigBuilder); + + SummarymapConfig summarymapConfig = null; + if (summarymapConfigProducer != null) { + SummarymapConfig.Builder summarymapConfigBuilder = new SummarymapConfig.Builder(); + summarymapConfigProducer.getConfig(summarymapConfigBuilder); + summarymapConfig = new SummarymapConfig(summarymapConfigBuilder); + } + + for (SummaryConfig.Classes sclass : summaryConfig.classes()) { + DocumentdbInfoConfig.Documentdb.Summaryclass.Builder sumClassBuilder = new DocumentdbInfoConfig.Documentdb.Summaryclass.Builder(); + sumClassBuilder. + id(sclass.id()). + name(sclass.name()); + for (SummaryConfig.Classes.Fields field : sclass.fields()) { + DocumentdbInfoConfig.Documentdb.Summaryclass.Fields.Builder fieldsBuilder = new DocumentdbInfoConfig.Documentdb.Summaryclass.Fields.Builder(); + fieldsBuilder.name(field.name()) + .type(field.type()) + .dynamic(isDynamic(field.name(), summarymapConfig)); + sumClassBuilder.fields(fieldsBuilder); + } + docDb.summaryclass(sumClassBuilder); + } + } + + /** Returns true if this is a dynamic summary field */ + private boolean isDynamic(String fieldName, SummarymapConfig summarymapConfig) { + if (summarymapConfig == null) return false; // not know for streaming, but also not used + + for (SummarymapConfig.Override override : summarymapConfig.override()) { + if ( ! fieldName.equals(override.field())) continue; + if (SummaryMap.isDynamicCommand(override.command())) return true; + } + return false; + } + + protected void addRankProfilesConfig(DocumentdbInfoConfig.Documentdb.Builder docDbBuilder, RankProfilesConfig rankProfilesCfg) { + for (RankProfilesConfig.Rankprofile rankProfile : rankProfilesCfg.rankprofile()) { + DocumentdbInfoConfig.Documentdb.Rankprofile.Builder rpB = new DocumentdbInfoConfig.Documentdb.Rankprofile.Builder(); + rpB.name(rankProfile.name()); + rpB.hasSummaryFeatures(containsPropertiesWithPrefix(RawRankProfile.summaryFeatureFefPropertyPrefix, rankProfile.fef())); + rpB.hasRankFeatures(containsPropertiesWithPrefix(RawRankProfile.rankFeatureFefPropertyPrefix, rankProfile.fef())); + docDbBuilder.rankprofile(rpB); + } + } + + private boolean containsPropertiesWithPrefix(String prefix, RankProfilesConfig.Rankprofile.Fef fef) { + for (RankProfilesConfig.Rankprofile.Fef.Property p : fef.property()) { + if (p.name().startsWith(prefix)) + return true; + } + return false; + } + + protected abstract void deriveAllSearchDefinitions(List<SearchDefinitionSpec> localSearches, + List<com.yahoo.searchdefinition.Search> globalSearches); + + public abstract void defaultDocumentsConfig(); + public abstract DerivedConfiguration getSdConfig(); + protected abstract void exportSdFiles(File toDir) throws IOException; + protected final void writeSdFiles(File toDir) throws IOException { + assureSdConsistent(); + exportSdFiles(toDir); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchColumn.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchColumn.java new file mode 100644 index 00000000000..adeee257317 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchColumn.java @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +import com.yahoo.config.model.producer.AbstractConfigProducer; + +import java.util.LinkedList; +import java.util.List; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + */ +public class SearchColumn extends AbstractConfigProducer { + + // All search nodes contained in this column, these also exist as child config producers. + private final List<SearchNode> nodes = new LinkedList<>(); + + public SearchColumn(SearchCluster parent, String name, int index) { + super(parent, name); + } + + /** @return The number of rows in this column. */ + public int getNumRows() { return nodes.size(); } + + /** @return All search nodes contained in this column. */ + public List<SearchNode> getSearchNodes() { return nodes; } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinition.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinition.java new file mode 100644 index 00000000000..2904c561d71 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinition.java @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +import com.yahoo.searchdefinition.Search; + +import java.util.Collection; + +/** + * @author tonytv + */ +public class SearchDefinition { + + private final Search search; + private final String name; + + public static final String fileNameSuffix = ".sd"; + + public Search getSearch() { + return search; + } + + public String getName() { + return name; + } + + public SearchDefinition(String name, Search search) { + this.name = name; + this.search = search; + } + + //Find search definition from a collection with the name specified + public static SearchDefinition findByName(final String searchDefinitionName, Collection<SearchDefinition> searchDefinitions) { + for (SearchDefinition candidate : searchDefinitions) { + if (candidate.getName().equals(searchDefinitionName) ) + return candidate; + } + + return null; + } + + // Used by admin interface + public String getFilename() { + return getName() + fileNameSuffix; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinitionXMLHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinitionXMLHandler.java new file mode 100644 index 00000000000..5ac43b01a5f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchDefinitionXMLHandler.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import org.w3c.dom.Element; + +import java.io.Serializable; +import java.util.List; + +/** + * Represents a single searchdefinition file. + * + * @author arnej27959 + */ +public class SearchDefinitionXMLHandler implements Serializable { + + private String sdName; + + public SearchDefinitionXMLHandler(ModelElement elem) { + sdName = elem.getStringAttribute("name"); + if (sdName == null) { + sdName = elem.getStringAttribute("type"); + } + } + + public String getName() { return sdName; } + + public SearchDefinition getResponsibleSearchDefinition(List<SearchDefinition> searchDefinitions) { + return SearchDefinition.findByName( getName(), searchDefinitions ); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchInterface.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchInterface.java new file mode 100644 index 00000000000..bedc3684ea9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchInterface.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +/** + * This represents an interface for searching. + * It can be both a backend search node or a dispatcher. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public interface SearchInterface { + + NodeSpec getNodeSpec(); + String getDispatcherConnectSpec(); + String getHostName(); + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java new file mode 100644 index 00000000000..9bc968bfd1a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -0,0 +1,288 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.metrics.MetricsmanagerConfig; +import com.yahoo.searchlib.TranslogserverConfig; +import com.yahoo.vespa.config.content.LoadTypeConfig; +import com.yahoo.vespa.config.content.StorFilestorConfig; +import com.yahoo.vespa.config.content.core.StorBucketmoverConfig; +import com.yahoo.vespa.config.content.core.StorCommunicationmanagerConfig; +import com.yahoo.vespa.config.content.core.StorServerConfig; +import com.yahoo.vespa.config.content.core.StorStatusConfig; +import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.config.storage.StorDevicesConfig; +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.admin.MonitoringSystem; +import com.yahoo.vespa.model.application.validation.RestartConfigs; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.content.ContentNode; +import org.w3c.dom.Element; + +import java.util.HashMap; +import java.util.Optional; + +/** + * Represents a search node (proton). + * <p> + * Due to the current disconnect between StorageNode and SearchNode, we have to + * duplicate the set of RestartConfigs classes from StorageNode here, as SearchNode + * runs in a content/storage node context without this being immediately obvious + * in the model. + * + * @author arnej27959 + * @author <a href="mailto:musum@yahoo-inc.com">Harald Musum</a> + */ +@RestartConfigs({ProtonConfig.class, MetricsmanagerConfig.class, TranslogserverConfig.class, + StorDevicesConfig.class, StorFilestorConfig.class, StorBucketmoverConfig.class, + StorCommunicationmanagerConfig.class, StorStatusConfig.class, + StorServerConfig.class, LoadTypeConfig.class}) +public class SearchNode extends AbstractService implements + SearchInterface, + ProtonConfig.Producer, + MetricsmanagerConfig.Producer, + TranslogserverConfig.Producer { + + private static final long serialVersionUID = 1L; + private final boolean flushOnShutdown; + private NodeSpec nodeSpec; + private int distributionKey; + private final String clusterName; + private TransactionLogServer tls; + private AbstractService serviceLayerService; + + public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<SearchNode> { + + private final String name; + private final NodeSpec nodeSpec; + private final String clusterName; + private final ContentNode contentNode; + private final boolean flushOnShutdown; + public Builder(String name, NodeSpec nodeSpec, String clusterName, ContentNode node, boolean flushOnShutdown) { + this.name = name; + this.nodeSpec = nodeSpec; + this.clusterName = clusterName; + this.contentNode = node; + this.flushOnShutdown = flushOnShutdown; + } + + @Override + protected SearchNode doBuild(AbstractConfigProducer ancestor, Element producerSpec) { + return new SearchNode(ancestor, name, contentNode.getDistributionKey(), nodeSpec, clusterName, contentNode, flushOnShutdown); + } + } + + /** + * Creates a SearchNode in elastic mode. + */ + public static SearchNode create(AbstractConfigProducer parent, String name, int distributionKey, NodeSpec nodeSpec, + String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown) { + return new SearchNode(parent, name, distributionKey, nodeSpec, clusterName, serviceLayerService, flushOnShutdown); + } + + private SearchNode(AbstractConfigProducer parent, String name, int distributionKey, NodeSpec nodeSpec, + String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown) { + this(parent, name, nodeSpec, clusterName, flushOnShutdown); + this.distributionKey = distributionKey; + this.serviceLayerService = serviceLayerService; + setPropertiesElastic(clusterName, distributionKey); + } + + private SearchNode(AbstractConfigProducer parent, String name, NodeSpec nodeSpec, String clusterName, boolean flushOnShutdown) { + super(parent, name); + this.nodeSpec = nodeSpec; + this.clusterName = clusterName; + this.flushOnShutdown = flushOnShutdown; + portsMeta.on(0).tag("rpc").tag("rtc").tag("admin").tag("status"); + portsMeta.on(1).tag("fs4"); + portsMeta.on(2).tag("srmp").tag("hack").tag("test"); + portsMeta.on(3).tag("rpc").tag("engines-provider"); + portsMeta.on(4).tag("http").tag("json").tag("health").tag("state"); + // Properties are set in DomSearchBuilder + monitorService(); + } + + private void setPropertiesElastic(String clusterName, int distributionKey) { + setProp("index", distributionKey). + setProp("clustertype", "search"). + setProp("clustername", clusterName); + } + + public String getClusterName() { + return clusterName; + } + + private String getClusterConfigId() { + return getParent().getConfigId(); + } + + private String getBaseDir() { + return Defaults.getDefaults().vespaHome() + "var/db/vespa/search/cluster." + getClusterName() + "/n" + distributionKey; + } + + public void updatePartition(int partitionId) { + nodeSpec = new NodeSpec(nodeSpec.rowId(), partitionId); + } + + @Override + public NodeSpec getNodeSpec() { + return nodeSpec; + } + + /** + * Returns the connection spec string that resolves to this search node. + * + * @return The connection string. + */ + public String getConnectSpec() { + return "tcp/" + getHost().getHostName() + ":" + getRpcPort(); + } + + /** + * Returns the number of ports needed by this service. + * + * @return The number of ports. + */ + @Override + public int getPortCount() { + return 5; + } + + /** + * Returns the RPC port used by this searchnode. + * + * @return The port. + */ + public int getRpcPort() { + return getRelativePort(0); + } + + protected int getSlimeMessagingPort() { + return getRelativePort(2); + } + + /* + * Returns the rpc port used for the engines provider interface. + * @return The port + */ + + public int getPersistenceProviderRpcPort() { + return getRelativePort(3); + } + + @Override + public int getHealthPort() { + return getHttpPort(); + } + + @Override + public String getServiceType() { + return "searchnode"; + } + + public int getDistributionKey() { + return distributionKey; + } + + /** + * Returns the connection spec string that resolves to the dispatcher service + * on this node. + * + * @return The connection string. + */ + public String getDispatcherConnectSpec() { + return "tcp/" + getHost().getHostName() + ":" + getDispatchPort(); + } + + public int getDispatchPort() { + return getRelativePort(1); + } + + public int getHttpPort() { + return getRelativePort(4); + } + + @Override + public void getConfig(TranslogserverConfig.Builder builder) { + tls.getConfig(builder); + } + + @Override + public String toString() { + return getHostName(); + } + + public TransactionLogServer getTransactionLogServer() { + return tls; + } + + public void setTls(TransactionLogServer tls) { + this.tls = tls; + } + + public AbstractService getServiceLayerService() { + return serviceLayerService; + } + + @Override + public String getStartupCommand() { + String startup = getMMapNoCoreEnvVariable() + "exec $ROOT/sbin/proton " + "--identity " + getConfigId(); + if (serviceLayerService != null) { + startup = startup + " --serviceidentity " + serviceLayerService.getConfigId(); + } + return startup; + } + + @Override + public void getConfig(ProtonConfig.Builder builder) { + builder. + ptport(getDispatchPort()). + rpcport(getRpcPort()). + slime_messaging_port(getSlimeMessagingPort()). + rtcspec(getConnectSpec()). + httpport(getHttpPort()). + partition(getNodeSpec().partitionId()). + persistenceprovider(new ProtonConfig.Persistenceprovider.Builder().port(getPersistenceProviderRpcPort())). + clustername(getClusterName()). + basedir(getBaseDir()). + tlsspec("tcp/" + getHost().getHostName() + ":" + getTransactionLogServer().getTlsPort()). + tlsconfigid(getConfigId()). + slobrokconfigid(getClusterConfigId()). + routingconfigid(getClusterConfigId()). + distributionkey(getDistributionKey()); + if (isHostedVespa()) { + // 4 days, 1 hour, 1 minute due to failed nodes can be in failed for 4 days and we want at least one hour more + // to make sure the node failer has done its work + builder.pruneremoveddocumentsage(4 * 24 * 3600 + 3600 + 60); + } + } + + @Override + public HashMap<String, String> getDefaultMetricDimensions() { + HashMap<String, String> dimensions = new HashMap<>(); + if (clusterName != null) { + dimensions.put("clustername", clusterName); + } + return dimensions; + } + + @Override + public void getConfig(MetricsmanagerConfig.Builder builder) { + MonitoringSystem point = getMonitoringService(); + if (point != null) { + builder.snapshot(new MetricsmanagerConfig.Snapshot.Builder(). + periods(point.getIntervalSeconds()).periods(300)); + } + builder.consumer( + new MetricsmanagerConfig.Consumer.Builder(). + name("log"). + tags("logdefault")); + } + + @Override + public Optional<String> getPreShutdownCommand() { + return Optional.ofNullable(flushOnShutdown ? Defaults.getDefaults().vespaHome() + "bin/vespa-proton-cmd " + getRpcPort() + " prepareRestart" : null); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNodeWrapper.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNodeWrapper.java new file mode 100644 index 00000000000..caa918aab97 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNodeWrapper.java @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +public class SearchNodeWrapper implements SearchInterface { + + private final NodeSpec nodeSpec; + private final SearchNode node; + + public SearchNodeWrapper(NodeSpec nodeSpec, SearchNode node) { + this.nodeSpec = nodeSpec; + this.node = node; + } + + @Override + public NodeSpec getNodeSpec() { + return nodeSpec; + } + + @Override + public String getDispatcherConnectSpec() { + return node.getDispatcherConnectSpec(); + } + + @Override + public String getHostName() { + return node.getHostName(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java new file mode 100644 index 00000000000..2d89eb59b58 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -0,0 +1,123 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; +import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.SummaryConfig; +import com.yahoo.vespa.config.search.SummarymapConfig; +import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; +import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig; +import java.io.File; +import java.io.IOException; +import java.util.List; + +/** + * A search cluster of type streaming. + * + * @author balder + * @author vegardh + */ +public class StreamingSearchCluster extends SearchCluster implements + DocumentdbInfoConfig.Producer, + RankProfilesConfig.Producer, + VsmsummaryConfig.Producer, + VsmfieldsConfig.Producer, + SummarymapConfig.Producer, + SummaryConfig.Producer +{ + + private final String storageRouteSpec; + private DerivedConfiguration sdConfig = null; + + public StreamingSearchCluster(AbstractConfigProducer parent, String clusterName, int index, String storageClusterName, String storageRouteSpec) { + super(parent, clusterName, index); + this.storageRouteSpec = storageRouteSpec; + } + + @Override + protected IndexingMode getIndexingMode() { return IndexingMode.STREAMING; } + public final String getStorageRouteSpec() { return storageRouteSpec; } + public int getRowBits() { return 0; } + + @Override + public void getConfig(DocumentdbInfoConfig.Builder builder) { + DocumentdbInfoConfig.Documentdb.Builder docDb = new DocumentdbInfoConfig.Documentdb.Builder(); + String searchName = sdConfig.getSearch().getName(); + docDb.name(searchName); + SummaryConfig.Producer prod = sdConfig.getSummaries(); + convertSummaryConfig(prod, null, docDb); + RankProfilesConfig.Builder rpb = new RankProfilesConfig.Builder(); + sdConfig.getRankProfileList().getConfig(rpb); + addRankProfilesConfig(docDb, new RankProfilesConfig(rpb)); + builder.documentdb(docDb); + } + + @Override + protected void assureSdConsistent() { + if (sdConfig == null) { + throw new IllegalStateException("Search cluster '" + getClusterName() + "' does not have any search definitions"); + } + } + + protected void deriveAllSearchDefinitions(List<SearchDefinitionSpec> local, + List<com.yahoo.searchdefinition.Search> global) { + if (local.size() == 1) { + deriveSingleSearchDefinition(local.get(0).getSearchDefinition().getSearch(), global); + } else if (local.size() > 1){ + throw new IllegalStateException("Logical indexes are not supported: Got " + local.size() + " search definitions, expected 1"); + } + } + private void deriveSingleSearchDefinition(com.yahoo.searchdefinition.Search localSearch, + List<com.yahoo.searchdefinition.Search> globalSearches) { + this.sdConfig = new DerivedConfiguration(localSearch, globalSearches, deployLogger(), getRoot().getDeployState().rankProfileRegistry()); + } + @Override + public DerivedConfiguration getSdConfig() { + return sdConfig; + } + @Override + protected void exportSdFiles(File toDir) throws IOException { + if (sdConfig!=null) { + sdConfig.export(toDir.getCanonicalPath()); + } + } + @Override + public void defaultDocumentsConfig() { } + + @Override + public void getConfig(AttributesConfig.Builder builder) { + if (getSdConfig()!=null) getSdConfig().getAttributeFields().getConfig(builder); + } + + @Override + public void getConfig(VsmsummaryConfig.Builder builder) { + if (getSdConfig()!=null) + if (getSdConfig().getVsmSummary()!=null) + getSdConfig().getVsmSummary().getConfig(builder); + } + + @Override + public void getConfig(VsmfieldsConfig.Builder builder) { + if (getSdConfig()!=null) + if (getSdConfig().getVsmFields()!=null) + getSdConfig().getVsmFields().getConfig(builder); + } + + @Override + public void getConfig(SummarymapConfig.Builder builder) { + if (getSdConfig()!=null) + if (getSdConfig().getSummaryMap()!=null) + getSdConfig().getSummaryMap().getConfig(builder); + } + + @Override + public void getConfig(SummaryConfig.Builder builder) { + if (getSdConfig()!=null) + if (getSdConfig().getSummaries()!=null) + getSdConfig().getSummaries().getConfig(builder); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java b/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java new file mode 100644 index 00000000000..08c9d0ff4ce --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +import com.yahoo.searchlib.TranslogserverConfig; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import org.w3c.dom.Element; + +/** + * @author <a href="musum@yahoo-inc.com">Harald Musum</a> + */ +public class TransactionLogServer extends AbstractService { + + private static final long serialVersionUID = 1L; + + public TransactionLogServer(AbstractConfigProducer searchNode, String clusterName) { + super(searchNode, "transactionlogserver"); + portsMeta.on(0).tag("tls"); + setProp("clustername", clusterName); + setProp("clustertype", "search"); + } + + public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<TransactionLogServer> { + private final String clusterName; + public Builder(String clusterName) { + this.clusterName = clusterName; + } + + @Override + protected TransactionLogServer doBuild(AbstractConfigProducer ancestor, Element producerSpec) { + return new TransactionLogServer(ancestor, clusterName); + } + } + + public int getPortCount() { + return 1; + } + + /** + * Returns the port used by the TLS. + * + * @return The port. + */ + public int getTlsPort() { + return getRelativePort(0); + } + + /** + * Returns the directory used by the TLS. + * + * @return The directory. + */ + private String getTlsDir() { + return "tls"; + } + + public void getConfig(TranslogserverConfig.Builder builder) { + builder.listenport(getTlsPort()).basedir(getTlsDir()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java new file mode 100644 index 00000000000..8493f5414be --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java @@ -0,0 +1,377 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.search; + +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.config.search.core.PartitionsConfig; +import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.model.content.TuningDispatch; + +import static com.yahoo.text.Lowercase.toLowerCase; + +/** + * Class representing the tuning config used for a search cluster. + * Take a look at proton.def and vespa doc for detailed explanations. + * + * @author <a href="mailto:geirst@yahoo-inc.com">Geir Storli</a> + */ +public class Tuning extends AbstractConfigProducer implements PartitionsConfig.Producer, ProtonConfig.Producer { + + public static class Dispatch implements PartitionsConfig.Producer { + + public Integer maxHitsPerPartition = null; + public TuningDispatch.DispatchPolicy policy = TuningDispatch.DispatchPolicy.ROUNDROBIN; + public boolean useLocalNode = false; + public Double minGroupCoverage = null; + public Double minActiveDocsCoverage = null; + + @Override + public void getConfig(PartitionsConfig.Builder builder) { + if (maxHitsPerPartition != null) { + for (PartitionsConfig.Dataset.Builder dataset : builder.dataset) { + dataset.maxhitspernode(maxHitsPerPartition); + } + } + if (minGroupCoverage != null) { + for (PartitionsConfig.Dataset.Builder dataset : builder.dataset) { + dataset.min_group_coverage(minGroupCoverage); + } + } + if (minActiveDocsCoverage != null) { + for (PartitionsConfig.Dataset.Builder dataset : builder.dataset) { + dataset.min_activedocs_coverage(minActiveDocsCoverage); + } + } + for (PartitionsConfig.Dataset.Builder dataset : builder.dataset) { + switch (policy) { + case RANDOM: + dataset.useroundrobinforfixedrow(false); + break; + case ROUNDROBIN: + default: + dataset.useroundrobinforfixedrow(true); + break; + } + } + } + } + + public static class SearchNode implements ProtonConfig.Producer { + + public enum IoType { + NORMAL("NORMAL"), + DIRECTIO("DIRECTIO"), + MMAP("MMAP"); + + public final String name; + + IoType(String name) { + this.name = name; + } + + public static IoType fromString(String name) { + for (IoType type : IoType.values()) { + if (toLowerCase(name).equals(toLowerCase(type.name))) { + return type; + } + } + return NORMAL; + } + } + + public static class RequestThreads implements ProtonConfig.Producer { + public Integer numSearchThreads = null; + public Integer numThreadsPerSearch = null; + public Integer numSummaryThreads = null; + + @Override + public void getConfig(ProtonConfig.Builder builder) { + if (numSearchThreads!=null) builder.numsearcherthreads(numSearchThreads); + if (numThreadsPerSearch!=null) builder.numthreadspersearch(numThreadsPerSearch); + if (numSummaryThreads!=null) builder.numsummarythreads(numSummaryThreads); + } + } + + public static class FlushStrategy implements ProtonConfig.Producer { + public Long totalMaxMemoryGain = null; + public Double totalDiskBloatFactor = null; + public Long componentMaxMemoryGain = null; + public Double componentDiskBloatFactor = null; + public Double componentMaxage = null; + public Long transactionLogMaxEntries = null; + public Long transactionLogMaxSize = null; + + @Override + public void getConfig(ProtonConfig.Builder builder) { + // Here, the config building gets very ugly, because we have to check for null because of autoconversion Long/long etc. + + ProtonConfig.Flush.Memory.Builder memoryBuilder = new ProtonConfig.Flush.Memory.Builder(); + if (totalMaxMemoryGain != null) memoryBuilder.maxmemory(totalMaxMemoryGain); + if (totalDiskBloatFactor != null) memoryBuilder.diskbloatfactor(totalDiskBloatFactor); + if (transactionLogMaxSize != null) memoryBuilder.maxtlssize(transactionLogMaxSize); + + ProtonConfig.Flush.Memory.Each.Builder eachBuilder = new ProtonConfig.Flush.Memory.Each.Builder(); + if (componentMaxMemoryGain != null) eachBuilder.maxmemory(componentMaxMemoryGain); + if (componentDiskBloatFactor != null) eachBuilder.diskbloatfactor(componentDiskBloatFactor); + memoryBuilder.each(eachBuilder); + + ProtonConfig.Flush.Memory.Maxage.Builder maxageBuilder = new ProtonConfig.Flush.Memory.Maxage.Builder(); + if (componentMaxage != null) maxageBuilder.time(componentMaxage); + if (transactionLogMaxEntries != null) maxageBuilder.serial(transactionLogMaxEntries); + memoryBuilder.maxage(maxageBuilder); + + builder. + flush(new ProtonConfig.Flush.Builder(). + memory(memoryBuilder)); + } + } + + public static class Resizing implements ProtonConfig.Producer { + public Integer initialDocumentCount = null; + + @Override + public void getConfig(ProtonConfig.Builder builder) { + if (initialDocumentCount!=null) builder. + grow(new ProtonConfig.Grow.Builder(). + initial(initialDocumentCount)); + } + + } + + public static class Index implements ProtonConfig.Producer { + public static class Io implements ProtonConfig.Producer { + public IoType write = null; + public IoType read = null; + public IoType search = null; + + @Override + public void getConfig(ProtonConfig.Builder builder) { + ProtonConfig.Indexing.Builder indexingB = new ProtonConfig.Indexing.Builder(); + if (write != null) indexingB. + write(new ProtonConfig.Indexing.Write.Builder(). + io(ProtonConfig.Indexing.Write.Io.Enum.valueOf(write.name))); + if (read != null) indexingB. + read(new ProtonConfig.Indexing.Read.Builder(). + io(ProtonConfig.Indexing.Read.Io.Enum.valueOf(read.name))); + if (search != null) builder.search(new ProtonConfig.Search.Builder(). + io(ProtonConfig.Search.Io.Enum.valueOf(search.name))); + builder.indexing(indexingB); + } + + } + + public Io io; + + @Override + public void getConfig(ProtonConfig.Builder builder) { + if (io != null) io.getConfig(builder); + } + } + + public static class Attribute implements ProtonConfig.Producer { + public static class Io implements ProtonConfig.Producer { + public IoType write = null; + + public Io() {} + + @Override + public void getConfig(ProtonConfig.Builder builder) { + if (write != null) builder.attribute(new ProtonConfig.Attribute.Builder(). + write(new ProtonConfig.Attribute.Write.Builder(). + io(ProtonConfig.Attribute.Write.Io.Enum.valueOf(write.name)))); + } + + } + + public Io io; + + @Override + public void getConfig(ProtonConfig.Builder builder) { + if (io != null) io.getConfig(builder); + } + + } + + public static class Summary implements ProtonConfig.Producer { + public static class Io { + public IoType write = null; + public IoType read = null; + + public void getConfig(ProtonConfig.Summary.Builder builder) { + if (write != null) builder. + write(new ProtonConfig.Summary.Write.Builder(). + io(ProtonConfig.Summary.Write.Io.Enum.valueOf(write.name))); + if (read != null) builder. + read(new ProtonConfig.Summary.Read.Builder(). + io(ProtonConfig.Summary.Read.Io.Enum.valueOf(read.name))); + } + } + + public static class Store { + public static class Compression { + public enum Type { + NONE("NONE"), + LZ4("LZ4"); + + public final String name; + + Type(String name) { + this.name = name; + } + public static Type fromString(String name) { + for (Type type : Type.values()) { + if (toLowerCase(name).equals(toLowerCase(type.name))) { + return type; + } + } + return NONE; + } + } + public Type type = null; + public Integer level = null; + + public void getConfig(ProtonConfig.Summary.Cache.Compression.Builder compression) { + if (type != null) compression.type(ProtonConfig.Summary.Cache.Compression.Type.Enum.valueOf(type.name)); + if (level != null) compression.level(level); + } + + public void getConfig(ProtonConfig.Summary.Log.Chunk.Compression.Builder compression) { + if (type != null) compression.type(ProtonConfig.Summary.Log.Chunk.Compression.Type.Enum.valueOf(type.name)); + if (level != null) compression.level(level); + } + } + + public static class Component { + public Long maxSize = null; + public Long maxEntries = null; + public Long initialEntries = null; + public Compression compression = null; + private final boolean outputInt; + + public Component() { + this.outputInt = false; + } + + public Component(boolean outputInt) { + this.outputInt = outputInt; + } + + public void getConfig(ProtonConfig.Summary.Cache.Builder cache) { + if (outputInt) { + if (maxSize!=null) cache.maxbytes(maxSize.intValue()); + if (maxEntries!=null) cache.initialentries(maxEntries.intValue()); + if (initialEntries!=null) cache.initialentries(initialEntries.intValue()); + } else { + if (maxSize!=null) cache.maxbytes(maxSize); + if (maxEntries!=null) cache.initialentries(maxEntries); + if (initialEntries!=null) cache.initialentries(initialEntries); + } + if (compression != null) { + ProtonConfig.Summary.Cache.Compression.Builder compressionB = new ProtonConfig.Summary.Cache.Compression.Builder(); + compression.getConfig(compressionB); + cache.compression(compressionB); + } + } + + public void getConfig(ProtonConfig.Summary.Log.Chunk.Builder chunk) { + if (outputInt) { + if (maxSize!=null) chunk.maxbytes(maxSize.intValue()); + if (maxEntries!=null) chunk.maxentries(maxEntries.intValue()); + } else { + throw new IllegalStateException("Fix this, chunk does not have long types"); + } + if (compression != null) { + ProtonConfig.Summary.Log.Chunk.Compression.Builder compressionB = new ProtonConfig.Summary.Log.Chunk.Compression.Builder(); + compression.getConfig(compressionB); + chunk.compression(compressionB); + } + } + } + + public static class LogStore { + public Long maxFileSize = null; + public Double maxDiskBloatFactor = null; + public Integer numThreads = null; + public Component chunk = null; + public Double minFileSizeFactor = null; + + public void getConfig(ProtonConfig.Summary.Log.Builder log) { + if (maxFileSize!=null) log.maxfilesize(maxFileSize); + if (maxDiskBloatFactor!=null) log.maxdiskbloatfactor(maxDiskBloatFactor); + if (minFileSizeFactor!=null) log.minfilesizefactor(minFileSizeFactor); + if (numThreads != null) log.numthreads(numThreads); + if (chunk != null) { + ProtonConfig.Summary.Log.Chunk.Builder chunkB = new ProtonConfig.Summary.Log.Chunk.Builder(); + chunk.getConfig(chunkB); + log.chunk(chunkB); + } + } + } + + public Component cache; + public LogStore logStore; + + public void getConfig(ProtonConfig.Summary.Builder builder) { + if (cache != null) { + ProtonConfig.Summary.Cache.Builder cacheB=new ProtonConfig.Summary.Cache.Builder(); + cache.getConfig(cacheB); + builder.cache(cacheB); + + } + if (logStore != null) { + ProtonConfig.Summary.Log.Builder logB = new ProtonConfig.Summary.Log.Builder(); + logStore.getConfig(logB); + builder.log(logB); + } + } + } + + public Io io; + public Store store; + + @Override + public void getConfig(ProtonConfig.Builder builder) { + ProtonConfig.Summary.Builder summary = new ProtonConfig.Summary.Builder(); + if (io != null) io.getConfig(summary); + if (store != null) store.getConfig(summary); + builder.summary(summary); + } + } + + public RequestThreads threads = null; + public FlushStrategy strategy = null; + public Resizing resizing = null; + public Index index = null; + public Attribute attribute = null; + public Summary summary = null; + + @Override + public void getConfig(ProtonConfig.Builder builder) { + if (threads != null) threads.getConfig(builder); + if (strategy != null) strategy.getConfig(builder); + if (resizing != null) resizing.getConfig(builder); + if (index != null) index.getConfig(builder); + if (attribute != null) attribute.getConfig(builder); + if (summary != null) summary.getConfig(builder); + } + } + + public Dispatch dispatch; + public SearchNode searchNode; + + public Tuning(AbstractConfigProducer parent) { + super(parent, "tuning"); + } + + @Override + public void getConfig(PartitionsConfig.Builder builder) { + if (dispatch != null) { + dispatch.getConfig(builder); + } + } + + @Override + public void getConfig(ProtonConfig.Builder builder) { + if (searchNode != null) searchNode.getConfig(builder); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/Duration.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/Duration.java new file mode 100644 index 00000000000..1c51f021fef --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/Duration.java @@ -0,0 +1,69 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.utils; + +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Parses a string on the form: + * + * [numbers]\s*[unit]? + * + * Where numbers is a double, and unit is one of: + * d - days + * m - minutes + * s - seconds + * ms - milliseconds + * + * Default is seconds. + */ +public class Duration { + private static Pattern pattern = Pattern.compile("([0-9\\.]+)\\s*([a-z]+)?"); + private static Map<String, Integer> unitMultiplier = new HashMap<>(); + static { + unitMultiplier.put("s", 1000); + unitMultiplier.put("d", 1000 * 3600 * 24); + unitMultiplier.put("ms", 1); + unitMultiplier.put("m", 60 * 1000); + unitMultiplier.put("h", 60 * 60 * 1000); + } + + long value; + + public Duration(String value) { + Matcher matcher = pattern.matcher(value); + + if (!matcher.matches()) { + throw new IllegalArgumentException("Illegal duration format: " + value); + } + + double num = Double.parseDouble(matcher.group(1)); + String unit = matcher.group(2); + + if (unit != null) { + Integer multiplier = unitMultiplier.get(unit); + if (multiplier == null) { + throw new IllegalArgumentException("Unknown time unit: " + unit + " in time value " + value); + } + + num *= multiplier; + } else { + num *= 1000; + } + + this.value = (long)num; + } + + public double getSeconds() { + return value / 1000.0; + } + + public long getMilliSeconds() { + return value; + } + +} + diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java new file mode 100644 index 00000000000..12de5a3fbb0 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java @@ -0,0 +1,149 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.utils; + +import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.producer.UserConfigRepo; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.*; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.config.ConfigDefinition.DefaultValued; +import com.yahoo.vespa.model.AbstractService; + +import java.io.Serializable; +import java.util.*; + +/** + * Utility methods for sending files to a collection of nodes. + * + * @author gjoranv + * @since 5.1.9 + */ +public class FileSender implements Serializable { + + /** + * Send the given file to all given services. + * + * @param relativePath The path to the file, relative to the app pkg. + * @param services The services to send the file to. + * @return The file reference that the file was given, never null. + * @throws IllegalStateException if services is empty. + */ + public static FileReference sendFileToServices(String relativePath, + Collection<? extends AbstractService> services) { + if (services.isEmpty()) { + throw new IllegalStateException("'sendFileToServices called for empty services!" + + " - This should never happen!"); + } + FileReference fileref = null; + for (AbstractService service : services) { + // The same reference will be returned from each call. + fileref = service.sendFile(relativePath); + } + return fileref; + } + + /** + * Sends all user configured files for a producer to all given services. + */ + public static <PRODUCER extends AbstractConfigProducer<?>> + void sendUserConfiguredFiles(PRODUCER producer, Collection<? extends AbstractService> services, DeployLogger logger) { + if (services.isEmpty()) + return; + + UserConfigRepo userConfigs = producer.getUserConfigs(); + Map<String, FileReference> sentFiles = new HashMap<>(); + for (ConfigDefinitionKey key : userConfigs.configsProduced()) { + ConfigPayloadBuilder builder = userConfigs.get(key); + try { + sendUserConfiguredFiles(builder, sentFiles, services, key, logger); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Unable to send files for " + key, e); + } + } + } + + private static void sendUserConfiguredFiles(ConfigPayloadBuilder builder, + Map<String, FileReference> sentFiles, + Collection<? extends AbstractService> services, + ConfigDefinitionKey key, DeployLogger logger) { + ConfigDefinition configDefinition = builder.getConfigDefinition(); + if (configDefinition == null) { + // TODO: throw new IllegalArgumentException("Not able to find config definition for " + builder); + logger.log(LogLevel.WARNING, "Not able to find config definition for " + key + ". Will not send files for this config"); + return; + } + // Inspect fields at this level + sendEntries(builder, sentFiles, services, configDefinition.getFileDefs()); + sendEntries(builder, sentFiles, services, configDefinition.getPathDefs()); + + // Inspect arrays + for (Map.Entry<String, ConfigDefinition.ArrayDef> entry : configDefinition.getArrayDefs().entrySet()) { + if (isFileOrPathArray(entry)) { + ConfigPayloadBuilder.Array array = builder.getArray(entry.getKey()); + sendFileEntries(array.getElements(), sentFiles, services); + } + } + // Maps + for (Map.Entry<String, ConfigDefinition.LeafMapDef> entry : configDefinition.getLeafMapDefs().entrySet()) { + if (isFileOrPathMap(entry)) { + ConfigPayloadBuilder.MapBuilder map = builder.getMap(entry.getKey()); + sendFileEntries(map.getElements(), sentFiles, services); + } + } + + // Inspect inner fields + for (String name : configDefinition.getStructDefs().keySet()) { + sendUserConfiguredFiles(builder.getObject(name), sentFiles, services, key, logger); + } + for (String name : configDefinition.getInnerArrayDefs().keySet()) { + ConfigPayloadBuilder.Array array = builder.getArray(name); + for (ConfigPayloadBuilder element : array.getElements()) { + sendUserConfiguredFiles(element, sentFiles, services, key, logger); + } + } + for (String name : configDefinition.getStructMapDefs().keySet()) { + ConfigPayloadBuilder.MapBuilder map = builder.getMap(name); + for (ConfigPayloadBuilder element : map.getElements()) { + sendUserConfiguredFiles(element, sentFiles, services, key, logger); + } + } + + } + + private static boolean isFileOrPathMap(Map.Entry<String, ConfigDefinition.LeafMapDef> entry) { + String mapType = entry.getValue().getTypeSpec().getType(); + return ("file".equals(mapType) || "path".equals(mapType)); + } + + private static boolean isFileOrPathArray(Map.Entry<String, ConfigDefinition.ArrayDef> entry) { + String arrayType = entry.getValue().getTypeSpec().getType(); + return ("file".equals(arrayType) || "path".equals(arrayType)); + } + + private static void sendEntries(ConfigPayloadBuilder builder, Map<String, FileReference> sentFiles, Collection<? extends AbstractService> services, Map<String, ? extends DefaultValued<String>> entries) { + for (String name : entries.keySet()) { + ConfigPayloadBuilder fileEntry = builder.getObject(name); + if (fileEntry.getValue() == null) { + throw new IllegalArgumentException("Unable to send file for field '" + name + "'. Invalid config value " + fileEntry.getValue()); + } + sendFileEntry(fileEntry, sentFiles, services); + } + } + + private static void sendFileEntries(Collection<ConfigPayloadBuilder> builders, Map<String, FileReference> sentFiles, Collection<? extends AbstractService> services) { + for (ConfigPayloadBuilder builder : builders) { + sendFileEntry(builder, sentFiles, services); + } + } + + private static void sendFileEntry(ConfigPayloadBuilder builder, Map<String, FileReference> sentFiles, Collection<? extends AbstractService> services) { + String path = builder.getValue(); + FileReference reference = sentFiles.get(path); + if (reference == null) { + reference = sendFileToServices(path, services); + sentFiles.put(path, reference); + } + builder.setValue(reference.value()); + } +}
\ No newline at end of file diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/FreezableMap.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/FreezableMap.java new file mode 100644 index 00000000000..a05008cc9a0 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/FreezableMap.java @@ -0,0 +1,91 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.utils; + +import java.util.*; + +/** + * Delegates to a map that can be froozen. + * Not thread safe. + * @author tonytv + */ +public class FreezableMap<K, V> implements Map<K, V> { + private boolean frozen = false; + private Map<K, V> map; + + //TODO: review the use of unchecked. + @SuppressWarnings("unchecked") + public FreezableMap(Class<LinkedHashMap> mapClass) { + try { + map = mapClass.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + public int size() { + return map.size(); + } + + public boolean isEmpty() { + return map.isEmpty(); + } + + public boolean containsKey(Object o) { + return map.containsKey(o); + } + + public boolean containsValue(Object o) { + return map.containsValue(o); + } + + public V get(Object o) { + return map.get(o); + } + + public V put(K key, V value) { + return map.put(key, value); + } + + public V remove(Object o) { + return map.remove(o); + } + + public void putAll(Map<? extends K, ? extends V> map) { + this.map.putAll(map); + } + + public void clear() { + map.clear(); + } + + public Set<K> keySet() { + return map.keySet(); + } + + public Collection<V> values() { + return map.values(); + } + + public Set<Entry<K, V>> entrySet() { + return map.entrySet(); + } + + public boolean equals(Object o) { + return map.equals(o); + } + + public int hashCode() { + return map.hashCode(); + } + + public void freeze() { + if (frozen) + throw new RuntimeException("The map has already been frozen."); + frozen = true; + map = Collections.unmodifiableMap(map); + } + + public boolean isFrozen() { + return frozen; + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/internal/ReflectionUtil.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/internal/ReflectionUtil.java new file mode 100644 index 00000000000..2bc7d2dfbe7 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/internal/ReflectionUtil.java @@ -0,0 +1,114 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.utils.internal; + +import com.yahoo.config.ChangesRequiringRestart; +import com.yahoo.config.ConfigInstance; +import com.yahoo.vespa.config.ConfigKey; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * Utility class containing static methods for retrievinig information about the config producer tree. + * + * @author lulf + * @author bjorncs + * @since 5.1 + */ +public final class ReflectionUtil { + + private ReflectionUtil() { + } + + /** + * Returns a set of all the configs produced by a given producer. + * + * @param iface The config producer or interface to check for producers. + * @param configId The config id to use when creating keys. + * @return A set of config keys. + */ + public static Set<ConfigKey<?>> configsProducedByInterface(Class<?> iface, String configId) { + Set<ConfigKey<?>> ret = new LinkedHashSet<>(); + if (isConcreteProducer(iface)) { + ret.add(createConfigKeyFromInstance(iface.getEnclosingClass(), configId)); + } + for (Class<?> parentIface : iface.getInterfaces()) { + ret.addAll(configsProducedByInterface(parentIface, configId)); + } + return ret; + } + + /** + * Determines if the config class contains the methods required for detecting config value changes + * between two config instances. + */ + public static boolean hasRestartMethods(Class<? extends ConfigInstance> configClass) { + try { + configClass.getDeclaredMethod("containsFieldsFlaggedWithRestart"); + configClass.getDeclaredMethod("getChangesRequiringRestart", configClass); + return true; + } catch (NoSuchMethodException e) { + return false; + } + } + + /** + * Determines if the config definition for the given config class contains key-values flagged with restart. + */ + public static boolean containsFieldsFlaggedWithRestart(Class<? extends ConfigInstance> configClass) { + try { + Method m = configClass.getDeclaredMethod("containsFieldsFlaggedWithRestart"); + m.setAccessible(true); + return (boolean) m.invoke(null); + } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + /** + * Compares the config instances and lists any differences that will require service restart. + * @param from The previous config. + * @param to The new config. + * @return An object describing the difference. + */ + public static ChangesRequiringRestart getChangesRequiringRestart(ConfigInstance from, ConfigInstance to) { + Class<?> clazz = from.getClass(); + if (!clazz.equals(to.getClass())) { + throw new IllegalArgumentException(String.format("%s != %s", clazz, to.getClass())); + } + try { + Method m = clazz.getDeclaredMethod("getChangesRequiringRestart", clazz); + m.setAccessible(true); + return (ChangesRequiringRestart) m.invoke(from, to); + } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + private static ConfigKey<?> createConfigKeyFromInstance(Class<?> configInstClass, String configId) { + try { + String defName = ConfigInstance.getDefName(configInstClass); + String defNamespace = ConfigInstance.getDefNamespace(configInstClass); + return new ConfigKey<>(defName, configId, defNamespace); + } catch (IllegalArgumentException | SecurityException e) { + throw new RuntimeException(e); + } + } + + private static boolean classIsConfigInstanceProducer(Class<?> clazz) { + return clazz.getName().equals(ConfigInstance.Producer.class.getName()); + } + + private static boolean isConcreteProducer(Class<?> producerInterface) { + boolean parentIsConfigInstance = false; + for (Class<?> ifaceParent : producerInterface.getInterfaces()) { + if (classIsConfigInstanceProducer(ifaceParent)) { + parentIsConfigInstance = true; + } + } + return (ConfigInstance.Producer.class.isAssignableFrom(producerInterface) && parentIsConfigInstance && !classIsConfigInstanceProducer(producerInterface)); + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/package-info.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/package-info.java new file mode 100644 index 00000000000..c19058af4fa --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.model.utils; + +import com.yahoo.osgi.annotation.ExportPackage; |