diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /config-model/src/main/java/com/yahoo/config |
Publish
Diffstat (limited to 'config-model/src/main/java/com/yahoo/config')
41 files changed, 3953 insertions, 0 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java new file mode 100644 index 00000000000..1ef0ed6ec0a --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -0,0 +1,279 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +import com.yahoo.cloud.config.*; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Version; +import com.yahoo.vespa.config.content.LoadTypeConfig; +import com.yahoo.cloud.config.ModelConfig.Hosts; +import com.yahoo.cloud.config.ModelConfig.Hosts.Services; +import com.yahoo.cloud.config.ModelConfig.Hosts.Services.Ports; +import com.yahoo.cloud.config.log.LogdConfig; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.document.DocumenttypesConfig; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; +import com.yahoo.messagebus.MessagebusConfig; +import com.yahoo.vespa.configmodel.producers.DocumentManager; +import com.yahoo.vespa.configmodel.producers.DocumentTypes; +import com.yahoo.vespa.documentmodel.DocumentModel; +import com.yahoo.vespa.model.*; +import com.yahoo.vespa.model.admin.Admin; +import com.yahoo.vespa.model.clients.Clients; +import com.yahoo.vespa.model.content.cluster.ContentCluster; +import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; +import com.yahoo.vespa.model.routing.Routing; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + + +/** + * This is the parent of all ConfigProducers in the system resulting from configuring an application. + * + * @author gjoranv + */ +public class ApplicationConfigProducerRoot extends AbstractConfigProducer<AbstractConfigProducer<?>> implements CommonConfigsProducer { + + private final DocumentModel documentModel; + private Routing routing = null; + // The ConfigProducers contained in this Vespa. (configId->producer) + Map<String, ConfigProducer> id2producer = new LinkedHashMap<>(); + private Admin admin = null; + private HostSystem hostSystem = null; + private final Version vespaVersion; + private final ApplicationId applicationId; + + /** + * Creates and initializes a new Vespa from the service config file + * in the given application directory. + * + * @param parent The parent, usually VespaModel + * @param name The name, used as configId + * @param documentModel DocumentModel to serve global document config from. + */ + public ApplicationConfigProducerRoot(AbstractConfigProducer parent, String name, DocumentModel documentModel, Version vespaVersion, ApplicationId applicationId) { + super(parent, name); + this.documentModel = documentModel; + this.vespaVersion = vespaVersion; + this.applicationId = applicationId; + } + + /** + * @return an unmodifiable copy of the set of configIds in this VespaModel. + */ + public Set<String> getConfigIds() { + return Collections.unmodifiableSet(id2producer.keySet()); + } + + /** + * Returns the ConfigProducer with the given id, or null if no such + * configId exists. + * + * @param configId The configId, e.g. "search.0/tld.0" + * @return ConfigProducer with the given configId + */ + public ConfigProducer getConfigProducer(String configId) { + return id2producer.get(configId); + } + + /** + * Returns the Service with the given id, or null if no such + * configId exists or if it belongs to a non-Service ConfigProducer. + * + * @param configId The configId, e.g. "search.0/tld.0" + * @return Service with the given configId + */ + public Service getService(String configId) { + ConfigProducer cp = getConfigProducer(configId); + if (cp == null || !(cp instanceof Service)) { + return null; + } + return (Service) cp; + } + + /** + * Adds the descendant (at any depth level), so it can be looked up + * on configId in the Map. + * + * @param descendant The configProducer descendant to add + */ + // TODO: Make protected if this moves to the same package as AbstractConfigProducer + public void addDescendant(AbstractConfigProducer descendant) { + id2producer.put(descendant.getConfigId(), descendant); + } + + /** + * Prepares the model for start. The {@link VespaModel} calls + * this methods after it has loaded this and all plugins have been loaded and + * their initialize() methods have been called. + * + * @param plugins All initialized plugins of the vespa model. + */ + public void prepare(ConfigModelRepo plugins) { + if (routing != null) { + routing.deriveCommonSettings(plugins); + } + } + + public void setupAdmin(Admin admin) { + this.admin = admin; + } + + public void setupRouting(ConfigModelRepo configModels) { + if (admin != null) { + Routing routing = configModels.getRouting(); + if (routing == null) { + routing = new Routing(ConfigModelContext.createFromParentAndId(configModels, this, "routing")); + configModels.add(routing); + } + this.routing = routing; + } + } + + @Override + public void getConfig(DocumentmanagerConfig.Builder builder) { + new DocumentManager().produce(documentModel, builder); + } + + @Override + public void getConfig(DocumenttypesConfig.Builder builder) { + new DocumentTypes().produce(documentModel, builder); + } + + @Override + public void getConfig(DocumentrouteselectorpolicyConfig.Builder builder) { + if (routing != null) { + routing.getConfig(builder); + } + } + + @Override + public void getConfig(MessagebusConfig.Builder builder) { + if (routing != null) { + routing.getConfig(builder); + } + } + + @Override + public void getConfig(LogdConfig.Builder builder) { + if (admin != null) { + admin.getConfig(builder); + } + } + + @Override + public void getConfig(SlobroksConfig.Builder builder) { + if (admin != null) { + admin.getConfig(builder); + } + } + + @Override + public void getConfig(ZookeepersConfig.Builder builder) { + if (admin != null) { + admin.getConfig(builder); + } + } + + @Override + public void getConfig(LoadTypeConfig.Builder builder) { + VespaModel model = (VespaModel) getRoot(); + Clients clients = model.getClients(); + if (clients != null) { + clients.getConfig(builder); + } + } + + @Override + public void getConfig(ClusterListConfig.Builder builder) { + VespaModel model = (VespaModel) getRoot(); + for (ContentCluster cluster : model.getContentClusters().values()) { + ClusterListConfig.Storage.Builder storage = new ClusterListConfig.Storage.Builder(); + storage.name(cluster.getName()); + storage.configid(cluster.getConfigId()); + builder.storage(storage); + } + } + + @Override + public void getConfig(ModelConfig.Builder builder) { + builder.vespaVersion(vespaVersion.toSerializedForm()); + for (HostResource modelHost : getHostSystem().getHosts()) { + builder.hosts(new Hosts.Builder() + .name(modelHost.getHostName()) + .services(getServices(modelHost)) + ); + } + } + + private List<Services.Builder> getServices(HostResource modelHost) { + List<Services.Builder> ret = new ArrayList<>(); + for (Service modelService : modelHost.getServices()) { + ret.add(new Services.Builder() + .name(modelService.getServiceName()) + .type(modelService.getServiceType()) + .configid(modelService.getConfigId()) + .clustertype(modelService.getServicePropertyString("clustertype", "")) + .clustername(modelService.getServicePropertyString("clustername", "")) + .index(Integer.parseInt(modelService.getServicePropertyString("index", "999999"))) + .ports(getPorts(modelService)) + ); + } + return ret; + } + + private List<Ports.Builder> getPorts(Service modelService) { + List<Ports.Builder> ret = new ArrayList<>(); + PortsMeta portsMeta = modelService.getPortsMeta(); + for (int i = 0; i < portsMeta.getNumPorts(); i++) { + ret.add(new Ports.Builder() + .number(modelService.getRelativePort(i)) + .tags(getPortTags(portsMeta, i)) + ); + } + return ret; + } + + public static String getPortTags(PortsMeta portsMeta, int portNumber) { + StringBuilder sb = new StringBuilder(); + boolean firstTag = true; + for (String s : portsMeta.getTagsAt(portNumber)) { + if (!firstTag) { + sb.append(" "); + } else { + firstTag = false; + } + sb.append(s); + } + return sb.toString(); + } + + public void setHostSystem(HostSystem hostSystem) { + this.hostSystem = hostSystem; + } + + @Override + public HostSystem getHostSystem() { + return hostSystem; + } + + public FileDistributionConfigProducer getFileDistributionConfigProducer() { + return admin.getFileDistributionConfigProducer(); + } + + public Admin getAdmin() { + return admin; + } + + @Override + public void getConfig(ApplicationIdConfig.Builder builder) { + builder.tenant(applicationId.tenant().value()); + builder.application(applicationId.application().value()); + builder.instance(applicationId.instance().value()); + } +} diff --git a/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java b/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java new file mode 100644 index 00000000000..228c74480c4 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +import com.yahoo.cloud.config.ApplicationIdConfig; +import com.yahoo.vespa.config.content.LoadTypeConfig; +import com.yahoo.cloud.config.log.LogdConfig; +import com.yahoo.cloud.config.SlobroksConfig; +import com.yahoo.cloud.config.ClusterListConfig; +import com.yahoo.cloud.config.ZookeepersConfig; +import com.yahoo.cloud.config.ModelConfig; +import com.yahoo.document.DocumenttypesConfig; +import com.yahoo.document.config.DocumentmanagerConfig; +import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; +import com.yahoo.messagebus.MessagebusConfig; + + +/** + * This interface describes the configs that are produced by the model producer root. + * + * @author lulf + * @since 5.1 + */ +public interface CommonConfigsProducer extends DocumentmanagerConfig.Producer, + DocumenttypesConfig.Producer, + MessagebusConfig.Producer, + DocumentrouteselectorpolicyConfig.Producer, + LogdConfig.Producer, + SlobroksConfig.Producer, + ZookeepersConfig.Producer, + LoadTypeConfig.Producer, + ClusterListConfig.Producer, + ModelConfig.Producer, + ApplicationIdConfig.Producer { +} diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java new file mode 100644 index 00000000000..10342d44914 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +/** + * A config model is an abstract representation of a subsystem, which is used + * by the builder of that subsystem to derive a config producer tree for the subsystem, and by other + * builders to access information about the subsystem for config production at a suitable abstraction level. + * + * @author gjoranv + * @author bratseth + * @author lulf + */ +public abstract class ConfigModel { + + private final String id; + + /** + * Constructs a new config model given a context. + * + * @param modelContext The model context. + */ + public ConfigModel(ConfigModelContext modelContext) { + super(); + this.id = modelContext.getProducerId(); + } + + /** Returns the id of this model */ + public final String getId() { return id; } + + /** + * Initializes this model. All inter-model independent initialization + * is done by implementing this method. + * The model will be made available to dependent models by the framework when this returns. + * <p> + * TODO: Remove this method, as this is now done by the model builders. + * + * This default implementation does nothing. + * + * @param configModelRepo The ConfigModelRepo of the VespaModel + * @deprecated This will go away in the next Vespa major release. Instead, inject the models you depend on + * in your config model constructor. + */ + public void initialize(ConfigModelRepo configModelRepo) { return; } + + /** + * Prepares this model to start serving config requests, possibly using properties of other models. + * The framework will call this method after models have been built. The model + * should finalize its configurations that depend on other models in this step. + * + * This default implementation does nothing. + * + * @param configModelRepo The ConfigModelRepo of the system model + */ + public void prepare(ConfigModelRepo configModelRepo) { return; } + + /** + * <p>Returns whether this model must be maintained in memory for serving config requests. + * Models which are used to amend other models at build time should override this to return false.</p> + * + * <p>This default implementation returns true.</p> + */ + public boolean isServing() { return true; } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java new file mode 100644 index 00000000000..c375d8e28fa --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.producer.AbstractConfigProducer; + +/** + * This class contains a context that is passed to a model builder, and can be used to retrieve the application package, + * logger etc. + * + * @author lulf + * @since 5.1 + */ +public class ConfigModelContext { + + private final AbstractConfigProducer producer; + private final String producerId; + private final DeployState deployState; + private final ConfigModelRepoAdder configModelRepoAdder; + + private ConfigModelContext(DeployState deployState, ConfigModelRepoAdder configModelRepoAdder, + AbstractConfigProducer producer, String producerId) { + this.deployState = deployState; + this.configModelRepoAdder = configModelRepoAdder; + this.producer = producer; + this.producerId = producerId; + } + + public ApplicationPackage getApplicationPackage() { return deployState.getApplicationPackage(); } + public String getProducerId() { return producerId; } + public AbstractConfigProducer getParentProducer() { return producer; } + public DeployLogger getDeployLogger() { return deployState.getDeployLogger(); } + public DeployState getDeployState() { return deployState; } + + /** Returns write access to the config model repo, or null (only) if this is improperly initialized during testing */ + public ConfigModelRepoAdder getConfigModelRepoAdder() { return configModelRepoAdder; } + + /** + * Create a new context with a different parent, but with the same id and application package. + * + * @param newParent The parent to use for the new context. + * @return A new context. + */ + public ConfigModelContext modifyParent(AbstractConfigProducer newParent) { + return ConfigModelContext.create(deployState, configModelRepoAdder, newParent, producerId); + } + + /** + * Create an application context from a parent producer and an id. + * @param deployState The global deploy state for this model. + * @param parent The parent to be used for the config model. + * @param id The id to be used for the config model. + * @return An model context that can be passed to a model. + */ + public static ConfigModelContext create(DeployState deployState, ConfigModelRepoAdder configModelRepoAdder, + AbstractConfigProducer parent, String id) { + return new ConfigModelContext(deployState, configModelRepoAdder, parent, id); + } + + /** + * Create an application context from a parent producer and an id. + * @param parent The parent to be used for the config model. + * @param id The id to be used for the config model. + * @return An model context that can be passed to a model. + */ + public static ConfigModelContext createFromParentAndId(ConfigModelRepoAdder configModelRepoAdder, AbstractConfigProducer parent, String id) { + return create(parent.getRoot().getDeployState(), configModelRepoAdder, parent, id); + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelInstanceFactory.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelInstanceFactory.java new file mode 100644 index 00000000000..a9b3fc73eb3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelInstanceFactory.java @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +/** + * Interface for factories of config models. + * + * @author lulf + * @since 5.1 + */ +public interface ConfigModelInstanceFactory<MODEL extends ConfigModel> { + + /** + * Create an instance of {@link com.yahoo.config.model.ConfigModel} given the input context. + * + * @param context The {@link com.yahoo.config.model.ConfigModelContext} to use. + * @return an instance of {@link com.yahoo.config.model.ConfigModel} + */ + MODEL createModel(ConfigModelContext context); + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRegistry.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRegistry.java new file mode 100644 index 00000000000..2735b9f7fdf --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRegistry.java @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.config.model.builder.xml.ConfigModelId; + +import java.util.Collection; +import java.util.Collections; + +/** + * A resolver of implementations of named config models. + * Registries may be chained in a chain of command. + * + * @author bratseth + */ +public abstract class ConfigModelRegistry { + + private final ConfigModelRegistry chained; + + public ConfigModelRegistry() { + this(new EmptyTerminalRegistry()); + } + + /** Creates a config model class registry which forwards unresolved requests to the argument instance */ + public ConfigModelRegistry(ConfigModelRegistry chained) { + this.chained=chained; + } + + /** + * Returns the builders this id resolves to both in this and any chained registry. + * + * @return the resolved config model builders, or an empty list (never null) if none + */ + public abstract Collection<ConfigModelBuilder> resolve(ConfigModelId id); + + public ConfigModelRegistry chained() { return chained; } + + /** An empty registry which does not support chaining */ + private static class EmptyTerminalRegistry extends ConfigModelRegistry { + + public EmptyTerminalRegistry() { + super(null); + } + + @Override + public Collection<ConfigModelBuilder> resolve(ConfigModelId id) { + return Collections.emptyList(); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java new file mode 100644 index 00000000000..fa46c33a97f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java @@ -0,0 +1,256 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.config.model.builder.xml.ConfigModelId; +import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.config.model.graph.ModelGraphBuilder; +import com.yahoo.config.model.graph.ModelNode; +import com.yahoo.config.model.provision.HostsXmlProvisioner; +import com.yahoo.log.LogLevel; +import com.yahoo.path.Path; +import com.yahoo.text.XML; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.builder.VespaModelBuilder; +import com.yahoo.vespa.model.clients.Clients; +import com.yahoo.vespa.model.content.Content; +import com.yahoo.vespa.model.routing.Routing; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.io.Reader; +import java.io.Serializable; +import java.io.StringReader; +import java.util.*; +import java.util.logging.Logger; + +/** + * A collection of config model instances owned by a system model + * + * @author gjoranv + */ +public class ConfigModelRepo implements ConfigModelRepoAdder, Serializable, Iterable<ConfigModel> { + + private static final long serialVersionUID = 1L; + + private static final Logger log = Logger.getLogger(ConfigModelRepo.class.getPackage().toString()); + + private final Map<String,ConfigModel> configModelMap = new TreeMap<>(); + private final List<ConfigModel> configModels = new ArrayList<>(); + + /** + * Returns a config model for a given id + * + * @param id the id of the model to return + * @return the model, or none if a model with this id is not present in this + */ + public ConfigModel get(String id) { + return configModelMap.get(id); + } + + /** Adds a new config model instance in this */ + @Override + public void add(ConfigModel model) { + configModelMap.put(model.getId(), model); + configModels.add(model); + } + + /** Returns the models in this as an iterator */ + public Iterator<ConfigModel> iterator() { + return configModels.iterator(); + } + + /** Returns a read-only view of the config model instances of this */ + public Map<String,ConfigModel> asMap() { return Collections.unmodifiableMap(configModelMap); } + + /** Initialize part 1.: Reads the config models used in the application package. */ + public void readConfigModels(DeployState deployState, VespaModelBuilder builder, + ApplicationConfigProducerRoot root, ConfigModelRegistry configModelRegistry) throws IOException, SAXException { + Element userServicesElement = getServicesFromApp(deployState.getApplicationPackage()); + readConfigModels(root, userServicesElement, deployState, configModelRegistry); + builder.postProc(root, this); + } + + private Element getServicesFromApp(ApplicationPackage applicationPackage) throws IOException, SAXException { + try (Reader servicesFile = applicationPackage.getServices()) { + return getServicesFromReader(servicesFile); + } + } + + /** + * If the top level is <services>, it contains a list of services elements, + * otherwise, the top level tag is a single service. + */ + private List<Element> getServiceElements(Element servicesRoot) { + if (servicesRoot.getTagName().equals("services")) + return XML.getChildren(servicesRoot); + + List<Element> singleServiceList = new ArrayList<>(1); + singleServiceList.add(servicesRoot); + return singleServiceList; + } + + /** + * Creates all the config models specified in the given XML element and + * passes their respective XML node as parameter. + * + * @param root The Root to set as parent for all plugins + * @param servicesRoot XML root node of the services file + */ + private void readConfigModels(ApplicationConfigProducerRoot root, Element servicesRoot, DeployState deployState, ConfigModelRegistry configModelRegistry) throws IOException, SAXException { + final Map<ConfigModelBuilder, List<Element>> model2Element = new LinkedHashMap<>(); + ModelGraphBuilder graphBuilder = new ModelGraphBuilder(); + + final List<Element> children = getServiceElements(servicesRoot); + + if (XML.getChild(servicesRoot, "admin") == null) + children.add(getImplicitAdmin(deployState)); + + children.addAll(getPermanentServices(deployState)); + + for (Element servicesElement : children) { + String tagName = servicesElement.getTagName(); + if (tagName.equals("config")) continue; // TODO: Remove on Vespa 6 + if (tagName.equals("cluster")) continue; // TODO: Remove on Vespa 6 + if ((tagName.equals("clients")) && deployState.isHostedVespa()) + throw new IllegalArgumentException("<" + tagName + "> is not allowed when running Vespa in a hosted environment"); + + String tagVersion = servicesElement.getAttribute("version"); + ConfigModelId xmlId = ConfigModelId.fromNameAndVersion(tagName, tagVersion); + + Collection<ConfigModelBuilder> builders = configModelRegistry.resolve(xmlId); + + if (builders.isEmpty()) + throw new RuntimeException("Could not resolve tag <" + tagName + " version=\"" + tagVersion + "\"> to a config model component"); + + for (ConfigModelBuilder builder : builders) { + if ( ! model2Element.containsKey(builder)) { + model2Element.put(builder, new ArrayList<>()); + graphBuilder.addBuilder(builder); + } + model2Element.get(builder).add(servicesElement); + } + } + + for (ModelNode node : graphBuilder.build().topologicalSort()) + buildModels(node, deployState, root, model2Element.get(node.builder)); + for (ConfigModel model : configModels) + model.initialize(ConfigModelRepo.this); + } + + private Collection<Element> getPermanentServices(DeployState deployState) throws IOException, SAXException { + List<Element> permanentServices = new ArrayList<>(); + Optional<ApplicationPackage> applicationPackage = deployState.getPermanentApplicationPackage(); + if (applicationPackage.isPresent()) { + ApplicationFile file = applicationPackage.get().getFile(Path.fromString(ApplicationPackage.PERMANENT_SERVICES)); + if (file.exists()) { + try (Reader reader = file.createReader()) { + Element permanentServicesRoot = getServicesFromReader(reader); + permanentServices.addAll(getServiceElements(permanentServicesRoot)); + } + } + } + return permanentServices; + } + + private Element getServicesFromReader(Reader reader) throws IOException, SAXException { + Document doc = XmlHelper.getDocumentBuilder().parse(new InputSource(reader)); + return doc.getDocumentElement(); + } + + private void buildModels(ModelNode node, DeployState deployState, AbstractConfigProducer parent, List<Element> elements) { + for (Element servicesElement : elements) { + ConfigModel model = buildModel(node, deployState, parent, servicesElement); + if (model.isServing()) + add(model); + } + } + + private ConfigModel buildModel(ModelNode node, DeployState deployState, AbstractConfigProducer parent, Element servicesElement) { + ConfigModelBuilder builder = node.builder; + ConfigModelContext context = ConfigModelContext.create(deployState, this, parent, getIdString(servicesElement)); + return builder.build(node, servicesElement, context); + } + + private static String getIdString(Element spec) { + String idString = XmlHelper.getIdString(spec); + if (idString == null || idString.isEmpty()) { + idString = spec.getTagName(); + } + return idString; + } + + /** + * Initialize part 2.: + * Prepare all config models for starting. Must be called after plugins are loaded and frozen. + */ + public void prepareConfigModels() { + for (ConfigModel model : configModels) { + model.prepare(ConfigModelRepo.this); + } + } + + @SuppressWarnings("unchecked") + public <T extends ConfigModel> List<T> getModels(Class<T> modelClass) { + List<T> modelsOfModelClass = new ArrayList<>(); + + for (ConfigModel model : asMap().values()) { + if (modelClass.isInstance(model)) + modelsOfModelClass.add((T)model); + } + return modelsOfModelClass; + } + + public Clients getClients() { + for (ConfigModel m : configModels) { + if (m instanceof Clients) { + return (Clients)m; + } + } + return null; + } + + public Routing getRouting() { + for (ConfigModel m : configModels) { + if (m instanceof Routing) { + return (Routing)m; + } + } + return null; + } + + public Content getContent() { + for (ConfigModel m : configModels) { + if (m instanceof Content) { + return (Content)m; + } + } + return null; + } + + // TODO: Doctoring on the XML is the wrong level for this. We should be able to mark a model as default instead -Jon + private static Element getImplicitAdmin(DeployState deployState) throws IOException, SAXException { + final boolean hostedVespa = deployState.isHostedVespa(); + String defaultAdminElement = hostedVespa ? getImplicitAdminV4() : getImplicitAdminV2(); + log.log(LogLevel.DEBUG, "No <admin> defined, using " + defaultAdminElement); + return XmlHelper.getDocumentBuilder().parse(new InputSource(new StringReader(defaultAdminElement))).getDocumentElement(); + } + + private static String getImplicitAdminV2() { + return "<admin version='2.0'>\n" + + " <adminserver hostalias='" + HostsXmlProvisioner.IMPLICIT_ADMIN_HOSTALIAS + "'/>\n" + + "</admin>\n"; + } + + private static String getImplicitAdminV4() { + return "<admin version='4.0'>\n" + + " <nodes count='1' />\n" + + "</admin>\n"; + } +} diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepoAdder.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepoAdder.java new file mode 100644 index 00000000000..93c591b3a8e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepoAdder.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +/** + * An interface which provides addition of new config models. + * This exists because some models need to add additional models during the build phase so *write* access + * to the config model repo is needed. *Read* access, on the other hand needs to happen through config model dependency + * inkection to avoid circular dependencies or undeclared dependencies working by accident. + * + * @author bratseth + */ +public interface ConfigModelRepoAdder { + + void add(ConfigModel model); + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelUtils.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelUtils.java new file mode 100644 index 00000000000..f64eee6e863 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelUtils.java @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; +import java.io.Serializable; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static javax.xml.xpath.XPathConstants.BOOLEAN; + +import static com.yahoo.text.Lowercase.toLowerCase; + +/** + * Utilities for config models + * + * @author gjoranv + */ +// TODO: Split this into appropriate classes, or move to ConfigModel superclass +public class ConfigModelUtils implements Serializable { + + private static final long serialVersionUID = 1L; + + public static Pattern hourNmin = Pattern.compile("(\\d\\d):(\\d\\d)"); + + public static Map<String, Integer> day2int; + static { + day2int = new HashMap<>(); + day2int.put("sunday", 0); + day2int.put("monday", 1); + day2int.put("tuesday", 2); + day2int.put("wednesday", 3); + day2int.put("thursday", 4); + day2int.put("friday", 5); + day2int.put("saturday", 6); + } + + /** Parses a 24 hour clock that must be the five characters ##:## to an int stating minutes after midnight. */ + public static int getTimeOfDay(String time) { + Matcher m = ConfigModelUtils.hourNmin.matcher(time); + if (m.matches()) { + return Integer.parseInt(m.group(1)) * 60 + Integer.parseInt(m.group(2)); + } + throw new IllegalArgumentException("The string '" + time + "' is not in ##:## format."); + } + + /** Parses a day of week name in english to an int, where 0 is sunday, 6 saturday. */ + public static int getDayOfWeek(String day) { + return ConfigModelUtils.day2int.get(toLowerCase(day)); + } + + /** + * Create a string with link to documentation for latest release. + * + * @param filePath Relative path of the file to link to, e.g. reference/services-jdisc.html + * @return a String with link to documentation + */ + public static String createDocLink(String filePath) { + return filePath; + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/MapConfigModelRegistry.java b/config-model/src/main/java/com/yahoo/config/model/MapConfigModelRegistry.java new file mode 100644 index 00000000000..d47a009928d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/MapConfigModelRegistry.java @@ -0,0 +1,63 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +import com.google.inject.Inject; +import com.yahoo.component.provider.ComponentRegistry; +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.config.model.builder.xml.ConfigModelId; +import com.yahoo.log.LogLevel; +import org.osgi.framework.Bundle; +import org.osgi.framework.FrameworkUtil; + +import java.util.*; +import java.util.logging.Logger; + +/** + * @author lulf + * @since 5.1 + */ +public class MapConfigModelRegistry extends ConfigModelRegistry { + + private static final Logger log = Logger.getLogger(MapConfigModelRegistry.class.getPackage().getName()); + private final List<ConfigModelBuilder> builders; + + /** + * Constructs a registry of config models, where the components are injected. + * + * @param registry a component registry + */ + @Inject + public MapConfigModelRegistry(ComponentRegistry<? extends ConfigModelBuilder> registry) { + this(registry.allComponents()); + } + + /** + * Constructs a registry of config models. + * + * @param builderCollection A collection of builders used to populate the registry. + */ + public MapConfigModelRegistry(Collection<? extends ConfigModelBuilder> builderCollection) { + super(); + builders = new ArrayList<>(builderCollection); + } + + @Override + public Collection<ConfigModelBuilder> resolve(ConfigModelId id) { + Set<ConfigModelBuilder> matchingBuilders = new HashSet<>(chained().resolve(id)); + for (ConfigModelBuilder builder : builders) + if (builder.handlesElements().contains(id)) + matchingBuilders.add(builder); + return matchingBuilders; + } + + /** + * Create a registry from a variable argument list of builders. + * + * @param builders A variable argument list of builders to use in this map + * @return a ConfigModelRegistry instance. + */ + public static ConfigModelRegistry createFromList(ConfigModelBuilder<? extends ConfigModel> ... builders) { + return new MapConfigModelRegistry(Arrays.asList(builders)); + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/NullConfigModelRegistry.java b/config-model/src/main/java/com/yahoo/config/model/NullConfigModelRegistry.java new file mode 100644 index 00000000000..6930c8c792b --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/NullConfigModelRegistry.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model; + +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.config.model.builder.xml.ConfigModelId; + +import java.util.Collection; + +/** + * A config model class registry that only forwards to the chained registry. + * + * @author bratseth + */ +public class NullConfigModelRegistry extends ConfigModelRegistry { + + @Override + public Collection<ConfigModelBuilder> resolve(ConfigModelId id) { + return chained().resolve(id); + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java new file mode 100644 index 00000000000..8b6ea27ccab --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java @@ -0,0 +1,113 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.admin; + +import com.google.common.collect.ImmutableList; +import com.yahoo.config.model.ConfigModel; +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.config.model.ApplicationConfigProducerRoot; +import com.yahoo.config.model.deploy.DeployProperties; +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.config.model.builder.xml.ConfigModelId; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.vespa.model.admin.Admin; +import com.yahoo.vespa.model.builder.xml.dom.DomAdminV2Builder; +import com.yahoo.vespa.model.builder.xml.dom.DomAdminV4Builder; +import com.yahoo.vespa.model.container.ContainerModel; +import org.w3c.dom.Element; + +import java.util.*; + +/** + * Config model adaptor of the Admin class. + * + * @author lulf + * @since 5.1 + */ +public class AdminModel extends ConfigModel { + + private Admin admin = null; + private final Collection<ContainerModel> containerModels; + + /** + * Constructs a new config model given a context. + * + * @param modelContext the model context. + */ + public AdminModel(ConfigModelContext modelContext, Collection<ContainerModel> containerModels) { + super(modelContext); + this.containerModels = containerModels; + } + + public Admin getAdmin() { return admin; } + + private Collection<ContainerModel> getContainerModels() { return containerModels; } + + @Override + public void prepare(ConfigModelRepo configModelRepo) { + verifyClusterControllersOnlyDefinedForContent(configModelRepo); + if (admin == null || admin.getClusterControllers() == null) return; + admin.getClusterControllers().prepare(); + } + + private void verifyClusterControllersOnlyDefinedForContent(ConfigModelRepo configModelRepo) { + Admin admin = getAdmin(); + if (admin == null || admin.getClusterControllers() == null) return; + if (configModelRepo.getContent() == null) { + throw new IllegalArgumentException("Declaring <clustercontrollers> in <admin> in services.xml will not work when <content> is not defined"); + } + } + + public static class BuilderV2 extends ConfigModelBuilder<AdminModel> { + + public static final List<ConfigModelId> configModelIds = + ImmutableList.of(ConfigModelId.fromNameAndVersion("admin", "2.0"), + ConfigModelId.fromNameAndVersion("admin", "1.0")); + + public BuilderV2() { + super(AdminModel.class); + } + + @Override + public List<ConfigModelId> handlesElements() { return configModelIds; } + + @Override + public void doBuild(AdminModel model, Element adminElement, ConfigModelContext modelContext) { + AbstractConfigProducer parent = modelContext.getParentProducer(); + DeployProperties properties = modelContext.getDeployState().getProperties(); + DomAdminV2Builder domBuilder = new DomAdminV2Builder(modelContext.getDeployState().getFileRegistry(), properties.multitenant(), properties.configServerSpecs()); + model.admin = domBuilder.build(parent, adminElement); + // TODO: Is required since other models depend on admin. + if (parent instanceof ApplicationConfigProducerRoot) { + ((ApplicationConfigProducerRoot)parent).setupAdmin(model.admin); + } + } + } + + public static class BuilderV4 extends ConfigModelBuilder<AdminModel> { + + public static final List<ConfigModelId> configModelIds = + ImmutableList.of(ConfigModelId.fromNameAndVersion("admin", "3.0"), + ConfigModelId.fromNameAndVersion("admin", "4.0")); + + public BuilderV4() { + super(AdminModel.class); + } + + @Override + public List<ConfigModelId> handlesElements() { return configModelIds; } + + @Override + public void doBuild(AdminModel model, Element adminElement, ConfigModelContext modelContext) { + AbstractConfigProducer parent = modelContext.getParentProducer(); + DeployProperties properties = modelContext.getDeployState().getProperties(); + DomAdminV4Builder domBuilder = new DomAdminV4Builder(modelContext, properties.multitenant(), properties.configServerSpecs(), model.getContainerModels()); + model.admin = domBuilder.build(parent, adminElement); + // TODO: Is required since other models depend on admin. + if (parent instanceof ApplicationConfigProducerRoot) { + ((ApplicationConfigProducerRoot)parent).setupAdmin(model.admin); + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java new file mode 100644 index 00000000000..a0b1be20df6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java @@ -0,0 +1,119 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.builder.xml; + +import com.yahoo.component.AbstractComponent; +import com.yahoo.config.model.ConfigModel; +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.ConfigModelInstanceFactory; +import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.config.model.api.ConfigModelPlugin; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import org.w3c.dom.Element; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.List; + +/** + * Builds a config model using DOM parsers + * + * @author vegardh + * @since 5.1.10 + */ +public abstract class ConfigModelBuilder<MODEL extends ConfigModel> extends AbstractComponent implements ConfigModelPlugin { + + private Class<MODEL> configModelClass; + + public ConfigModelBuilder(Class<MODEL> configModelClass) { + this.configModelClass = configModelClass; + } + + /** + * Method that must return the XML elements this builder handles. Subclasses must implement this in order to + * get called when one of the elements have been encountered when parsing. + * + * @return A list of elements that this builder handles. + */ + public abstract List<ConfigModelId> handlesElements(); + + /** + * Convenience hook called from {@link #build}. Implement this method to build a config model. + * + * @param spec The XML element that this builder should handle. + * @param modelContext A model context that contains the application package and other data needed by the + * config model constructor. + */ + public abstract void doBuild(MODEL model, Element spec, ConfigModelContext modelContext); + + /** + * Builds an instance of this component model. + * This calls instantiate(...), instance.setUp(...), doBuild(instance, ...). + * + * @param deployState a global deployment state used for this model. + * @param parent the root config producer this should be added to + * @param spec the XML element this is constructed from + */ + public final MODEL build(DeployState deployState, ConfigModelRepo configModelRepo, AbstractConfigProducer parent, Element spec) { + ConfigModelContext context = ConfigModelContext.create(deployState, configModelRepo, parent, getIdString(spec)); + return build(new DefaultModelInstanceFactory(), spec, context); + } + + /** + * Builds an instance of this component model. + * This calls instantiate(...), instance.setUp(...), doBuild(instance, ...). + * + * @param factory A factory capable of creating models. + * @param spec the XML element this is constructed from + * @param context A context object containing various data used by builders. + */ + public MODEL build(ConfigModelInstanceFactory<MODEL> factory, Element spec, ConfigModelContext context) { + MODEL model = factory.createModel(context); + doBuild(model, spec, context); + return model; + } + + public Class<MODEL> getModelClass() { + return configModelClass; + } + + private static String getIdString(Element spec) { + String idString = XmlHelper.getIdString(spec); + if (idString == null || idString.isEmpty()) { + idString = spec.getTagName(); + } + return idString; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof ConfigModelBuilder)) { + return false; + } + ConfigModelBuilder otherBuilder = (ConfigModelBuilder) other; + List<ConfigModelId> thisIds = this.handlesElements(); + List<ConfigModelId> otherIds = otherBuilder.handlesElements(); + if (thisIds.size() != otherIds.size()) { + return false; + } + for (int i = 0; i < thisIds.size(); i++) { + if (!thisIds.get(i).equals(otherIds.get(i))) { + return false; + } + } + return true; + } + + + private class DefaultModelInstanceFactory implements ConfigModelInstanceFactory<MODEL> { + @Override + public MODEL createModel(ConfigModelContext context) { + try { + Constructor<MODEL> constructor = configModelClass.getConstructor(ConfigModelContext.class); + return constructor.newInstance(context); + } catch (NoSuchMethodException | InvocationTargetException | InstantiationException | IllegalAccessException e) { + throw new RuntimeException("Error constructing model '" + configModelClass.getName() + "'", e); + } + } + } +} diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java new file mode 100644 index 00000000000..fca114757ec --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java @@ -0,0 +1,94 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.builder.xml; + +import com.yahoo.component.Version; + +/** + * A {@link ConfigModelId} describes an element handled by a {@link ConfigModelBuilder}. + * + * @author lulf + * @since 5.1 + */ +public class ConfigModelId implements Comparable<ConfigModelId> { + + private final String name; + private final Version version; + private final String stringValue; + + private ConfigModelId(String name, Version version) { + this.name = name; + this.version = version; + this.stringValue = toStringValue(); + } + + /** + * Create id with a name and version + * @param tagName Name of the id + * @param tagVersion Version of the id + * @return A ConfigModelId instance + */ + public static ConfigModelId fromNameAndVersion(String tagName, String tagVersion) { + return new ConfigModelId(tagName, Version.fromString(tagVersion)); + } + + /** + * Create id with given name, using default version 1. + * + * @param tagName Name of the id + * @return A ConfigModelId instance + */ + public static ConfigModelId fromName(String tagName) { + return new ConfigModelId(tagName, new Version(1)); + } + + @Override + public boolean equals(Object object) { + if (!(object instanceof ConfigModelId)) return false; + ConfigModelId other = (ConfigModelId)object; + return this.name.equals(other.name) && this.version.equals(other.version); + } + + @Override + public int compareTo(ConfigModelId other) { + if (other == this) return 0; + int cmp = this.name.compareTo(other.name); + if (cmp == 0) { + cmp = this.version.compareTo(other.version); + } + return cmp; + } + + @Override + public String toString() { + return stringValue; + } + + @Override + public int hashCode() { + return stringValue.hashCode(); + } + + /** + * Return the XML element name. + * @return the name of the config model + */ + public String getName() { + return name; + } + + /** + * Return the XML element version. + * @return the version of the config model + */ + Version getVersion() { + return version; + } + + private String toStringValue() { + StringBuilder sb = new StringBuilder(); + sb.append(name); + sb.append("."); + sb.append(version); + return sb.toString(); + } +} diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java new file mode 100644 index 00000000000..924d888b0d0 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java @@ -0,0 +1,135 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.builder.xml; + +import com.yahoo.component.ComponentId; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.log.LogLevel; +import com.yahoo.text.XML; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import java.io.IOException; +import java.io.Reader; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.logging.Logger; + + +/** + * Static methods for helping dom building + * + * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a> + */ +public final class XmlHelper { + private static final Logger log = Logger.getLogger(XmlHelper.class.getPackage().toString()); + + + private static final String idReference = "idref"; + // Access to this needs to be synchronized (as it is in getDocumentBuilder() below) + public static final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + + static { + XmlHelper.factory.setNamespaceAware(true); + // if use jdom and jaxen this will fail badly: + XmlHelper.factory.setXIncludeAware(true); + } + + private XmlHelper() {} + + public static String nullIfEmpty(String attribute) { + if (attribute.isEmpty()) + return null; + else + return attribute; + } + + /** + * For searchers inside search chains, the id may be both a reference and an id at once, or just a reference. + * In other cases, it is clear which one it is from context, so I think the difference is not worth bothering users + * with, unless they are XML purists in which case they will have the option of writing this correctly. + * - Jon + */ + public static String getIdString(Element element) { + String idString = element.getAttribute("id"); + if (idString == null || idString.trim().equals("")) + idString = element.getAttribute(idReference); + if (idString == null || idString.trim().equals("")) + idString = element.getAttribute("ident"); + return idString; + } + + public static ComponentId getId(Element element) { + return new ComponentId(getIdString(element)); + } + + public static ComponentSpecification getIdRef(Element element) { + return new ComponentSpecification(getIdString(element)); + } + + public static Document getDocument(Reader reader) { + Document doc; + try { + doc = getDocumentBuilder().parse(new InputSource(reader)); + } catch (SAXException | IOException e) { + throw new IllegalArgumentException(e); + } + return doc; + } + + public static List<String> splitAndDiscardEmpty(String field, String regex) { + List<String> ret = new ArrayList<>(); + for (String t : field.split(regex)) { + if (!t.isEmpty()) { + ret.add(t); + } + } + return ret; + } + + public static List<String> spaceSeparatedSymbols(String field) { + return splitAndDiscardEmpty(field, " "); + } + + public static Collection<String> spaceSeparatedSymbolsFromAttribute(Element spec, String name) { + return spaceSeparatedSymbols(spec.getAttribute(name)); + } + + public static Collection<String> valuesFromElements(Element parent, String elementName) { + List<String> symbols = new ArrayList<>(); + for (Element symbol : XML.getChildren(parent, elementName)) { + symbols.add(XML.getValue(symbol).trim()); + } + return symbols; + } + + public static boolean isReference(Element element) { + return element.hasAttribute(idReference); + } + + /** + * Creates a new XML document builder. + * + * @return A new DocumentBuilder instance, or null if we fail to get one. + */ + public static synchronized DocumentBuilder getDocumentBuilder() { + try { + DocumentBuilder docBuilder = factory.newDocumentBuilder(); + log.log(LogLevel.DEBUG, "XML parser now operational!"); + return docBuilder; + } catch (ParserConfigurationException e) { + log.log(LogLevel.WARNING, "No XML parser available - " + e); + return null; + } + } + + public static Optional<String> getOptionalAttribute(Element element, String name) { + return Optional.ofNullable(element.getAttribute(name)).filter(s -> !s.isEmpty()); + } +} diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/package-info.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/package-info.java new file mode 100644 index 00000000000..6ce400dc922 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.config.model.builder.xml; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java b/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java new file mode 100644 index 00000000000..22ea054a27d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.deploy; + +import com.yahoo.vespa.config.ConfigDefinition; +import com.yahoo.vespa.config.ConfigDefinitionKey; + +/** + * @author lulf + * @since 5.1 + */ +public interface ConfigDefinitionStore { + + ConfigDefinition getConfigDefinition(ConfigDefinitionKey defKey); + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java new file mode 100644 index 00000000000..3f6ef494a27 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java @@ -0,0 +1,115 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.deploy; + +import com.yahoo.config.model.api.ConfigServerSpec; +import com.yahoo.config.provision.*; + +import java.util.ArrayList; +import java.util.List; + +/** + * Collection of properties for a deployment. + * + * @author lulf + * @since 5.17 + */ +public class DeployProperties { + private final boolean multitenant; + private final ApplicationId applicationId; + private final List<ConfigServerSpec> serverSpecs = new ArrayList<>(); + private final boolean hostedVespa; + private final Version vespaVersion; + private final Zone zone; + + private DeployProperties(boolean multitenant, + ApplicationId applicationId, + List<ConfigServerSpec> configServerSpecs, + boolean hostedVespa, Version vespaVersion, Zone zone) { + this.vespaVersion = vespaVersion; + this.zone = zone; + this.multitenant = multitenant || hostedVespa || Boolean.getBoolean("multitenant"); + this.applicationId = applicationId; + this.serverSpecs.addAll(configServerSpecs); + this.hostedVespa = hostedVespa; + } + + + public boolean multitenant() { + return multitenant; + } + + public ApplicationId applicationId() { + return applicationId; + } + + public List<ConfigServerSpec> configServerSpecs() { + return serverSpecs; + } + + public Quota quota() { + return new Quota(Integer.MAX_VALUE); + } + + public boolean hostedVespa() { + return hostedVespa; + } + + public Version vespaVersion() { + return vespaVersion; + } + + public Zone zone() { + return zone; + } + + public static class Builder { + + private ApplicationId applicationId = ApplicationId.defaultId(); + private boolean multitenant = false; + private List<ConfigServerSpec> configServerSpecs = new ArrayList<>(); + private Quota quota = new Quota(Integer.MAX_VALUE); + private boolean hostedVespa = false; + private Version vespaVersion = Version.fromIntValues(1, 0, 0); + private Zone zone = Zone.defaultZone(); + + public Builder applicationId(ApplicationId applicationId) { + this.applicationId = applicationId; + return this; + } + + public Builder multitenant(boolean multitenant) { + this.multitenant = multitenant; + return this; + } + + public Builder configServerSpecs(List<ConfigServerSpec> configServerSpecs) { + this.configServerSpecs = configServerSpecs; + return this; + } + + public Builder quota(Quota quota) { + this.quota = quota; + return this; + } + + public Builder vespaVersion(Version version) { + this.vespaVersion = version; + return this; + } + + public Builder hostedVespa(boolean hostedVespa) { + this.hostedVespa = hostedVespa; + return this; + } + + public Builder zone(Zone zone) { + this.zone = zone; + return this; + } + + public DeployProperties build() { + return new DeployProperties(multitenant, applicationId, configServerSpecs, hostedVespa, vespaVersion, zone); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java new file mode 100644 index 00000000000..5896dc59df2 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -0,0 +1,381 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.deploy; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.config.application.api.UnparsedConfigDefinition; +import com.yahoo.config.codegen.CNode; +import com.yahoo.config.model.api.ConfigDefinitionRepo; +import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.provision.HostsXmlProvisioner; +import com.yahoo.config.model.provision.SingleNodeProvisioner; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.Rotation; +import com.yahoo.config.provision.Zone; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.log.LogLevel; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.config.ConfigDefinition; +import com.yahoo.vespa.config.ConfigDefinitionBuilder; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.documentmodel.DocumentModel; +import com.yahoo.vespa.model.application.validation.ValidationOverrides; +import com.yahoo.vespa.model.application.validation.xml.ValidationOverridesXMLReader; +import com.yahoo.vespa.model.container.search.QueryProfiles; +import com.yahoo.vespa.model.container.search.QueryProfilesBuilder; +import com.yahoo.vespa.model.container.search.SemanticRuleBuilder; +import com.yahoo.vespa.model.container.search.SemanticRules; +import com.yahoo.vespa.model.search.SearchDefinition; + +import java.io.File; +import java.io.IOException; +import java.io.Reader; +import java.time.Instant; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.logging.Logger; + +/** + * Contains various state during deploy that should be reachable to all builders of a {@link com.yahoo.config.model.ConfigModel} + * + * @author lulf + * @since 5.8 + */ +public class DeployState implements ConfigDefinitionStore { + + private static final Logger log = Logger.getLogger(DeployState.class.getName()); + + private final DeployLogger logger; + private final FileRegistry fileRegistry; + private final DocumentModel documentModel; + private final List<SearchDefinition> searchDefinitions; + private final ApplicationPackage applicationPackage; + private final Optional<ConfigDefinitionRepo> configDefinitionRepo; + private final Optional<ApplicationPackage> permanentApplicationPackage; + private final Optional<Model> previousModel; + private final DeployProperties properties; + private final Set<Rotation> rotations; + private final Zone zone; + private final QueryProfiles queryProfiles; + private final SemanticRules semanticRules; + private final ValidationOverrides validationOverrides; + + private final HostProvisioner provisioner; + + public static DeployState createTestState() { + return new Builder().build(); + } + + public static DeployState createTestState(ApplicationPackage applicationPackage) { + return new Builder().applicationPackage(applicationPackage).build(); + } + + private DeployState(ApplicationPackage applicationPackage, SearchDocumentModel searchDocumentModel, RankProfileRegistry rankProfileRegistry, + FileRegistry fileRegistry, DeployLogger deployLogger, Optional<HostProvisioner> hostProvisioner, DeployProperties properties, + Optional<ApplicationPackage> permanentApplicationPackage, Optional<ConfigDefinitionRepo> configDefinitionRepo, + java.util.Optional<Model> previousModel, Set<Rotation> rotations, Zone zone, QueryProfiles queryProfiles, SemanticRules semanticRules, Instant now) { + this.logger = deployLogger; + this.fileRegistry = fileRegistry; + this.rankProfileRegistry = rankProfileRegistry; + this.applicationPackage = applicationPackage; + this.properties = properties; + this.previousModel = previousModel; + if (hostProvisioner.isPresent()) { + this.provisioner = hostProvisioner.get(); + } else { + this.provisioner = getDefaultModelHostProvisioner(applicationPackage); + } + this.searchDefinitions = searchDocumentModel.getSearchDefinitions(); + this.documentModel = searchDocumentModel.getDocumentModel(); + this.permanentApplicationPackage = permanentApplicationPackage; + this.configDefinitionRepo = configDefinitionRepo; + this.rotations = rotations; + this.zone = zone; + this.queryProfiles = queryProfiles; // TODO: Remove this by seeing how pagetemplates are propagated + this.semanticRules = semanticRules; // TODO: Remove this by seeing how pagetemplates are propagated + this.validationOverrides = new ValidationOverridesXMLReader().read(applicationPackage.getValidationOverrides(), now); + + } + + public static HostProvisioner getDefaultModelHostProvisioner(ApplicationPackage applicationPackage) { + if (applicationPackage.getHosts() == null) { + return new SingleNodeProvisioner(); + } else { + return new HostsXmlProvisioner(applicationPackage.getHosts()); + } + } + + /** Get the global rank profile registry for this application. */ + public final RankProfileRegistry rankProfileRegistry() { return rankProfileRegistry; } + + /** Returns the validation overrides of this. This is never null */ + public ValidationOverrides validationOverrides() { return validationOverrides; } + + /** + * Returns the config def with the given name and namespace. + * + * @param defKey The {@link ConfigDefinitionKey} that will uniquely identify a config definition. + * @return The definition with a matching name and namespace + * @throws java.lang.IllegalArgumentException if def is not found. + */ + public final ConfigDefinition getConfigDefinition(ConfigDefinitionKey defKey) { + if (existingConfigDefs == null) { + existingConfigDefs = new LinkedHashMap<>(); + if (configDefinitionRepo.isPresent()) { + existingConfigDefs.putAll(createLazyMapping(configDefinitionRepo.get())); + } + existingConfigDefs.putAll(applicationPackage.getAllExistingConfigDefs()); + } + log.log(LogLevel.DEBUG, "Getting config definition " + defKey); + // Fall back to default namespace if not found. + if (defKey.getNamespace() == null || defKey.getNamespace().isEmpty()) { + defKey = new ConfigDefinitionKey(defKey.getName(), CNode.DEFAULT_NAMESPACE); + } + ConfigDefinitionKey lookupKey = defKey; + // Fall back to just using name + if (!existingConfigDefs.containsKey(lookupKey)) { + + int count = 0; + for (ConfigDefinitionKey entry : existingConfigDefs.keySet()) { + if (entry.getName().equals(defKey.getName())) { + count++; + } + } + if (count > 1) { + throw new IllegalArgumentException("Using config definition '" + defKey.getName() + "' is ambiguous, there are more than one config definitions with this name, please specify namespace"); + } + + lookupKey = null; + log.log(LogLevel.DEBUG, "Could not find config definition '" + defKey + "', trying with same name in all namespaces"); + for (ConfigDefinitionKey entry : existingConfigDefs.keySet()) { + if (entry.getName().equals(defKey.getName()) && defKey.getNamespace().equals(CNode.DEFAULT_NAMESPACE)) { + log.log(LogLevel.INFO, "Could not find config definition '" + defKey + "'" + + ", using config definition '" + entry + "' with same name instead (please use new namespace when specifying this config)"); + lookupKey = entry; + break; + } + } + } + + if (lookupKey == null) { + throw new IllegalArgumentException("Could not find a config definition with name '" + defKey + "'."); + } + if (defArchive.get(defKey) != null) { + log.log(LogLevel.DEBUG, "Found in archive: " + defKey); + return defArchive.get(defKey); + } + + log.log(LogLevel.DEBUG, "Retrieving config definition: " + defKey); + ConfigDefinition def = existingConfigDefs.get(lookupKey).parse(); + + log.log(LogLevel.DEBUG, "Adding " + def + " to archive"); + defArchive.put(defKey, def); + return def; + } + + private static Map<ConfigDefinitionKey, UnparsedConfigDefinition> createLazyMapping(final ConfigDefinitionRepo configDefinitionRepo) { + Map<ConfigDefinitionKey, UnparsedConfigDefinition> keyToRepo = new LinkedHashMap<>(); + for (final Map.Entry<ConfigDefinitionKey, com.yahoo.vespa.config.buildergen.ConfigDefinition> defEntry : configDefinitionRepo.getConfigDefinitions().entrySet()) { + keyToRepo.put(defEntry.getKey(), new UnparsedConfigDefinition() { + @Override + public ConfigDefinition parse() { + return ConfigDefinitionBuilder.createConfigDefinition(configDefinitionRepo.getConfigDefinitions().get(defEntry.getKey()).getCNode()); + } + + @Override + public String getUnparsedContent() { + throw new UnsupportedOperationException("Cannot get unparsed content from " + defEntry.getKey()); + } + }); + } + return keyToRepo; + } + + // Global registry of rank profiles. + // TODO: I think this can be removed when we remove "<search version=2.0>" and only support content. + private final RankProfileRegistry rankProfileRegistry; + + // Mapping from key to something that can create a config definition. + private Map<ConfigDefinitionKey, UnparsedConfigDefinition> existingConfigDefs = null; + + // Cache of config defs for all [def,version] combinations looked up so far. + private final Map<ConfigDefinitionKey, ConfigDefinition> defArchive = new LinkedHashMap<>(); + + public ApplicationPackage getApplicationPackage() { + return applicationPackage; + } + public List<SearchDefinition> getSearchDefinitions() { + return searchDefinitions; + } + + public DocumentModel getDocumentModel() { + return documentModel; + } + + public DeployLogger getDeployLogger() { + return logger; + } + + public FileRegistry getFileRegistry() { + return fileRegistry; + } + + public HostProvisioner getProvisioner() { return provisioner; } + + public Optional<ApplicationPackage> getPermanentApplicationPackage() { + return permanentApplicationPackage; + } + + public DeployProperties getProperties() { return properties; } + + public Optional<Model> getPreviousModel() { return previousModel; } + + public boolean isHostedVespa() { + return properties.hostedVespa(); + } + + public Set<Rotation> getRotations() { + return this.rotations; // todo: consider returning a copy or immutable view + } + + /** Returns the zone in which this is currently running */ + public Zone zone() { return zone; } + + public QueryProfiles getQueryProfiles() { return queryProfiles; } + + public SemanticRules getSemanticRules() { return semanticRules; } + + public static class Builder { + + private ApplicationPackage applicationPackage = MockApplicationPackage.createEmpty(); + private FileRegistry fileRegistry = new MockFileRegistry(); + private DeployLogger logger = new BaseDeployLogger(); + private Optional<HostProvisioner> hostProvisioner = Optional.empty(); + private Optional<ApplicationPackage> permanentApplicationPackage = Optional.empty(); + private DeployProperties properties = new DeployProperties.Builder().build(); + private Optional<ConfigDefinitionRepo> configDefinitionRepo = Optional.empty(); + private Optional<Model> previousModel = Optional.empty(); + private Set<Rotation> rotations = new HashSet<>(); + private Zone zone = Zone.defaultZone(); + private Instant now = Instant.now(); + + public Builder applicationPackage(ApplicationPackage applicationPackage) { + this.applicationPackage = applicationPackage; + return this; + } + + public Builder fileRegistry(FileRegistry fileRegistry) { + this.fileRegistry = fileRegistry; + return this; + } + + public Builder deployLogger(DeployLogger logger) { + this.logger = logger; + return this; + } + + public Builder modelHostProvisioner(HostProvisioner modelProvisioner) { + this.hostProvisioner = Optional.of(modelProvisioner); + return this; + } + + public Builder permanentApplicationPackage(Optional<ApplicationPackage> permanentApplicationPackage) { + this.permanentApplicationPackage = permanentApplicationPackage; + return this; + } + + public Builder properties(DeployProperties properties) { + this.properties = properties; + return this; + } + + public Builder configDefinitionRepo(ConfigDefinitionRepo configDefinitionRepo) { + this.configDefinitionRepo = Optional.of(configDefinitionRepo); + return this; + } + + public Builder previousModel(Model previousModel) { + this.previousModel = Optional.of(previousModel); + return this; + } + + public Builder rotations(Set<Rotation> rotations) { + this.rotations = rotations; + return this; + } + + public Builder zone(Zone zone) { + this.zone = zone; + return this; + } + + public Builder now(Instant now) { + this.now = now; + return this; + } + + public DeployState build() { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + QueryProfiles queryProfiles = new QueryProfilesBuilder().build(applicationPackage); + SemanticRules semanticRules = new SemanticRuleBuilder().build(applicationPackage); + SearchDocumentModel searchDocumentModel = createSearchDocumentModel(rankProfileRegistry, logger, queryProfiles); + return new DeployState(applicationPackage, searchDocumentModel, rankProfileRegistry, fileRegistry, logger, hostProvisioner, + properties, permanentApplicationPackage, configDefinitionRepo, previousModel, rotations, zone, queryProfiles, semanticRules, now); + } + + private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, DeployLogger logger, QueryProfiles queryProfiles) { + Collection<NamedReader> readers = applicationPackage.getSearchDefinitions(); + Map<String, String> names = new LinkedHashMap<>(); + SearchBuilder builder = new SearchBuilder(applicationPackage, rankProfileRegistry); + for (NamedReader reader : readers) { + try { + String readerName = reader.getName(); + String searchName = builder.importReader(reader, readerName, logger); + String sdName = stripSuffix(readerName, ApplicationPackage.SD_NAME_SUFFIX); + names.put(searchName, sdName); + if (!sdName.equals(searchName)) { + throw new IllegalArgumentException("Search definition file name ('" + sdName + "') and name of " + + "search element ('" + searchName + "') are not equal for file '" + readerName + "'"); + } + } catch (ParseException e) { + throw new IllegalArgumentException("Could not parse search definition file '" + getSearchDefinitionRelativePath(reader.getName()) + "': " + e.getMessage(), e); + } catch (IOException e) { + throw new IllegalArgumentException("Could not read search definition file '" + getSearchDefinitionRelativePath(reader.getName()) + "': " + e.getMessage(), e); + } finally { + closeIgnoreException(reader.getReader()); + } + } + builder.build(logger, queryProfiles); + return SearchDocumentModel.fromBuilderAndNames(builder, names); + } + + private String getSearchDefinitionRelativePath(String name) { + return ApplicationPackage.SEARCH_DEFINITIONS_DIR + File.separator + name; + } + + private static String stripSuffix(String nodeName, String postfix) { + assert (nodeName.endsWith(postfix)); + return nodeName.substring(0, nodeName.length() - postfix.length()); + } + + @SuppressWarnings("EmptyCatchBlock") + private static void closeIgnoreException(Reader reader) { + try { + reader.close(); + } catch(Exception e) {} + } + } + +} + diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java b/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java new file mode 100644 index 00000000000..1fbd72599fe --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.deploy; + +import com.yahoo.searchdefinition.SearchBuilder; +import com.yahoo.vespa.documentmodel.DocumentModel; +import com.yahoo.vespa.model.search.SearchDefinition; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Internal helper class to retrieve document model and search definitions. + * + * @author lulf + * @since 5.1 + */ +public class SearchDocumentModel { + + private final DocumentModel documentModel; + private final List<SearchDefinition> searchDefinitions; + + public SearchDocumentModel(DocumentModel documentModel, List<SearchDefinition> searchDefinitions) { + this.documentModel = documentModel; + this.searchDefinitions = searchDefinitions; + + } + + public DocumentModel getDocumentModel() { + return documentModel; + } + + public List<SearchDefinition> getSearchDefinitions() { + return searchDefinitions; + } + + public static SearchDocumentModel fromBuilderAndNames(SearchBuilder builder, Map<String, String> names) { + List<SearchDefinition> ret = new ArrayList<>(); + for (com.yahoo.searchdefinition.Search search : builder.getSearchList()) { + ret.add(new SearchDefinition(names.get(search.getName()), search)); + } + return new SearchDocumentModel(builder.getModel(), ret); + } + + public static SearchDocumentModel fromBuilder(SearchBuilder builder) { + List<SearchDefinition> ret = new ArrayList<>(); + for (com.yahoo.searchdefinition.Search search : builder.getSearchList()) { + ret.add(new SearchDefinition(search.getName(), search)); + } + return new SearchDocumentModel(builder.getModel(), ret); + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/package-info.java b/config-model/src/main/java/com/yahoo/config/model/deploy/package-info.java new file mode 100644 index 00000000000..3feb934615f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.config.model.deploy; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/config/model/graph/ModelGraph.java b/config-model/src/main/java/com/yahoo/config/model/graph/ModelGraph.java new file mode 100644 index 00000000000..a435733e60f --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/graph/ModelGraph.java @@ -0,0 +1,80 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.graph; + +import com.yahoo.component.ComponentId; + +import java.util.*; + +/** + * A model graph contains the dependency graph of config models. + * + * @author lulf + * @since 5.1 + */ +public class ModelGraph { + + private final List<ModelNode> modelNodes; + private final List<ModelNode> roots; + public ModelGraph(List<ModelNode> modelNodes, List<ModelNode> roots) { + this.modelNodes = modelNodes; + this.roots = roots; + } + + /** + * Performs a topological sort ot the models stored in this graph. The algorithm is based on Kahn topological sort. + * + * @return a sorted list of {@link com.yahoo.config.model.graph.ModelNode} in dependency order. + */ + public List<ModelNode> topologicalSort() { + DependencyMap dependencyMap = new DependencyMap(modelNodes); + Queue<ModelNode> unprocessed = new LinkedList<>(); + unprocessed.addAll(roots); + List<ModelNode> sortedList = new ArrayList<>(); + while (!unprocessed.isEmpty()) { + ModelNode sortedNode = unprocessed.remove(); + sortedList.add(sortedNode); + for (ModelNode node : modelNodes) { + if (dependencyMap.dependsOn(node, sortedNode)) { + dependencyMap.removeDependency(node, sortedNode); + if (!dependencyMap.hasDependencies(node)) { + unprocessed.add(node); + } + } + } + } + for (ModelNode node : modelNodes) { + if (dependencyMap.hasDependencies(node)) { + throw new IllegalArgumentException("Unable to sort graph because it contains cycles"); + } + } + return sortedList; + } + + List<ModelNode> getNodes() { + return modelNodes; + } + + private static class DependencyMap { + private final Map<ComponentId, Set<ComponentId>> map = new LinkedHashMap<>(); + DependencyMap(List<ModelNode> modelNodes) { + for (ModelNode node : modelNodes) { + Set<ComponentId> ids = new LinkedHashSet<>(); + ids.addAll(node.listDependencyIds()); + map.put(node.id, ids); + } + } + + public boolean dependsOn(ModelNode node, ModelNode sortedNode) { + return map.get(node.id).contains(sortedNode.id); + } + + public void removeDependency(ModelNode node, ModelNode sortedNode) { + map.get(node.id).remove(sortedNode.id); + } + + public boolean hasDependencies(ModelNode node) { + return !map.get(node.id).isEmpty(); + } + } +} + diff --git a/config-model/src/main/java/com/yahoo/config/model/graph/ModelGraphBuilder.java b/config-model/src/main/java/com/yahoo/config/model/graph/ModelGraphBuilder.java new file mode 100644 index 00000000000..97d914c6012 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/graph/ModelGraphBuilder.java @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.graph; + +import com.yahoo.config.model.ConfigModel; +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; + +import java.util.ArrayList; +import java.util.List; + +/** + * Used to add builders and elements in addBuilder, and then build a dependency graph based on the + * constructor arguments. + * + * @author lulf + * @since 5.1 + */ +public class ModelGraphBuilder { + + private final List<ConfigModelBuilder<? extends ConfigModel>> builders = new ArrayList<>(); + + /** + * Add a {@link com.yahoo.config.model.builder.xml.ConfigModelBuilder} to this graph. + * + * @param builder The {@link com.yahoo.config.model.builder.xml.ConfigModelBuilder} to add. + * @return this for convenience + */ + public ModelGraphBuilder addBuilder(ConfigModelBuilder<? extends ConfigModel> builder) { + builders.add(builder); + return this; + } + + /** + * Build a {@link com.yahoo.config.model.graph.ModelGraph} based on the {@link com.yahoo.config.model.builder.xml.ConfigModelBuilder}s + * added to this. + * + * @return A {@link com.yahoo.config.model.graph.ModelGraph} representing the dependency graph. + */ + public ModelGraph build() { + List<ModelNode> modelNodes = new ArrayList<>(); + for (ConfigModelBuilder<? extends ConfigModel> builder : builders) { + modelNodes.add(new ModelNode(builder)); + } + List<ModelNode> roots = new ArrayList<>(); + for (ModelNode modelNode : modelNodes) { + int numDependencies = modelNode.addDependenciesFrom(modelNodes); + if (numDependencies == 0) { + roots.add(modelNode); + } + } + return new ModelGraph(modelNodes, roots); + } +} diff --git a/config-model/src/main/java/com/yahoo/config/model/graph/ModelNode.java b/config-model/src/main/java/com/yahoo/config/model/graph/ModelNode.java new file mode 100644 index 00000000000..0cd045b09c8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/graph/ModelNode.java @@ -0,0 +1,135 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.graph; + +import com.google.inject.Inject; +import com.yahoo.component.ComponentId; +import com.yahoo.config.model.ConfigModel; +import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.config.model.ConfigModelInstanceFactory; + +import java.lang.reflect.*; +import java.util.*; + +/** + * Represents a node in the dependency graph, and contains information about a builders dependencies. + * Constructor signatures of model classes must have ConfigModelContext as the first argument and + * ConfigModel subclasses or Collection of ConfigModels as subsequent arguments. + * Only Collection, not Collection subtypes can be used. + * + * @author lulf + * @since 5.1 + */ +public class ModelNode<MODEL extends ConfigModel> implements ConfigModelInstanceFactory<MODEL> { + + final ComponentId id; + public final ConfigModelBuilder<MODEL> builder; + final Class<MODEL> clazz; + final Constructor<MODEL> constructor; + final List<MODEL> instances = new ArrayList<>(); + private final Map<ComponentId, ModelNode> dependencies = new HashMap<>(); + + public ModelNode(ConfigModelBuilder<MODEL> builder) { + this.id = builder.getId(); + this.builder = builder; + this.clazz = builder.getModelClass(); + this.constructor = findConstructor(clazz); + } + + private Constructor<MODEL> findConstructor(Class<MODEL> clazz) { + for (Constructor<?> ctor : clazz.getDeclaredConstructors()) { + if (ctor.getAnnotation(Inject.class) != null) { + return (Constructor<MODEL>) ctor; + } + } + return (Constructor<MODEL>) clazz.getDeclaredConstructors()[0]; + } + + boolean hasDependencies() { + return !dependencies.isEmpty(); + } + + boolean dependsOn(ModelNode node) { + return dependencies.containsKey(node.id); + } + + /** + * This adds dependencies base on constructor arguments in the model classes themselves. + * These then have to be created by this mini-di framework and then handed to the builders + * that will fill them. + * + * TODO: This should be changed to model dependencies between model builders instead, such + * that they can create their model objects, and eventually make them immutable. + */ + int addDependenciesFrom(List<ModelNode> modelNodes) { + int numDependencies = 0; + for (Type param : constructor.getGenericParameterTypes()) { + for (ModelNode node : modelNodes) { + if (param.equals(node.clazz) || isCollectionOf(param, node.clazz)) { + addDependency(node); + numDependencies++; + } + } + } + return numDependencies; + } + + private boolean isCollectionOf(Type type, Class<?> nodeClazz) { + if (type instanceof ParameterizedType) { + ParameterizedType t = (ParameterizedType) type; + // Note: IntelliJ says the following cannot be equal but that is wrong + return (t.getRawType().equals(java.util.Collection.class) && t.getActualTypeArguments().length == 1 && t.getActualTypeArguments()[0].equals(nodeClazz)); + } + return false; + } + + private boolean isCollection(Type type) { + if (type instanceof ParameterizedType) { + ParameterizedType t = (ParameterizedType) type; + // Note: IntelliJ says the following cannot be equal but that is wrong + return (t.getRawType().equals(java.util.Collection.class) && t.getActualTypeArguments().length == 1); + } + return false; + } + + private void addDependency(ModelNode node) { + dependencies.put(node.id, node); + } + + Collection<ComponentId> listDependencyIds() { + return dependencies.keySet(); + } + + @Override + public MODEL createModel(ConfigModelContext context) { + try { + Type [] params = constructor.getGenericParameterTypes(); + if (params.length < 1 || ! params[0].equals(ConfigModelContext.class)) { + throw new IllegalArgumentException("Constructor for " + clazz.getName() + " must have as its first argument a " + ConfigModelContext.class.getName()); + } + Object arguments[] = new Object[params.length]; + arguments[0] = context; + for (int i = 1; i < params.length; i++) + arguments[i] = findArgument(params[i]); + MODEL instance = constructor.newInstance(arguments); + instances.add(instance); + return instance; + } catch (InvocationTargetException | InstantiationException | IllegalAccessException e) { + throw new RuntimeException("Error constructing model '" + clazz.getName() + "'", e); + } + } + + private Object findArgument(Type param) { + for (ModelNode dependency : dependencies.values()) { + if (param.equals(dependency.clazz)) + return dependency.instances.get(0); + if (isCollectionOf(param, dependency.clazz)) + return Collections.unmodifiableCollection(dependency.instances); + } + // For collections, we don't require that dependency has been added, we just give an empty collection + if (isCollection(param)) + return Collections.emptyList(); + throw new IllegalArgumentException("Unable to find constructor argument " + param + " for " + clazz.getName()); + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/package-info.java b/config-model/src/main/java/com/yahoo/config/model/package-info.java new file mode 100644 index 00000000000..594f52c0d14 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.config.model; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java new file mode 100644 index 00000000000..41927bc09a9 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java @@ -0,0 +1,448 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.producer; + +import com.google.common.annotations.Beta; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.subscription.ConfigInstanceUtil; +import com.yahoo.config.model.ApplicationConfigProducerRoot; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.log.LogLevel; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.config.*; +import com.yahoo.vespa.model.*; +import com.yahoo.vespa.model.admin.Admin; +import com.yahoo.vespa.model.admin.MonitoringSystem; +import com.yahoo.vespa.model.utils.FreezableMap; + +import java.io.*; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Type; +import java.util.*; +import java.util.logging.Logger; + +/** + * Superclass for all config producers. + * Config producers constructs and returns config instances on request. + * + * @author gjoranv + */ +public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProducer<?>> + implements ConfigProducer, ConfigInstance.Producer, Serializable { + + private static final long serialVersionUID = 1L; + public static final Logger log = Logger.getLogger(AbstractConfigProducer.class.getPackage().toString()); + private final String subId; + private final boolean hostedVespa; + private String configId = null; + + private List<Service> descendantServices = new ArrayList<>(); + + private AbstractConfigProducer parent = null; + + private UserConfigRepo userConfigs = new UserConfigRepo(); + + private final FreezableMap<String, CHILD> childrenBySubId = new FreezableMap<>(LinkedHashMap.class); + + private static boolean isHostedVespa(AbstractConfigProducer parent) { + return (parent != null) + && (parent.getRoot() != null) + && (parent.getRoot().getDeployState() != null) + && parent.getRoot().getDeployState().isHostedVespa(); + } + + /** + * Creates a new AbstractConfigProducer with the given parent and subId. + * This constructor will add the resulting producer to the children of parent. + * + * @param parent The parent of this ConfigProducer + * @param subId The fragment of the config id for the producer + */ + public AbstractConfigProducer(AbstractConfigProducer parent, String subId) { + this(subId, isHostedVespa(parent)); + + if (parent != null) { + parent.addChild(this); + } + } + + protected final void setParent(AbstractConfigProducer parent) { this.parent = parent; } + public final String getSubId() { return subId; } + public final boolean isHostedVespa() { return hostedVespa; } + + /** + * Create an config producer with a configId only. Used e.g. to create root nodes, and producers + * that are given children after construction using {@link #addChild(AbstractConfigProducer)}. + * + * @param subId The sub configId. Note that this can be prefixed when calling addChild with this producer as arg. + */ + public AbstractConfigProducer(String subId) { + this(subId, false); + } + + private AbstractConfigProducer(String subId, boolean hostedVespa) { + if (subId.indexOf('/') != -1) { + throw new IllegalArgumentException("A subId might not contain '/' : '" + subId + "'"); + } + this.subId = subId; + this.hostedVespa = hostedVespa; + } + + /** + * Adds a child to this config producer. + * + * @param child The child config producer to add. + */ + protected void addChild(CHILD child) { + if (child == null) { + throw new IllegalArgumentException("Trying to add null child for: " + this); + } + if (child instanceof AbstractConfigProducerRoot) { + throw new IllegalArgumentException("Child cannot be a root node: " + child); + } + + child.setParent(this); + if (childrenBySubId.get(child.getSubId()) != null) { + throw new IllegalArgumentException("Multiple services/instances of the id '" + child.getSubId() + "' under the service/instance " + + errorMsgClassName() + " '" + subId + "'. (This is commonly caused by service/node index " + + "collisions in the config.)." + + "\nExisting instance: " + childrenBySubId.get(child.getSubId()) + + "\nAttempted to add: " + child); + } + childrenBySubId.put(child.getSubId(), child); + + if (child instanceof Service) { + addDescendantService((Service)child); + } + } + + public void removeChild(CHILD child) { + if (child.getParent() != this) + throw new IllegalArgumentException("Could not remove " + child + ": Expected its parent to be " + + this + ", but was " + child.getParent()); + + if (child instanceof Service) + descendantServices.remove(child); + + childrenBySubId.remove(child.getSubId()); + child.setParent(null); + } + + /** + * Helper to provide an error message on collisions of sub ids (ignore SimpleConfigProducer, use the parent in that case) + */ + private String errorMsgClassName() { + if (getClass().equals(SimpleConfigProducer.class)) return parent.getClass().getSimpleName(); + return getClass().getSimpleName(); + } + + /** + * Sets the user configs for this producer. + * + * @param repo User configs repo. + */ + public void setUserConfigs(UserConfigRepo repo) { this.userConfigs = repo; } + + /** Returns the user configs of this */ + @Override + public UserConfigRepo getUserConfigs() { return userConfigs; } + + /** + * ConfigProducers that must have a special config id should use + * setConfigId() instead of overloading this method. This is + * because config IDs must be registered through setConfigId(). + */ + public final String getConfigId() { + if (configId == null) throw new RuntimeException("The system topology must be frozen first."); + return configId; + } + + /** + * Sets the config id for this producer. Will also add this + * service to the root node, so the new config id will be picked + * up. Note that this producer will be known with both the old + * and the new config id in the root node after using this method. + */ + protected void addConfigId(String id) { + if (id == null) throw new NullPointerException("Config ID cannot be null."); + getRoot().addDescendant(id, this); + if (!isVespa() && (getVespa() != null)) + getVespa().addDescendant(this); + } + + /** Returns this ConfigProducer's children (only 1st level) */ + public Map<String, CHILD> getChildren() { return Collections.unmodifiableMap(childrenBySubId); } + + @Beta + public <J extends AbstractConfigProducer<?>> List<J> getChildrenByTypeRecursive(Class<J> type) { + List<J> validChildren = new ArrayList<>(); + + if (this.getClass().equals(type)) { + validChildren.add(type.cast(this)); + } + + Map<String, ? extends AbstractConfigProducer<?>> children = this.getChildren(); + for (AbstractConfigProducer<?> child : children.values()) { + validChildren.addAll(child.getChildrenByTypeRecursive(type)); + } + + return Collections.unmodifiableList(validChildren); + } + + /** Returns a list of all the children of this who are instances of Service */ + public List<Service> getDescendantServices() { return Collections.unmodifiableList(descendantServices); } + + protected void addDescendantService(Service s) { descendantServices.add(s); } + + @Override + public final boolean cascadeConfig(ConfigInstance.Builder builder) { + boolean found=false; + if (parent != null) + found = parent.cascadeConfig(builder); + + boolean foundHere = builder.dispatchGetConfig(this); + if (log.isLoggable(LogLevel.DEBUG)) { + log.log(LogLevel.DEBUG, "cascadeconfig in " + this + ", getting config " + + builder.getClass().getDeclaringClass().getName() + " for config id '" + configId + "' found here=" + foundHere); + } + found = found || foundHere; + return found; + } + + @Override + public final boolean addUserConfig(ConfigInstance.Builder builder) { + boolean didApply = false; + if (parent != null) { + didApply = parent.addUserConfig(builder); + } + + if (log.isLoggable(LogLevel.SPAM)) { + log.log(LogLevel.SPAM, "User configs is: " + userConfigs.toString()); + } + // TODO: What do we do with md5. Currently ignored for user configs? + ConfigDefinitionKey key = new ConfigDefinitionKey(builder.getDefName(), builder.getDefNamespace()); + if (userConfigs.get(key) != null) { + if (log.isLoggable(LogLevel.SPAM)) { + log.log(LogLevel.SPAM, "Apply in " + configId); + } + applyUserConfig(builder, userConfigs.get(key)); + didApply = true; + } + return didApply; + } + + private void applyUserConfig(ConfigInstance.Builder builder, ConfigPayloadBuilder payloadBuilder) { + ConfigInstance.Builder override; + if (builder instanceof GenericConfig.GenericConfigBuilder) { + // Means that the builder is unknown and that we should try to apply the payload without + // the real builder + override = getGenericConfigBuilderOverride((GenericConfig.GenericConfigBuilder) builder, payloadBuilder); + } else { + override = getConfigInstanceBuilderOverride(builder, ConfigPayload.fromBuilder(payloadBuilder)); + } + ConfigInstanceUtil.setValues(builder, override); + } + + private ConfigInstance.Builder getGenericConfigBuilderOverride(GenericConfig.GenericConfigBuilder builder, ConfigPayloadBuilder payloadBuilder) { + ConfigDefinitionKey key = new ConfigDefinitionKey(builder.getDefName(), builder.getDefNamespace()); + return new GenericConfig.GenericConfigBuilder(key, payloadBuilder); + } + + private ConfigInstance.Builder getConfigInstanceBuilderOverride(ConfigInstance.Builder builder, ConfigPayload payload) { + try { + ConfigTransformer transformer = new ConfigTransformer(builder.getClass().getEnclosingClass()); + return transformer.toConfigBuilder(payload); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Error applying override to builder", e); + } + } + + /** + * Returns the one and only HostSystem of the root node + * Must be overridden by root node. + */ + public HostSystem getHostSystem() { + return getRoot().getHostSystem(); + } + + public AbstractConfigProducerRoot getRoot() { + return parent == null ? null : parent.getRoot(); + } + + /** + * Returns the {@link ApplicationConfigProducerRoot} that is the parent of this sub-tree, or null + * if this sub-tree has no Vespa parent. + */ + private ApplicationConfigProducerRoot getVespa() { + if (isRoot()) return null; + return isVespa() ? (ApplicationConfigProducerRoot)this : parent.getVespa(); + } + + private boolean isRoot() { + return parent == null; + } + + private boolean isVespa() { + return ((this instanceof ApplicationConfigProducerRoot) && parent.isRoot()); + } + + public AbstractConfigProducer getParent() { return parent; } + + /** + * Writes files that need to be written. The files will usually only be + * written when the Vespa model is generated through the deploy-application + * script. + * + * TODO: Make sure all implemented ConfigProducers call createConfig() + * instead of getConfig() when implementing this method. + */ + public void writeFiles(File directory) throws java.io.IOException { + if (!directory.isDirectory() && !directory.mkdirs()) { + throw new java.io.IOException("Cannot create directory: "+ directory); + } + for (Method m : getClass().getMethods()) { + try { + ConfigInstance.Builder builder = getBuilderIfIsGetConfig(m); + if (builder!=null) { + writeBuilder(directory, m, builder); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + + private void writeBuilder(File directory, Method m, + ConfigInstance.Builder builder) throws IllegalAccessException, + InvocationTargetException, InstantiationException, + NoSuchMethodException, IOException { + m.invoke(this, builder); + Class<?> configInstClass = builder.getClass().getEnclosingClass(); + ConfigInstance inst = (ConfigInstance) configInstClass.getConstructor(builder.getClass()).newInstance(builder); + List<String> payloadList = ConfigInstance.serialize(inst); + File outfn = new File(directory, ConfigInstance.getDefName(inst.getClass()) + ".MODEL.cfg"); + FileOutputStream out = new FileOutputStream(outfn); + for (String s : payloadList) { + out.write(Utf8.toBytes(s)); + out.write('\n'); + } + } + + /** + * New Builder instance if m is getConfig(SomeConfig.Builder), or null + */ + private ConfigInstance.Builder getBuilderIfIsGetConfig(Method m) throws InstantiationException, IllegalAccessException { + if (!"getConfig".equals(m.getName())) return null; + Type[] params = m.getParameterTypes(); + if (params.length!=1) return null; + Type param = params[0]; + if (!(param instanceof Class)) return null; + Class<?> paramClass = (Class<?>) param; + if (!(ConfigInstance.Builder.class.isAssignableFrom(paramClass))) return null; + return (ConfigInstance.Builder) paramClass.newInstance(); + } + + public void dump(PrintStream out) { + for (ConfigProducer c : getChildren().values()) { + out.println("id: " + c.getConfigId()); + if (c.getChildren().size() > 0) { + c.dump(out); + } + } + } + + void setupConfigId(String parentConfigId) { + if (this instanceof AbstractConfigProducerRoot) { + configId = ""; + } else { + configId = parentConfigId + subId; + addConfigId(configId); + } + + if (this instanceof AbstractConfigProducerRoot || this instanceof ApplicationConfigProducerRoot) { + setupChildConfigIds(""); + } else { + setupChildConfigIds(configId + '/'); + } + } + + private static ClassLoader findInheritedClassLoader(Class clazz, String producerName) { + Class<?>[] interfazes = clazz.getInterfaces(); + for (Class interfaze : interfazes) { + if (producerName.equals(interfaze.getName())) { + return interfaze.getClassLoader(); + } + } + if (clazz.getSuperclass() == null) + return null; + return findInheritedClassLoader(clazz.getSuperclass(), producerName); + } + + public ClassLoader getConfigClassLoader(String producerName) { + ClassLoader classLoader = findInheritedClassLoader(getClass(), producerName); + if (classLoader != null) + return classLoader; + + // TODO: Make logic correct, so that the deepest child will be the one winning. + for (AbstractConfigProducer child : childrenBySubId.values()) { + ClassLoader loader = child.getConfigClassLoader(producerName); + if (loader != null) { + return loader; + } + } + return null; + } + + private void setupChildConfigIds(String currentConfigId) { + for (AbstractConfigProducer child : childrenBySubId.values()) { + child.setupConfigId(currentConfigId); + } + } + + void aggregateDescendantServices() { + for (AbstractConfigProducer child : childrenBySubId.values()) { + child.aggregateDescendantServices(); + descendantServices.addAll(child.descendantServices); + } + } + + void freeze() { + childrenBySubId.freeze(); + for (AbstractConfigProducer child : childrenBySubId.values()) { + child.freeze(); + } + } + + public void mergeUserConfigs(UserConfigRepo newRepo) { + userConfigs.merge(newRepo); + } + + @Override + public void validate() throws Exception { + assert (childrenBySubId.isFrozen()); + + for (AbstractConfigProducer child : childrenBySubId.values()) { + child.validate(); + } + } + + /** Returns a logger to be used for warnings and messages during initialization, never null */ + public DeployLogger deployLogger() { + return parent.deployLogger(); + } + + // TODO: Make producers depend on AdminModel instead + /** Returns a monitoring service (yamas if that is configured, null otherwise) */ + protected MonitoringSystem getMonitoringService() { + AbstractConfigProducerRoot root = getRoot(); + Admin admin = (root == null? null : root.getAdmin()); + if (admin == null) { + return null; + } + if (admin.getYamas() != null) { + return admin.getYamas(); + } + return null; + } +} diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducerRoot.java new file mode 100644 index 00000000000..b85e2b8fa97 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducerRoot.java @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.producer; + +import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.vespa.model.ConfigProducer; +import com.yahoo.vespa.model.ConfigProducerRoot; +import com.yahoo.vespa.model.Service; +import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; + +/** + * The parent class of classes having the role as the root of a config producer tree. + * + * @author tonytv + */ +public abstract class AbstractConfigProducerRoot extends AbstractConfigProducer<AbstractConfigProducer<?>> + implements ConfigProducerRoot { + + /** The ConfigProducers contained in this model indexed by config id */ + protected final Map<String, ConfigProducer> id2producer = new LinkedHashMap<>(); + + public AbstractConfigProducerRoot(String rootConfigId) { + super(rootConfigId); + } + + public AbstractConfigProducerRoot getRoot() { + return this; + } + + public abstract FileDistributionConfigProducer getFileDistributionConfigProducer(); + + /** + * Freezes the parent - child connections of the model + * and sets information derived from the topology. + */ + public void freezeModelTopology() { + freeze(); + setupConfigId(""); + aggregateDescendantServices(); + } + + public abstract ConfigModelRepo configModelRepo(); + + /** + * Returns the ConfigProducer with the given id if such configId exists. + * + * @param configId The configId, e.g. "search.0/tld.0" + * @return ConfigProducer with the given configId + */ + public Optional<ConfigProducer> getConfigProducer(String configId) { + return Optional.ofNullable(id2producer.get(configId)); + } + + /** + * Returns the Service with the given id if such configId exists and it belongs to a Service ConfigProducer. + * + * @param configId The configId, e.g. "search.0/tld.0" + * @return Service with the given configId + */ + public Optional<Service> getService(String configId) { + return getConfigProducer(configId) + .filter(Service.class::isInstance) + .map(Service.class::cast); + } +} diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/UserConfigRepo.java b/config-model/src/main/java/com/yahoo/config/model/producer/UserConfigRepo.java new file mode 100644 index 00000000000..d32b0c75e69 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/producer/UserConfigRepo.java @@ -0,0 +1,109 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.producer; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.*; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +/** + * A UserConfigRepo is a repository for user configs, typically for a particular config producer. The repo encapsulates + * how the user configs are stored, and defines the methods to retrieve user configs and merge the repo with others. + * + * @author lulf + * @since 5.1 + */ +public class UserConfigRepo { + private final Map<ConfigDefinitionKey, ConfigPayloadBuilder> userConfigsMap; + + public UserConfigRepo() { + this.userConfigsMap = new LinkedHashMap<>(); + } + + @Override + public UserConfigRepo clone() { + return new UserConfigRepo(copyBuilders(userConfigsMap)); + } + + /** + * Must copy the builder, because the merge method on {@link AbstractConfigProducer} might override the row's builders otherwise + */ + private Map<ConfigDefinitionKey, ConfigPayloadBuilder> copyBuilders(Map<ConfigDefinitionKey, ConfigPayloadBuilder> source) { + Map<ConfigDefinitionKey, ConfigPayloadBuilder> ret = new LinkedHashMap<>(); + for (Map.Entry<ConfigDefinitionKey, ConfigPayloadBuilder> e : source.entrySet()) { + ConfigDefinitionKey key = e.getKey(); + ConfigPayloadBuilder sourceVal = e.getValue(); + ConfigPayloadBuilder destVal = new ConfigPayloadBuilder(ConfigPayload.fromBuilder(sourceVal)); + ret.put(key, destVal); + } + return ret; + } + + public UserConfigRepo(Map<ConfigDefinitionKey, ConfigPayloadBuilder> map) { + this.userConfigsMap = map; + } + + public UserConfigRepo(UserConfigRepo userConfigRepo) { + this.userConfigsMap = userConfigRepo.userConfigsMap; + } + + public ConfigPayloadBuilder get(ConfigDefinitionKey key) { + return userConfigsMap.get(key); + } + + public void merge(UserConfigRepo newRepo) { + for (Map.Entry<ConfigDefinitionKey, ConfigPayloadBuilder> entry : newRepo.userConfigsMap.entrySet()) { + if (entry.getValue() == null) continue; + + ConfigDefinitionKey key = entry.getKey(); + if (userConfigsMap.containsKey(key)) { + ConfigPayloadBuilder lhsBuilder = userConfigsMap.get(key); + ConfigPayloadBuilder rhsBuilder = entry.getValue(); + lhsBuilder.override(rhsBuilder); + } else { + userConfigsMap.put(key, entry.getValue()); + } + } + } + + public boolean isEmpty() { + return userConfigsMap.isEmpty(); + } + + public int size() { + return userConfigsMap.size(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (ConfigDefinitionKey key : userConfigsMap.keySet()) { + sb.append(key.toString()); + } + return sb.toString(); + } + + /** + * The keys of all the configs contained in this. + * @return a set of ConfigDefinitionsKey + */ + public Set<ConfigDefinitionKey> configsProduced() { + return userConfigsMap.keySet(); + } + + /** + * Will take the warning messages stored on the payload builders, and apply them to the producer's {@link DeployLogger} + * @param producer the producer to apply warnings to + */ + public void applyWarnings(AbstractConfigProducer<?> producer) { + for (ConfigPayloadBuilder b : userConfigsMap.values()) { + for (String warning : b.warnings()) { + producer.deployLogger().log(LogLevel.WARNING, warning); + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/package-info.java b/config-model/src/main/java/com/yahoo/config/model/producer/package-info.java new file mode 100644 index 00000000000..c58a116b196 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/producer/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.config.model.producer; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/Host.java b/config-model/src/main/java/com/yahoo/config/model/provision/Host.java new file mode 100644 index 00000000000..9d97308fc76 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/provision/Host.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.provision; + +import java.util.ArrayList; +import java.util.List; + +/** + * A hostname with zero or more aliases. + * + * @author musum + */ +public class Host { + + private final String hostname; + private final List<String> hostAliases; + + public Host(String hostname) { + this.hostname = hostname; + this.hostAliases = new ArrayList<>(); + } + + public Host(String hostname, List<String> hostAliases) { + this.hostname = hostname; + this.hostAliases = hostAliases; + } + + public String getHostname() { + return hostname; + } + + public List<String> getHostAliases() { + return hostAliases; + } + + @Override + public String toString() { + return hostname + " (aliases: " + hostAliases + ")"; + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java new file mode 100644 index 00000000000..ece781dd5bd --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java @@ -0,0 +1,120 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.provision; + +import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.log.LogLevel; +import com.yahoo.net.HostName; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.io.Reader; +import java.util.*; +import java.util.logging.Logger; + +/** + * TODO: What is this? + * + * @author musum + */ +public class Hosts { + + public static final Logger log = Logger.getLogger(Hosts.class.getPackage().toString()); + + private final HashMap<String, Host> hosts = new LinkedHashMap<>(); + private final Map<String, String> alias2hostname = new LinkedHashMap<>(); + private final Map<String, Host> alias2host = new LinkedHashMap<>(); + + /** + * Builds host system from a hosts.xml file + * + * @param hostsFile a reader for host from application package + * @return the HostSystem for this application package + */ + public static Hosts getHosts(Reader hostsFile) { + Hosts hosts = new Hosts(); + Document doc; + try { + doc = XmlHelper.getDocumentBuilder().parse(new InputSource(hostsFile)); + } catch (SAXException | IOException e) { + throw new IllegalArgumentException(e); + } + for (Element hostE : XML.getChildren(doc.getDocumentElement(), "host")) { + String name = hostE.getAttribute("name"); + if (name.equals("")) { + throw new RuntimeException("Missing 'name' attribute for host."); + } + if ("localhost".equals(name)) { + name = HostName.getLocalhost(); + } + final List<String> hostAliases = VespaDomBuilder.getHostAliases(hostE.getChildNodes()); + if (hostAliases.isEmpty()) { + throw new IllegalArgumentException("No host aliases defined for host '" + name + "'"); + } + Host host = new Host(name, hostAliases); + hosts.addHost(host, hostAliases); + } + log.log(LogLevel.DEBUG, "Created hosts:" + hosts); + return hosts; + } + + public Collection<Host> getHosts() { + return hosts.values(); + } + + /** + * Adds one host to this host system. + * + * @param host The host to add + * @param aliases The aliases for this host. + */ + public void addHost(Host host, List<String> aliases) { + hosts.put(host.getHostname(), host); + if ((aliases != null) && (aliases.size() > 0)) { + addHostAliases(aliases, host); + } + } + + /** + * Add all aliases for one host + * + * @param hostAliases A list of host aliases + * @param host The Host instance to add the alias for + */ + private void addHostAliases(List<String> hostAliases, Host host) { + if (hostAliases.size() < 1) { + throw new RuntimeException("Host '" + host.getHostname() + "' must have at least one <alias> tag."); + } + for (String alias : hostAliases) { + addHostAlias(alias, host); + } + } + + /** + * Adds an alias for the given host + * + * @param alias alias (string) for a Host + * @param host the {@link Host} to add the alias for + */ + protected void addHostAlias(String alias, Host host) { + if (alias2hostname.containsKey(alias)) { + throw new RuntimeException("Alias '" + alias + "' must be used for only one host!"); + } + alias2hostname.put(alias, host.getHostname()); + alias2host.put(alias, host); + } + + public Map<String, Host> getAlias2host() { + return alias2host; + } + + @Override + public String toString() { + return "Hosts: " + hosts.keySet(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java new file mode 100644 index 00000000000..20dc190d8e7 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.provision; + +import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.provision.*; +import com.yahoo.vespa.model.container.Container; + +import java.io.Reader; +import java.util.List; + +/** + * A host provisioner based on a hosts.xml file. + * No state in this provisioner, i.e it does not know anything about the active + * application if one exists. Pre-condition: A valid hosts file. + * + * @author musum + * @since 5.11 + */ +public class HostsXmlProvisioner implements HostProvisioner { + + private final Hosts hosts; + public static final String IMPLICIT_ADMIN_HOSTALIAS = "INTERNAL_VESPA_IMPLICIT_ADMIN"; + + public HostsXmlProvisioner(Reader hosts) { + this.hosts = Hosts.getHosts(hosts); + } + + @Override + public HostSpec allocateHost(String alias) { + /** + * Some special rules to allow no admin elements as well + * as jdisc element without nodes. + */ + if (alias.equals(IMPLICIT_ADMIN_HOSTALIAS)) { + if (hosts.getHosts().size() > 1) { + throw new IllegalArgumentException("More than 1 host specified (" + hosts.getHosts().size() + ") and <admin> not specified"); + } else { + return host2HostSpec(getFirstHost()); + } + } else if (alias.equals(Container.SINGLENODE_CONTAINER_SERVICESPEC)) { + return host2HostSpec(getFirstHost()); + } + for (Host host : hosts.getHosts()) { + if (host.getHostAliases().contains(alias)) { + return new HostSpec(host.getHostname(), host.getHostAliases()); + } + } + throw new IllegalArgumentException("Unable to find host for alias '" + alias + "'"); + } + + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity quantity, int groups, ProvisionLogger logger) { + throw new UnsupportedOperationException(); + } + + private HostSpec host2HostSpec(Host host) { + return new HostSpec(host.getHostname(), host.getHostAliases()); + } + + private Host getFirstHost() { + return hosts.getHosts().iterator().next(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java new file mode 100644 index 00000000000..c06efece329 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -0,0 +1,164 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.provision; + +import com.yahoo.collections.ListMap; +import com.yahoo.collections.Pair; +import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.provision.*; + +import java.util.*; + +/** + * In memory host provisioner. NB! ATM cannot be reused after allocate has been called. + * + * @author musum + * @author bratseth + */ +public class InMemoryProvisioner implements HostProvisioner { + + /** + * If this is true an exception is thrown when all nodes are used. + * If false this will simply return nodes best effort, preferring to satisfy the + * number of groups requested when possible. + */ + private final boolean failOnOutOfCapacity; + + /** Hosts which should be returned as retired */ + private final Set<String> retiredHostNames; + + /** Free hosts of each flavor */ + private final ListMap<String, Host> freeNodes = new ListMap<>(); + private final Map<String, HostSpec> legacyMapping = new LinkedHashMap<>(); + private final Map<ClusterSpec, List<HostSpec>> allocations = new LinkedHashMap<>(); + + /** Indexes must be unique across all groups in a cluster */ + private final Map<Pair<ClusterSpec.Type,ClusterSpec.Id>, Integer> nextIndexInCluster = new HashMap<>(); + + /** Use this index as start index for all clusters */ + private final int startIndexForClusters; + + /** Creates this with a number of nodes of the flavor 'default' */ + public InMemoryProvisioner(int nodeCount) { + this(Collections.singletonMap("default", createHostInstances(nodeCount)), true, 0); + } + + /** Creates this with a set of host names of the flavor 'default' */ + public InMemoryProvisioner(boolean failOnOutOfCapacity, String... hosts) { + this(Collections.singletonMap("default", toHostInstances(hosts)), failOnOutOfCapacity, 0); + } + + /** Creates this with a set of hosts of the flavor 'default' */ + public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, String ... retiredHostNames) { + this(Collections.singletonMap("default", hosts.getHosts()), failOnOutOfCapacity, 0, retiredHostNames); + } + + /** Creates this with a set of hosts of the flavor 'default' */ + public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) { + this(Collections.singletonMap("default", hosts.getHosts()), failOnOutOfCapacity, startIndexForClusters, retiredHostNames); + } + + public InMemoryProvisioner(Map<String, Collection<Host>> hosts, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) { + this.failOnOutOfCapacity = failOnOutOfCapacity; + for (Map.Entry<String, Collection<Host>> hostsOfFlavor : hosts.entrySet()) + for (Host host : hostsOfFlavor.getValue()) + freeNodes.put(hostsOfFlavor.getKey(), host); + this.retiredHostNames = new HashSet<>(Arrays.asList(retiredHostNames)); + this.startIndexForClusters = startIndexForClusters; + } + + private static Collection<Host> toHostInstances(String[] hostnames) { + List<Host> hosts = new ArrayList<>(); + for (String hostname : hostnames) { + hosts.add(new Host(hostname)); + } + return hosts; + } + + private static Collection<Host> createHostInstances(int hostCount) { + List<Host> hosts = new ArrayList<>(); + for (int i = 1; i <= hostCount; i++) { + hosts.add(new Host("host" + i)); + } + return hosts; + } + + @Override + public HostSpec allocateHost(String alias) { + if (legacyMapping.containsKey(alias)) return legacyMapping.get(alias); + List<Host> defaultHosts = freeNodes.get("default"); + if (defaultHosts.isEmpty()) throw new IllegalArgumentException("No more hosts of default flavor available"); + Host newHost = freeNodes.removeValue("default", 0); + HostSpec hostSpec = new HostSpec(newHost.getHostname(), newHost.getHostAliases()); + legacyMapping.put(alias, hostSpec); + return hostSpec; + } + + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity requestedCapacity, int groups, ProvisionLogger logger) { + if (cluster.group().isPresent() && groups > 1) + throw new IllegalArgumentException("Cannot both be specifying a group and ask for groups to be created"); + if (requestedCapacity.nodeCount() % groups != 0) + throw new IllegalArgumentException("Requested " + requestedCapacity.nodeCount() + " nodes in " + + groups + " groups, but the node count is not divisible into this number of groups"); + + int capacity = failOnOutOfCapacity ? requestedCapacity.nodeCount() : + Math.min(requestedCapacity.nodeCount(), freeNodes.get("default").size() + totalAllocatedTo(cluster)); + if (groups > capacity) + groups = capacity; + + String flavor = requestedCapacity.flavor().orElse("default"); + + List<HostSpec> allocation = new ArrayList<>(); + if (groups == 1) { + allocation.addAll(allocateHostGroup(cluster, flavor, capacity, startIndexForClusters)); + } + else { + for (int i = 0; i < groups; i++) { + allocation.addAll(allocateHostGroup(cluster.changeGroup(Optional.of(ClusterSpec.Group.from(String.valueOf(i)))), + flavor, + capacity / groups, + allocation.size())); + } + } + for (ListIterator<HostSpec> i = allocation.listIterator(); i.hasNext(); ) { + HostSpec host = i.next(); + if (retiredHostNames.contains(host.hostname())) + i.set(retire(host)); + } + return allocation; + } + + private HostSpec retire(HostSpec host) { + return new HostSpec(host.hostname(), host.aliases(), host.membership().get().retire()); + } + + private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, String flavor, int nodesInGroup, int startIndex) { + List<HostSpec> allocation = allocations.getOrDefault(clusterGroup, new ArrayList<>()); + allocations.put(clusterGroup, allocation); + + int nextIndex = nextIndexInCluster.getOrDefault(new Pair<>(clusterGroup.type(), clusterGroup.id()), startIndex); + while (allocation.size() < nodesInGroup) { + if (freeNodes.get(flavor).isEmpty()) throw new IllegalArgumentException("No nodes of flavor '" + flavor + "' available"); + Host newHost = freeNodes.removeValue(flavor, 0); + ClusterMembership membership = ClusterMembership.from(clusterGroup, nextIndex++); + allocation.add(new HostSpec(newHost.getHostname(), newHost.getHostAliases(), membership)); + } + nextIndexInCluster.put(new Pair<>(clusterGroup.type(), clusterGroup.id()), nextIndex); + + while (allocation.size() > nodesInGroup) + allocation.remove(0); + + return allocation; + } + + private int totalAllocatedTo(ClusterSpec cluster) { + int count = 0; + for (Map.Entry<ClusterSpec, List<HostSpec>> allocation : allocations.entrySet()) { + if ( ! allocation.getKey().type().equals(cluster.type())) continue; + if ( ! allocation.getKey().id().equals(cluster.id())) continue; + count += allocation.getValue().size(); + } + return count; + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java new file mode 100644 index 00000000000..38ed728e4e8 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.provision; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.provision.*; +import com.yahoo.net.HostName; +import com.yahoo.vespa.model.HostSystem; + +import java.util.ArrayList; +import java.util.List; +import java.net.UnknownHostException; + +/** + * A host provisioner used when there is no hosts.xml file (using localhost as the only host) + * No state in this provisioner, i.e it does not know anything about the active + * application if one exists. + * + * @author musum + * @since 5.11 + */ +public class SingleNodeProvisioner implements HostProvisioner { + + private final Host host; // the only host in this system + private final HostSpec hostSpec; + private int counter = 0; + + public SingleNodeProvisioner() { + try { + host = new Host(HostSystem.lookupCanonicalHostname(HostName.getLocalhost())); + } catch (UnknownHostException e) { + throw new RuntimeException(e); + } + this.hostSpec = new HostSpec(host.getHostname(), host.getHostAliases()); + } + + @Override + public HostSpec allocateHost(String alias) { + return hostSpec; + } + + @Override + public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { // TODO: This should fail if capacity requested is more than 1 + List<HostSpec> hosts = new ArrayList<>(); + hosts.add(new HostSpec(host.getHostname(), host.getHostAliases(), ClusterMembership.from(cluster, counter++))); + return hosts; + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/package-info.java b/config-model/src/main/java/com/yahoo/config/model/provision/package-info.java new file mode 100644 index 00000000000..a39e4025efa --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/provision/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.config.model.provision; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/config/model/test/ConfigModelTestUtil.java b/config-model/src/main/java/com/yahoo/config/model/test/ConfigModelTestUtil.java new file mode 100644 index 00000000000..b75147774ab --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/test/ConfigModelTestUtil.java @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.test; + +import com.yahoo.collections.CollectionUtil; +import com.yahoo.config.model.builder.xml.XmlHelper; +import org.w3c.dom.Element; +import org.xml.sax.InputSource; + +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * @author lulf + * @since 5.1 + */ +public class ConfigModelTestUtil { + /** + * @param xmlLines XML with " replaced with ' + */ + public static Element parse(String... xmlLines) { + List<String> lines = new ArrayList<>(); + lines.add("<?xml version='1.0' encoding='utf-8' ?>"); + lines.addAll(Arrays.asList(xmlLines)); + + try { + return XmlHelper.getDocumentBuilder().parse( + inputSource((CollectionUtil.mkString(lines, "\n").replace("'", "\"")))) + .getDocumentElement(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static InputSource inputSource(String str) { + return new InputSource(new StringReader(str)); + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java new file mode 100644 index 00000000000..731410c9bf3 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -0,0 +1,243 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.test; + +import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.application.api.UnparsedConfigDefinition; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.provision.Version; +import com.yahoo.io.IOUtils; +import com.yahoo.path.Path; +import com.yahoo.io.reader.NamedReader; +import com.yahoo.searchdefinition.*; +import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.config.application.api.ApplicationPackage; + +import java.io.File; +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.util.*; + +/** + * For testing purposes only + * + * @author tonytv + */ +public class MockApplicationPackage implements ApplicationPackage { + public static final String MUSIC_SEARCHDEFINITION = createSearchDefinition("music", "foo"); + public static final String BOOK_SEARCHDEFINITION = createSearchDefinition("book", "bar"); + + private final String hostsS; + private final String servicesS; + private final List<String> searchDefinitions; + private final String searchDefinitionDir; + private final Optional<String> deploymentInfo; + private final Optional<String> validationOverrides; + private final boolean failOnValidateXml; + + private MockApplicationPackage(String hosts, String services, List<String> searchDefinitions, String searchDefinitionDir, + String deploymentInfo, String validationOverrides, boolean failOnValidateXml) { + this.hostsS = hosts; + this.servicesS = services; + this.searchDefinitions = searchDefinitions; + this.searchDefinitionDir = searchDefinitionDir; + this.deploymentInfo = Optional.ofNullable(deploymentInfo); + this.validationOverrides = Optional.ofNullable(validationOverrides); + this.failOnValidateXml = failOnValidateXml; + } + + @Override + public String getApplicationName() { + return "mock application"; + } + + @Override + public Reader getServices() { + return new StringReader(servicesS); + } + + @Override + public Reader getHosts() { + if (hostsS==null) return null; + return new StringReader(hostsS); + } + + @Override + public List<NamedReader> getSearchDefinitions() { + ArrayList<NamedReader> readers = new ArrayList<>(); + SearchBuilder searchBuilder = new SearchBuilder(this, new RankProfileRegistry()); + for (String sd : searchDefinitions) { + try { + String name = searchBuilder.importString(sd); + readers.add(new NamedReader(name + ApplicationPackage.SD_NAME_SUFFIX, new StringReader(sd))); + } catch (ParseException e) { + throw new RuntimeException(e); + } + } + return readers; + } + + @Override + public List<NamedReader> searchDefinitionContents() { + return new ArrayList<>(); + } + + @Override + public Map<ConfigDefinitionKey, UnparsedConfigDefinition> getAllExistingConfigDefs() { + return Collections.emptyMap(); + } + + @Override + public List<NamedReader> getFiles(Path dir,String fileSuffix,boolean recurse) { + return new ArrayList<>(); + } + + @Override + public ApplicationFile getFile(Path file) { + throw new UnsupportedOperationException(); + } + + @Override + public String getHostSource() { + return "mock source"; + } + + @Override + public String getServicesSource() { + return "mock source"; + } + + @Override + public Optional<Reader> getDeployment() { + return deploymentInfo.map(StringReader::new); + } + + @Override + public Optional<Reader> getValidationOverrides() { + return validationOverrides.map(StringReader::new); + } + + public List<ComponentInfo> getComponentsInfo(Version vespaVersion) { + return Collections.emptyList(); + } + + @Override + public Reader getRankingExpression(String name) { + File expressionFile = new File(searchDefinitionDir, name); + try { + return IOUtils.createReader(expressionFile, "utf-8"); + } + catch (IOException e) { + throw new IllegalArgumentException("Could not read ranking expression file '" + + expressionFile.getAbsolutePath() + "'", e); + } + } + + public static ApplicationPackage createEmpty() { + return new MockApplicationPackage.Builder().withHosts(emptyHosts).withServices(emptyServices).build(); + } + + public static ApplicationPackage fromSearchDefinitionDirectory(String dir) { + return new MockApplicationPackage.Builder() + .withEmptyHosts() + .withEmptyServices() + .withSearchDefinitionDir(dir).build(); + } + + public static class Builder { + private String hosts = null; + private String services = null; + private List<String> searchDefinitions = Collections.emptyList(); + private String searchDefinitionDir = null; + private String deploymentInfo = null; + private String validationOverrides = null; + private boolean failOnValidateXml = false; + + public Builder() { + } + + public Builder withEmptyHosts() { + return this.withHosts(emptyHosts); + } + + public Builder withHosts(String hosts) { + this.hosts = hosts; + return this; + } + + public Builder withEmptyServices() { + return this.withServices(emptyServices); + } + + public Builder withServices(String services) { + this.services = services; + return this; + } + + public Builder withSearchDefinition(String searchDefinition) { + this.searchDefinitions = Collections.singletonList(searchDefinition); + return this; + } + + public Builder withSearchDefinitions(List<String> searchDefinition) { + this.searchDefinitions = Collections.unmodifiableList(searchDefinition); + return this; + } + + public Builder withSearchDefinitionDir(String searchDefinitionDir) { + this.searchDefinitionDir = searchDefinitionDir; + return this; + } + + public Builder withDeploymentInfo(String deploymentInfo) { + this.deploymentInfo = deploymentInfo; + return this; + } + + public Builder withValidationOverrides(String validationOverrides) { + this.validationOverrides = validationOverrides; + return this; + } + + public Builder failOnValidateXml() { + this.failOnValidateXml = true; + return this; + } + + public ApplicationPackage build() { + return new MockApplicationPackage(hosts, services, searchDefinitions, searchDefinitionDir, + deploymentInfo, validationOverrides, failOnValidateXml); + } + } + + public static String createSearchDefinition(String name, String fieldName) { + return "search " + name + " {" + + " document " + name + " {" + + " field " + fieldName + " type string {}" + + " }" + + "}"; + } + + private static final String emptyServices = "<services version=\"1.0\">" + + " <admin version=\"2.0\">" + + " <adminserver hostalias=\"node1\" />" + + " </admin>" + + "</services>"; + + private static final String emptyHosts = "<hosts>" + + " <host name=\"localhost\">" + + " <alias>node1</alias>" + + " </host>" + + "</hosts>"; + + @Override + public void validateXML(DeployLogger logger) throws IOException { + if (failOnValidateXml) { + throw new IllegalArgumentException("Error in application package"); + } else { + throw new UnsupportedOperationException("This application package cannot validate XML"); + } + } +} diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java new file mode 100644 index 00000000000..fa84cf1c7eb --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java @@ -0,0 +1,169 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.test; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.model.ConfigModelRepo; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.builder.xml.XmlHelper; +import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.model.producer.AbstractConfigProducerRoot; +import com.yahoo.text.XML; +import com.yahoo.vespa.model.ConfigProducer; +import com.yahoo.vespa.model.HostSystem; +import com.yahoo.vespa.model.admin.Admin; +import com.yahoo.vespa.model.builder.xml.dom.DomAdminV2Builder; +import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; +import com.yahoo.vespa.model.filedistribution.FileDistributor; +import org.w3c.dom.Document; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.io.StringReader; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Set; + + +/** + * Use for testing. Use as parent for the config producer(s) you want to test, to test + * only a subtree of the producers. + * + * @author gjoranv + */ +public class MockRoot extends AbstractConfigProducerRoot { + private static final long serialVersionUID = 1L; + public static final String MOCKHOST = "mockhost"; + + private HostSystem hostSystem; + + private final DeployState deployState; + private FileDistributor fileDistributor; + private Admin admin; + + public MockRoot() { + this(""); + } + + public MockRoot(String rootConfigId) { + this(rootConfigId, new MockApplicationPackage.Builder().build()); + } + + public MockRoot(String rootConfigId, ApplicationPackage applicationPackage) { + this(rootConfigId, new DeployState.Builder().applicationPackage(applicationPackage).build()); + } + + public MockRoot(String rootConfigId, DeployState deployState) { + super(rootConfigId); + hostSystem = new HostSystem(this, "hostsystem", deployState.getProvisioner()); + this.deployState = deployState; + fileDistributor = new FileDistributor(deployState.getFileRegistry()); + } + + public FileDistributionConfigProducer getFileDistributionConfigProducer() { + return null; + } + + @Override + public ConfigModelRepo configModelRepo() { + return new ConfigModelRepo(); + } + + public Set<String> getConfigIds() { + return Collections.unmodifiableSet(id2producer.keySet()); + } + + @Override + public ConfigInstance.Builder getConfig(ConfigInstance.Builder builder, String configId) { + ConfigProducer cp = id2producer.get(configId); + if (cp == null) return null; + + cp.cascadeConfig(builder); + cp.addUserConfig(builder); + return builder; + } + + @SuppressWarnings("unchecked") + public <T extends ConfigInstance> T getConfig(Class<T> configClass, String configId) { + try { + ConfigInstance.Builder builder = getConfig(getBuilder(configClass).newInstance(), configId); + return configClass.getConstructor(builder.getClass()).newInstance(builder); + } catch (InstantiationException | NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + public ConfigProducer getProducer(String configId) { + return id2producer.get(configId); + } + + @SuppressWarnings("unchecked") + public static <T extends ConfigInstance> Class<? extends ConfigInstance.Builder> getBuilder(Class<T> configClass) { + for (Class<?> memberClass : configClass.getClasses()) { + if (memberClass.getSimpleName().equals("Builder")) + return (Class<? extends ConfigInstance.Builder>) memberClass; + } + throw new RuntimeException("Missing builder"); + } + + @Override + public DeployState getDeployState() { + return deployState; + } + + public FileDistributor getFileDistributor() { + return fileDistributor; + } + + public HostSystem getHostSystem() { + return hostSystem; + } + + public void addDescendant(String configId, AbstractConfigProducer descendant) { + if (id2producer.containsKey(configId)) { + throw new RuntimeException + ("Config ID '" + configId + "' cannot be reserved by an instance of class '" + + descendant.getClass().getName() + "' since it is already used by an instance of class '" + + id2producer.get(configId).getClass().getName() + "'"); + } + id2producer.put(configId, descendant); + } + + @Override + public void addChild(AbstractConfigProducer abstractConfigProducer) { + super.addChild(abstractConfigProducer); + } + + public final void setAdmin(String xml) { + String servicesXml = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + xml + "</services>"; + + try { + Document doc = XmlHelper.getDocumentBuilder().parse(new InputSource(new StringReader(servicesXml))); + setAdmin(new DomAdminV2Builder(deployState.getFileRegistry(), false, new ArrayList<>()). + build(this, XML.getChildren(doc.getDocumentElement(), "admin").get(0))); + } catch (SAXException | IOException e) { + throw new RuntimeException(e); + } + } + + public final void setAdmin(Admin admin) { + this.admin = admin; + } + + @Override + public final Admin getAdmin() { + return admin; + } + + @Override + public DeployLogger deployLogger() { + return new BaseDeployLogger(); + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java b/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java new file mode 100644 index 00000000000..4518e44d441 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java @@ -0,0 +1,111 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.test; + +import com.google.common.annotations.Beta; +import com.yahoo.config.model.MapConfigModelRegistry; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.application.provider.SchemaValidator; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.builder.xml.ConfigModelBuilder; +import com.yahoo.vespa.model.VespaModel; +import org.xml.sax.SAXException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Test driver for testing config models. Add custom builders for plugins to be tested. Builds a model from the + * xml string and returns a config producer that can be use to test getConfig. + * + * @author lulf + * @since 5.1.20 + */ +@Beta +public class TestDriver { + + private final List<ConfigModelBuilder> builders = new ArrayList<>(); + private final boolean validate; + + public TestDriver(boolean validate) { + this.validate = validate; + } + + public TestDriver() { + this(false); + } + + /** + * Add a new builder to the tester. + * + * @param builder builder to add. + * @return this for chaining + */ + public TestDriver addBuilder(ConfigModelBuilder builder) { + builders.add(builder); + return this; + } + + /** + * Build a model from an XML string. The hosts referenced in services must be set to 'mockhost' when using + * this method, as it automatically generates a hosts file for you. + * + * @param servicesXml The xml for services.xml + * @return a producer root capable of answering getConfig requests. + */ + public TestRoot buildModel(String servicesXml) { + return buildModel(servicesXml, "<hosts><host name='localhost'><alias>mockhost</alias></host></hosts>"); + } + + /** + * Build a model from an XML string of services and one of hosts. + * + * @param servicesXml The xml for services.xml + * @param hostsXml The xml for hosts.xml + * @return a producer root capable of answering getConfig requests. + */ + public TestRoot buildModel(String servicesXml, String hostsXml) { + if (!servicesXml.contains("<services")) { + servicesXml = "<services version='1.0'>" + servicesXml + "</services>"; + } + return buildModel(new MockApplicationPackage.Builder().withHosts(hostsXml).withServices(servicesXml).build()); + } + + /** + * Build a model from an application package. + * + * @param applicationPackage Any type of application package. + * @return a producer root capable of answering getConfig requests. + */ + public TestRoot buildModel(ApplicationPackage applicationPackage) { + return buildModel(new DeployState.Builder().applicationPackage(applicationPackage).build()); + } + + /** + * Build a model given a deploy state. + * + * @param deployState An instance of {@link com.yahoo.config.model.deploy.DeployState} + * @return a producer root capable of answering getConfig requests. + */ + public TestRoot buildModel(DeployState deployState) { + MapConfigModelRegistry registry = new MapConfigModelRegistry(builders); + try { + validate(deployState.getApplicationPackage()); + return new TestRoot(new VespaModel(registry, deployState)); + } catch (IOException | SAXException e) { + throw new RuntimeException(e); + } + } + + private void validate(ApplicationPackage appPkg) throws IOException { + if (!validate) { + return; + } + SchemaValidator validator = SchemaValidator.createTestValidatorHosts(); + if (appPkg.getHosts() != null) { + validator.validate(appPkg.getHosts()); + } + validator = SchemaValidator.createTestValidatorServices(); + validator.validate(appPkg.getServices()); + } +} diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java new file mode 100644 index 00000000000..44491e33479 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.test; + +import com.google.common.annotations.Beta; +import com.yahoo.config.ConfigInstance; +import com.yahoo.config.model.ConfigModel; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.VespaModel; + +import java.util.List; + +/** + * Test utility class that provides many methods for inspecting the state of a completely built model + * + * @author lulf + * @since 5.1 + */ +@Beta +public class TestRoot { + private final VespaModel model; + TestRoot(VespaModel model) { + this.model = model; + } + + /** + * Get a list of all config models of a particular type. + * + * @param clazz The class of the models to find. + * @return A list of models of given type. + */ + public <MODEL extends ConfigModel> List<MODEL> getConfigModels(Class<MODEL> clazz) { + return model.configModelRepo().getModels(clazz); + } + + /** + * Ask model to populate builder with config for a given config id. This method gives the same config as the + * configserver would return to its clients. + * @param builder The builder to populate + * @param configId The config id of the producer to ask for config. + * @return the same builder. + */ + public <BUILDER extends ConfigInstance.Builder> BUILDER getConfig(BUILDER builder, String configId) { + return (BUILDER) model.getConfig(builder, configId); + } + + /** + * Request config of a given type and id. This method gives the same config as the configserver would return to + * its clients. + * + * @param clazz Type of config to request. + * @param configId The config id of the producer to ask for config. + * @return A config object of the appropriate type with config values set. + */ + public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> clazz, String configId) { + return model.getConfig(clazz, configId); + } + + /** + * Retrieve the hosts available in this model. Useful to verify that hostnames are set correctly etc. + * + * @return A list of hosts. + */ + public List<HostResource> getHosts() { + return model.getHostSystem().getHosts(); + } +} diff --git a/config-model/src/main/java/com/yahoo/config/model/test/package-info.java b/config-model/src/main/java/com/yahoo/config/model/test/package-info.java new file mode 100644 index 00000000000..6c33505df83 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/config/model/test/package-info.java @@ -0,0 +1,6 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +//TODO: Temporary export due to standalone container package, remove later. +@ExportPackage +package com.yahoo.config.model.test; + +import com.yahoo.osgi.annotation.ExportPackage; |