summaryrefslogtreecommitdiffstats
path: root/config-model/src/main/java/com/yahoo/config
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@yahoo-inc.com>2016-06-15 23:09:44 +0200
committerJon Bratseth <bratseth@yahoo-inc.com>2016-06-15 23:09:44 +0200
commit72231250ed81e10d66bfe70701e64fa5fe50f712 (patch)
tree2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /config-model/src/main/java/com/yahoo/config
Publish
Diffstat (limited to 'config-model/src/main/java/com/yahoo/config')
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java279
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java34
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/ConfigModel.java64
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java72
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/ConfigModelInstanceFactory.java20
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/ConfigModelRegistry.java51
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java256
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/ConfigModelRepoAdder.java16
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/ConfigModelUtils.java64
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/MapConfigModelRegistry.java63
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/NullConfigModelRegistry.java21
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java113
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java119
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java94
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java135
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/builder/xml/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java15
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java115
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java381
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java53
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/deploy/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/graph/ModelGraph.java80
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/graph/ModelGraphBuilder.java52
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/graph/ModelNode.java135
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java448
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducerRoot.java67
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/producer/UserConfigRepo.java109
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/producer/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/provision/Host.java40
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java120
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java64
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java164
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java49
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/provision/package-info.java5
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/test/ConfigModelTestUtil.java40
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java243
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java169
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java111
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java66
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/test/package-info.java6
41 files changed, 3953 insertions, 0 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java
new file mode 100644
index 00000000000..1ef0ed6ec0a
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java
@@ -0,0 +1,279 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model;
+
+import com.yahoo.cloud.config.*;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Version;
+import com.yahoo.vespa.config.content.LoadTypeConfig;
+import com.yahoo.cloud.config.ModelConfig.Hosts;
+import com.yahoo.cloud.config.ModelConfig.Hosts.Services;
+import com.yahoo.cloud.config.ModelConfig.Hosts.Services.Ports;
+import com.yahoo.cloud.config.log.LogdConfig;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.document.DocumenttypesConfig;
+import com.yahoo.document.config.DocumentmanagerConfig;
+import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig;
+import com.yahoo.messagebus.MessagebusConfig;
+import com.yahoo.vespa.configmodel.producers.DocumentManager;
+import com.yahoo.vespa.configmodel.producers.DocumentTypes;
+import com.yahoo.vespa.documentmodel.DocumentModel;
+import com.yahoo.vespa.model.*;
+import com.yahoo.vespa.model.admin.Admin;
+import com.yahoo.vespa.model.clients.Clients;
+import com.yahoo.vespa.model.content.cluster.ContentCluster;
+import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer;
+import com.yahoo.vespa.model.routing.Routing;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+/**
+ * This is the parent of all ConfigProducers in the system resulting from configuring an application.
+ *
+ * @author gjoranv
+ */
+public class ApplicationConfigProducerRoot extends AbstractConfigProducer<AbstractConfigProducer<?>> implements CommonConfigsProducer {
+
+ private final DocumentModel documentModel;
+ private Routing routing = null;
+ // The ConfigProducers contained in this Vespa. (configId->producer)
+ Map<String, ConfigProducer> id2producer = new LinkedHashMap<>();
+ private Admin admin = null;
+ private HostSystem hostSystem = null;
+ private final Version vespaVersion;
+ private final ApplicationId applicationId;
+
+ /**
+ * Creates and initializes a new Vespa from the service config file
+ * in the given application directory.
+ *
+ * @param parent The parent, usually VespaModel
+ * @param name The name, used as configId
+ * @param documentModel DocumentModel to serve global document config from.
+ */
+ public ApplicationConfigProducerRoot(AbstractConfigProducer parent, String name, DocumentModel documentModel, Version vespaVersion, ApplicationId applicationId) {
+ super(parent, name);
+ this.documentModel = documentModel;
+ this.vespaVersion = vespaVersion;
+ this.applicationId = applicationId;
+ }
+
+ /**
+ * @return an unmodifiable copy of the set of configIds in this VespaModel.
+ */
+ public Set<String> getConfigIds() {
+ return Collections.unmodifiableSet(id2producer.keySet());
+ }
+
+ /**
+ * Returns the ConfigProducer with the given id, or null if no such
+ * configId exists.
+ *
+ * @param configId The configId, e.g. "search.0/tld.0"
+ * @return ConfigProducer with the given configId
+ */
+ public ConfigProducer getConfigProducer(String configId) {
+ return id2producer.get(configId);
+ }
+
+ /**
+ * Returns the Service with the given id, or null if no such
+ * configId exists or if it belongs to a non-Service ConfigProducer.
+ *
+ * @param configId The configId, e.g. "search.0/tld.0"
+ * @return Service with the given configId
+ */
+ public Service getService(String configId) {
+ ConfigProducer cp = getConfigProducer(configId);
+ if (cp == null || !(cp instanceof Service)) {
+ return null;
+ }
+ return (Service) cp;
+ }
+
+ /**
+ * Adds the descendant (at any depth level), so it can be looked up
+ * on configId in the Map.
+ *
+ * @param descendant The configProducer descendant to add
+ */
+ // TODO: Make protected if this moves to the same package as AbstractConfigProducer
+ public void addDescendant(AbstractConfigProducer descendant) {
+ id2producer.put(descendant.getConfigId(), descendant);
+ }
+
+ /**
+ * Prepares the model for start. The {@link VespaModel} calls
+ * this methods after it has loaded this and all plugins have been loaded and
+ * their initialize() methods have been called.
+ *
+ * @param plugins All initialized plugins of the vespa model.
+ */
+ public void prepare(ConfigModelRepo plugins) {
+ if (routing != null) {
+ routing.deriveCommonSettings(plugins);
+ }
+ }
+
+ public void setupAdmin(Admin admin) {
+ this.admin = admin;
+ }
+
+ public void setupRouting(ConfigModelRepo configModels) {
+ if (admin != null) {
+ Routing routing = configModels.getRouting();
+ if (routing == null) {
+ routing = new Routing(ConfigModelContext.createFromParentAndId(configModels, this, "routing"));
+ configModels.add(routing);
+ }
+ this.routing = routing;
+ }
+ }
+
+ @Override
+ public void getConfig(DocumentmanagerConfig.Builder builder) {
+ new DocumentManager().produce(documentModel, builder);
+ }
+
+ @Override
+ public void getConfig(DocumenttypesConfig.Builder builder) {
+ new DocumentTypes().produce(documentModel, builder);
+ }
+
+ @Override
+ public void getConfig(DocumentrouteselectorpolicyConfig.Builder builder) {
+ if (routing != null) {
+ routing.getConfig(builder);
+ }
+ }
+
+ @Override
+ public void getConfig(MessagebusConfig.Builder builder) {
+ if (routing != null) {
+ routing.getConfig(builder);
+ }
+ }
+
+ @Override
+ public void getConfig(LogdConfig.Builder builder) {
+ if (admin != null) {
+ admin.getConfig(builder);
+ }
+ }
+
+ @Override
+ public void getConfig(SlobroksConfig.Builder builder) {
+ if (admin != null) {
+ admin.getConfig(builder);
+ }
+ }
+
+ @Override
+ public void getConfig(ZookeepersConfig.Builder builder) {
+ if (admin != null) {
+ admin.getConfig(builder);
+ }
+ }
+
+ @Override
+ public void getConfig(LoadTypeConfig.Builder builder) {
+ VespaModel model = (VespaModel) getRoot();
+ Clients clients = model.getClients();
+ if (clients != null) {
+ clients.getConfig(builder);
+ }
+ }
+
+ @Override
+ public void getConfig(ClusterListConfig.Builder builder) {
+ VespaModel model = (VespaModel) getRoot();
+ for (ContentCluster cluster : model.getContentClusters().values()) {
+ ClusterListConfig.Storage.Builder storage = new ClusterListConfig.Storage.Builder();
+ storage.name(cluster.getName());
+ storage.configid(cluster.getConfigId());
+ builder.storage(storage);
+ }
+ }
+
+ @Override
+ public void getConfig(ModelConfig.Builder builder) {
+ builder.vespaVersion(vespaVersion.toSerializedForm());
+ for (HostResource modelHost : getHostSystem().getHosts()) {
+ builder.hosts(new Hosts.Builder()
+ .name(modelHost.getHostName())
+ .services(getServices(modelHost))
+ );
+ }
+ }
+
+ private List<Services.Builder> getServices(HostResource modelHost) {
+ List<Services.Builder> ret = new ArrayList<>();
+ for (Service modelService : modelHost.getServices()) {
+ ret.add(new Services.Builder()
+ .name(modelService.getServiceName())
+ .type(modelService.getServiceType())
+ .configid(modelService.getConfigId())
+ .clustertype(modelService.getServicePropertyString("clustertype", ""))
+ .clustername(modelService.getServicePropertyString("clustername", ""))
+ .index(Integer.parseInt(modelService.getServicePropertyString("index", "999999")))
+ .ports(getPorts(modelService))
+ );
+ }
+ return ret;
+ }
+
+ private List<Ports.Builder> getPorts(Service modelService) {
+ List<Ports.Builder> ret = new ArrayList<>();
+ PortsMeta portsMeta = modelService.getPortsMeta();
+ for (int i = 0; i < portsMeta.getNumPorts(); i++) {
+ ret.add(new Ports.Builder()
+ .number(modelService.getRelativePort(i))
+ .tags(getPortTags(portsMeta, i))
+ );
+ }
+ return ret;
+ }
+
+ public static String getPortTags(PortsMeta portsMeta, int portNumber) {
+ StringBuilder sb = new StringBuilder();
+ boolean firstTag = true;
+ for (String s : portsMeta.getTagsAt(portNumber)) {
+ if (!firstTag) {
+ sb.append(" ");
+ } else {
+ firstTag = false;
+ }
+ sb.append(s);
+ }
+ return sb.toString();
+ }
+
+ public void setHostSystem(HostSystem hostSystem) {
+ this.hostSystem = hostSystem;
+ }
+
+ @Override
+ public HostSystem getHostSystem() {
+ return hostSystem;
+ }
+
+ public FileDistributionConfigProducer getFileDistributionConfigProducer() {
+ return admin.getFileDistributionConfigProducer();
+ }
+
+ public Admin getAdmin() {
+ return admin;
+ }
+
+ @Override
+ public void getConfig(ApplicationIdConfig.Builder builder) {
+ builder.tenant(applicationId.tenant().value());
+ builder.application(applicationId.application().value());
+ builder.instance(applicationId.instance().value());
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java b/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java
new file mode 100644
index 00000000000..228c74480c4
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java
@@ -0,0 +1,34 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model;
+
+import com.yahoo.cloud.config.ApplicationIdConfig;
+import com.yahoo.vespa.config.content.LoadTypeConfig;
+import com.yahoo.cloud.config.log.LogdConfig;
+import com.yahoo.cloud.config.SlobroksConfig;
+import com.yahoo.cloud.config.ClusterListConfig;
+import com.yahoo.cloud.config.ZookeepersConfig;
+import com.yahoo.cloud.config.ModelConfig;
+import com.yahoo.document.DocumenttypesConfig;
+import com.yahoo.document.config.DocumentmanagerConfig;
+import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig;
+import com.yahoo.messagebus.MessagebusConfig;
+
+
+/**
+ * This interface describes the configs that are produced by the model producer root.
+ *
+ * @author lulf
+ * @since 5.1
+ */
+public interface CommonConfigsProducer extends DocumentmanagerConfig.Producer,
+ DocumenttypesConfig.Producer,
+ MessagebusConfig.Producer,
+ DocumentrouteselectorpolicyConfig.Producer,
+ LogdConfig.Producer,
+ SlobroksConfig.Producer,
+ ZookeepersConfig.Producer,
+ LoadTypeConfig.Producer,
+ ClusterListConfig.Producer,
+ ModelConfig.Producer,
+ ApplicationIdConfig.Producer {
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java
new file mode 100644
index 00000000000..10342d44914
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java
@@ -0,0 +1,64 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model;
+
+/**
+ * A config model is an abstract representation of a subsystem, which is used
+ * by the builder of that subsystem to derive a config producer tree for the subsystem, and by other
+ * builders to access information about the subsystem for config production at a suitable abstraction level.
+ *
+ * @author gjoranv
+ * @author bratseth
+ * @author lulf
+ */
+public abstract class ConfigModel {
+
+ private final String id;
+
+ /**
+ * Constructs a new config model given a context.
+ *
+ * @param modelContext The model context.
+ */
+ public ConfigModel(ConfigModelContext modelContext) {
+ super();
+ this.id = modelContext.getProducerId();
+ }
+
+ /** Returns the id of this model */
+ public final String getId() { return id; }
+
+ /**
+ * Initializes this model. All inter-model independent initialization
+ * is done by implementing this method.
+ * The model will be made available to dependent models by the framework when this returns.
+ * <p>
+ * TODO: Remove this method, as this is now done by the model builders.
+ *
+ * This default implementation does nothing.
+ *
+ * @param configModelRepo The ConfigModelRepo of the VespaModel
+ * @deprecated This will go away in the next Vespa major release. Instead, inject the models you depend on
+ * in your config model constructor.
+ */
+ public void initialize(ConfigModelRepo configModelRepo) { return; }
+
+ /**
+ * Prepares this model to start serving config requests, possibly using properties of other models.
+ * The framework will call this method after models have been built. The model
+ * should finalize its configurations that depend on other models in this step.
+ *
+ * This default implementation does nothing.
+ *
+ * @param configModelRepo The ConfigModelRepo of the system model
+ */
+ public void prepare(ConfigModelRepo configModelRepo) { return; }
+
+ /**
+ * <p>Returns whether this model must be maintained in memory for serving config requests.
+ * Models which are used to amend other models at build time should override this to return false.</p>
+ *
+ * <p>This default implementation returns true.</p>
+ */
+ public boolean isServing() { return true; }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java
new file mode 100644
index 00000000000..c375d8e28fa
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java
@@ -0,0 +1,72 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model;
+
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+
+/**
+ * This class contains a context that is passed to a model builder, and can be used to retrieve the application package,
+ * logger etc.
+ *
+ * @author lulf
+ * @since 5.1
+ */
+public class ConfigModelContext {
+
+ private final AbstractConfigProducer producer;
+ private final String producerId;
+ private final DeployState deployState;
+ private final ConfigModelRepoAdder configModelRepoAdder;
+
+ private ConfigModelContext(DeployState deployState, ConfigModelRepoAdder configModelRepoAdder,
+ AbstractConfigProducer producer, String producerId) {
+ this.deployState = deployState;
+ this.configModelRepoAdder = configModelRepoAdder;
+ this.producer = producer;
+ this.producerId = producerId;
+ }
+
+ public ApplicationPackage getApplicationPackage() { return deployState.getApplicationPackage(); }
+ public String getProducerId() { return producerId; }
+ public AbstractConfigProducer getParentProducer() { return producer; }
+ public DeployLogger getDeployLogger() { return deployState.getDeployLogger(); }
+ public DeployState getDeployState() { return deployState; }
+
+ /** Returns write access to the config model repo, or null (only) if this is improperly initialized during testing */
+ public ConfigModelRepoAdder getConfigModelRepoAdder() { return configModelRepoAdder; }
+
+ /**
+ * Create a new context with a different parent, but with the same id and application package.
+ *
+ * @param newParent The parent to use for the new context.
+ * @return A new context.
+ */
+ public ConfigModelContext modifyParent(AbstractConfigProducer newParent) {
+ return ConfigModelContext.create(deployState, configModelRepoAdder, newParent, producerId);
+ }
+
+ /**
+ * Create an application context from a parent producer and an id.
+ * @param deployState The global deploy state for this model.
+ * @param parent The parent to be used for the config model.
+ * @param id The id to be used for the config model.
+ * @return An model context that can be passed to a model.
+ */
+ public static ConfigModelContext create(DeployState deployState, ConfigModelRepoAdder configModelRepoAdder,
+ AbstractConfigProducer parent, String id) {
+ return new ConfigModelContext(deployState, configModelRepoAdder, parent, id);
+ }
+
+ /**
+ * Create an application context from a parent producer and an id.
+ * @param parent The parent to be used for the config model.
+ * @param id The id to be used for the config model.
+ * @return An model context that can be passed to a model.
+ */
+ public static ConfigModelContext createFromParentAndId(ConfigModelRepoAdder configModelRepoAdder, AbstractConfigProducer parent, String id) {
+ return create(parent.getRoot().getDeployState(), configModelRepoAdder, parent, id);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelInstanceFactory.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelInstanceFactory.java
new file mode 100644
index 00000000000..a9b3fc73eb3
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelInstanceFactory.java
@@ -0,0 +1,20 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model;
+
+/**
+ * Interface for factories of config models.
+ *
+ * @author lulf
+ * @since 5.1
+ */
+public interface ConfigModelInstanceFactory<MODEL extends ConfigModel> {
+
+ /**
+ * Create an instance of {@link com.yahoo.config.model.ConfigModel} given the input context.
+ *
+ * @param context The {@link com.yahoo.config.model.ConfigModelContext} to use.
+ * @return an instance of {@link com.yahoo.config.model.ConfigModel}
+ */
+ MODEL createModel(ConfigModelContext context);
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRegistry.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRegistry.java
new file mode 100644
index 00000000000..2735b9f7fdf
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRegistry.java
@@ -0,0 +1,51 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model;
+
+import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
+import com.yahoo.config.model.builder.xml.ConfigModelId;
+
+import java.util.Collection;
+import java.util.Collections;
+
+/**
+ * A resolver of implementations of named config models.
+ * Registries may be chained in a chain of command.
+ *
+ * @author bratseth
+ */
+public abstract class ConfigModelRegistry {
+
+ private final ConfigModelRegistry chained;
+
+ public ConfigModelRegistry() {
+ this(new EmptyTerminalRegistry());
+ }
+
+ /** Creates a config model class registry which forwards unresolved requests to the argument instance */
+ public ConfigModelRegistry(ConfigModelRegistry chained) {
+ this.chained=chained;
+ }
+
+ /**
+ * Returns the builders this id resolves to both in this and any chained registry.
+ *
+ * @return the resolved config model builders, or an empty list (never null) if none
+ */
+ public abstract Collection<ConfigModelBuilder> resolve(ConfigModelId id);
+
+ public ConfigModelRegistry chained() { return chained; }
+
+ /** An empty registry which does not support chaining */
+ private static class EmptyTerminalRegistry extends ConfigModelRegistry {
+
+ public EmptyTerminalRegistry() {
+ super(null);
+ }
+
+ @Override
+ public Collection<ConfigModelBuilder> resolve(ConfigModelId id) {
+ return Collections.emptyList();
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java
new file mode 100644
index 00000000000..fa46c33a97f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java
@@ -0,0 +1,256 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model;
+
+import com.yahoo.config.application.api.ApplicationFile;
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
+import com.yahoo.config.model.builder.xml.ConfigModelId;
+import com.yahoo.config.model.builder.xml.XmlHelper;
+import com.yahoo.config.model.graph.ModelGraphBuilder;
+import com.yahoo.config.model.graph.ModelNode;
+import com.yahoo.config.model.provision.HostsXmlProvisioner;
+import com.yahoo.log.LogLevel;
+import com.yahoo.path.Path;
+import com.yahoo.text.XML;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.builder.VespaModelBuilder;
+import com.yahoo.vespa.model.clients.Clients;
+import com.yahoo.vespa.model.content.Content;
+import com.yahoo.vespa.model.routing.Routing;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.Serializable;
+import java.io.StringReader;
+import java.util.*;
+import java.util.logging.Logger;
+
+/**
+ * A collection of config model instances owned by a system model
+ *
+ * @author gjoranv
+ */
+public class ConfigModelRepo implements ConfigModelRepoAdder, Serializable, Iterable<ConfigModel> {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger log = Logger.getLogger(ConfigModelRepo.class.getPackage().toString());
+
+ private final Map<String,ConfigModel> configModelMap = new TreeMap<>();
+ private final List<ConfigModel> configModels = new ArrayList<>();
+
+ /**
+ * Returns a config model for a given id
+ *
+ * @param id the id of the model to return
+ * @return the model, or none if a model with this id is not present in this
+ */
+ public ConfigModel get(String id) {
+ return configModelMap.get(id);
+ }
+
+ /** Adds a new config model instance in this */
+ @Override
+ public void add(ConfigModel model) {
+ configModelMap.put(model.getId(), model);
+ configModels.add(model);
+ }
+
+ /** Returns the models in this as an iterator */
+ public Iterator<ConfigModel> iterator() {
+ return configModels.iterator();
+ }
+
+ /** Returns a read-only view of the config model instances of this */
+ public Map<String,ConfigModel> asMap() { return Collections.unmodifiableMap(configModelMap); }
+
+ /** Initialize part 1.: Reads the config models used in the application package. */
+ public void readConfigModels(DeployState deployState, VespaModelBuilder builder,
+ ApplicationConfigProducerRoot root, ConfigModelRegistry configModelRegistry) throws IOException, SAXException {
+ Element userServicesElement = getServicesFromApp(deployState.getApplicationPackage());
+ readConfigModels(root, userServicesElement, deployState, configModelRegistry);
+ builder.postProc(root, this);
+ }
+
+ private Element getServicesFromApp(ApplicationPackage applicationPackage) throws IOException, SAXException {
+ try (Reader servicesFile = applicationPackage.getServices()) {
+ return getServicesFromReader(servicesFile);
+ }
+ }
+
+ /**
+ * If the top level is &lt;services&gt;, it contains a list of services elements,
+ * otherwise, the top level tag is a single service.
+ */
+ private List<Element> getServiceElements(Element servicesRoot) {
+ if (servicesRoot.getTagName().equals("services"))
+ return XML.getChildren(servicesRoot);
+
+ List<Element> singleServiceList = new ArrayList<>(1);
+ singleServiceList.add(servicesRoot);
+ return singleServiceList;
+ }
+
+ /**
+ * Creates all the config models specified in the given XML element and
+ * passes their respective XML node as parameter.
+ *
+ * @param root The Root to set as parent for all plugins
+ * @param servicesRoot XML root node of the services file
+ */
+ private void readConfigModels(ApplicationConfigProducerRoot root, Element servicesRoot, DeployState deployState, ConfigModelRegistry configModelRegistry) throws IOException, SAXException {
+ final Map<ConfigModelBuilder, List<Element>> model2Element = new LinkedHashMap<>();
+ ModelGraphBuilder graphBuilder = new ModelGraphBuilder();
+
+ final List<Element> children = getServiceElements(servicesRoot);
+
+ if (XML.getChild(servicesRoot, "admin") == null)
+ children.add(getImplicitAdmin(deployState));
+
+ children.addAll(getPermanentServices(deployState));
+
+ for (Element servicesElement : children) {
+ String tagName = servicesElement.getTagName();
+ if (tagName.equals("config")) continue; // TODO: Remove on Vespa 6
+ if (tagName.equals("cluster")) continue; // TODO: Remove on Vespa 6
+ if ((tagName.equals("clients")) && deployState.isHostedVespa())
+ throw new IllegalArgumentException("<" + tagName + "> is not allowed when running Vespa in a hosted environment");
+
+ String tagVersion = servicesElement.getAttribute("version");
+ ConfigModelId xmlId = ConfigModelId.fromNameAndVersion(tagName, tagVersion);
+
+ Collection<ConfigModelBuilder> builders = configModelRegistry.resolve(xmlId);
+
+ if (builders.isEmpty())
+ throw new RuntimeException("Could not resolve tag <" + tagName + " version=\"" + tagVersion + "\"> to a config model component");
+
+ for (ConfigModelBuilder builder : builders) {
+ if ( ! model2Element.containsKey(builder)) {
+ model2Element.put(builder, new ArrayList<>());
+ graphBuilder.addBuilder(builder);
+ }
+ model2Element.get(builder).add(servicesElement);
+ }
+ }
+
+ for (ModelNode node : graphBuilder.build().topologicalSort())
+ buildModels(node, deployState, root, model2Element.get(node.builder));
+ for (ConfigModel model : configModels)
+ model.initialize(ConfigModelRepo.this);
+ }
+
+ private Collection<Element> getPermanentServices(DeployState deployState) throws IOException, SAXException {
+ List<Element> permanentServices = new ArrayList<>();
+ Optional<ApplicationPackage> applicationPackage = deployState.getPermanentApplicationPackage();
+ if (applicationPackage.isPresent()) {
+ ApplicationFile file = applicationPackage.get().getFile(Path.fromString(ApplicationPackage.PERMANENT_SERVICES));
+ if (file.exists()) {
+ try (Reader reader = file.createReader()) {
+ Element permanentServicesRoot = getServicesFromReader(reader);
+ permanentServices.addAll(getServiceElements(permanentServicesRoot));
+ }
+ }
+ }
+ return permanentServices;
+ }
+
+ private Element getServicesFromReader(Reader reader) throws IOException, SAXException {
+ Document doc = XmlHelper.getDocumentBuilder().parse(new InputSource(reader));
+ return doc.getDocumentElement();
+ }
+
+ private void buildModels(ModelNode node, DeployState deployState, AbstractConfigProducer parent, List<Element> elements) {
+ for (Element servicesElement : elements) {
+ ConfigModel model = buildModel(node, deployState, parent, servicesElement);
+ if (model.isServing())
+ add(model);
+ }
+ }
+
+ private ConfigModel buildModel(ModelNode node, DeployState deployState, AbstractConfigProducer parent, Element servicesElement) {
+ ConfigModelBuilder builder = node.builder;
+ ConfigModelContext context = ConfigModelContext.create(deployState, this, parent, getIdString(servicesElement));
+ return builder.build(node, servicesElement, context);
+ }
+
+ private static String getIdString(Element spec) {
+ String idString = XmlHelper.getIdString(spec);
+ if (idString == null || idString.isEmpty()) {
+ idString = spec.getTagName();
+ }
+ return idString;
+ }
+
+ /**
+ * Initialize part 2.:
+ * Prepare all config models for starting. Must be called after plugins are loaded and frozen.
+ */
+ public void prepareConfigModels() {
+ for (ConfigModel model : configModels) {
+ model.prepare(ConfigModelRepo.this);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public <T extends ConfigModel> List<T> getModels(Class<T> modelClass) {
+ List<T> modelsOfModelClass = new ArrayList<>();
+
+ for (ConfigModel model : asMap().values()) {
+ if (modelClass.isInstance(model))
+ modelsOfModelClass.add((T)model);
+ }
+ return modelsOfModelClass;
+ }
+
+ public Clients getClients() {
+ for (ConfigModel m : configModels) {
+ if (m instanceof Clients) {
+ return (Clients)m;
+ }
+ }
+ return null;
+ }
+
+ public Routing getRouting() {
+ for (ConfigModel m : configModels) {
+ if (m instanceof Routing) {
+ return (Routing)m;
+ }
+ }
+ return null;
+ }
+
+ public Content getContent() {
+ for (ConfigModel m : configModels) {
+ if (m instanceof Content) {
+ return (Content)m;
+ }
+ }
+ return null;
+ }
+
+ // TODO: Doctoring on the XML is the wrong level for this. We should be able to mark a model as default instead -Jon
+ private static Element getImplicitAdmin(DeployState deployState) throws IOException, SAXException {
+ final boolean hostedVespa = deployState.isHostedVespa();
+ String defaultAdminElement = hostedVespa ? getImplicitAdminV4() : getImplicitAdminV2();
+ log.log(LogLevel.DEBUG, "No <admin> defined, using " + defaultAdminElement);
+ return XmlHelper.getDocumentBuilder().parse(new InputSource(new StringReader(defaultAdminElement))).getDocumentElement();
+ }
+
+ private static String getImplicitAdminV2() {
+ return "<admin version='2.0'>\n" +
+ " <adminserver hostalias='" + HostsXmlProvisioner.IMPLICIT_ADMIN_HOSTALIAS + "'/>\n" +
+ "</admin>\n";
+ }
+
+ private static String getImplicitAdminV4() {
+ return "<admin version='4.0'>\n" +
+ " <nodes count='1' />\n" +
+ "</admin>\n";
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepoAdder.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepoAdder.java
new file mode 100644
index 00000000000..93c591b3a8e
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepoAdder.java
@@ -0,0 +1,16 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model;
+
+/**
+ * An interface which provides addition of new config models.
+ * This exists because some models need to add additional models during the build phase so *write* access
+ * to the config model repo is needed. *Read* access, on the other hand needs to happen through config model dependency
+ * inkection to avoid circular dependencies or undeclared dependencies working by accident.
+ *
+ * @author bratseth
+ */
+public interface ConfigModelRepoAdder {
+
+ void add(ConfigModel model);
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelUtils.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelUtils.java
new file mode 100644
index 00000000000..f64eee6e863
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelUtils.java
@@ -0,0 +1,64 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model;
+
+import javax.xml.xpath.XPath;
+import javax.xml.xpath.XPathExpressionException;
+import javax.xml.xpath.XPathFactory;
+import java.io.Serializable;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static javax.xml.xpath.XPathConstants.BOOLEAN;
+
+import static com.yahoo.text.Lowercase.toLowerCase;
+
+/**
+ * Utilities for config models
+ *
+ * @author gjoranv
+ */
+// TODO: Split this into appropriate classes, or move to ConfigModel superclass
+public class ConfigModelUtils implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ public static Pattern hourNmin = Pattern.compile("(\\d\\d):(\\d\\d)");
+
+ public static Map<String, Integer> day2int;
+ static {
+ day2int = new HashMap<>();
+ day2int.put("sunday", 0);
+ day2int.put("monday", 1);
+ day2int.put("tuesday", 2);
+ day2int.put("wednesday", 3);
+ day2int.put("thursday", 4);
+ day2int.put("friday", 5);
+ day2int.put("saturday", 6);
+ }
+
+ /** Parses a 24 hour clock that must be the five characters ##:## to an int stating minutes after midnight. */
+ public static int getTimeOfDay(String time) {
+ Matcher m = ConfigModelUtils.hourNmin.matcher(time);
+ if (m.matches()) {
+ return Integer.parseInt(m.group(1)) * 60 + Integer.parseInt(m.group(2));
+ }
+ throw new IllegalArgumentException("The string '" + time + "' is not in ##:## format.");
+ }
+
+ /** Parses a day of week name in english to an int, where 0 is sunday, 6 saturday. */
+ public static int getDayOfWeek(String day) {
+ return ConfigModelUtils.day2int.get(toLowerCase(day));
+ }
+
+ /**
+ * Create a string with link to documentation for latest release.
+ *
+ * @param filePath Relative path of the file to link to, e.g. reference/services-jdisc.html
+ * @return a String with link to documentation
+ */
+ public static String createDocLink(String filePath) {
+ return filePath;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/MapConfigModelRegistry.java b/config-model/src/main/java/com/yahoo/config/model/MapConfigModelRegistry.java
new file mode 100644
index 00000000000..d47a009928d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/MapConfigModelRegistry.java
@@ -0,0 +1,63 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model;
+
+import com.google.inject.Inject;
+import com.yahoo.component.provider.ComponentRegistry;
+import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
+import com.yahoo.config.model.builder.xml.ConfigModelId;
+import com.yahoo.log.LogLevel;
+import org.osgi.framework.Bundle;
+import org.osgi.framework.FrameworkUtil;
+
+import java.util.*;
+import java.util.logging.Logger;
+
+/**
+ * @author lulf
+ * @since 5.1
+ */
+public class MapConfigModelRegistry extends ConfigModelRegistry {
+
+ private static final Logger log = Logger.getLogger(MapConfigModelRegistry.class.getPackage().getName());
+ private final List<ConfigModelBuilder> builders;
+
+ /**
+ * Constructs a registry of config models, where the components are injected.
+ *
+ * @param registry a component registry
+ */
+ @Inject
+ public MapConfigModelRegistry(ComponentRegistry<? extends ConfigModelBuilder> registry) {
+ this(registry.allComponents());
+ }
+
+ /**
+ * Constructs a registry of config models.
+ *
+ * @param builderCollection A collection of builders used to populate the registry.
+ */
+ public MapConfigModelRegistry(Collection<? extends ConfigModelBuilder> builderCollection) {
+ super();
+ builders = new ArrayList<>(builderCollection);
+ }
+
+ @Override
+ public Collection<ConfigModelBuilder> resolve(ConfigModelId id) {
+ Set<ConfigModelBuilder> matchingBuilders = new HashSet<>(chained().resolve(id));
+ for (ConfigModelBuilder builder : builders)
+ if (builder.handlesElements().contains(id))
+ matchingBuilders.add(builder);
+ return matchingBuilders;
+ }
+
+ /**
+ * Create a registry from a variable argument list of builders.
+ *
+ * @param builders A variable argument list of builders to use in this map
+ * @return a ConfigModelRegistry instance.
+ */
+ public static ConfigModelRegistry createFromList(ConfigModelBuilder<? extends ConfigModel> ... builders) {
+ return new MapConfigModelRegistry(Arrays.asList(builders));
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/NullConfigModelRegistry.java b/config-model/src/main/java/com/yahoo/config/model/NullConfigModelRegistry.java
new file mode 100644
index 00000000000..6930c8c792b
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/NullConfigModelRegistry.java
@@ -0,0 +1,21 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model;
+
+import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
+import com.yahoo.config.model.builder.xml.ConfigModelId;
+
+import java.util.Collection;
+
+/**
+ * A config model class registry that only forwards to the chained registry.
+ *
+ * @author bratseth
+ */
+public class NullConfigModelRegistry extends ConfigModelRegistry {
+
+ @Override
+ public Collection<ConfigModelBuilder> resolve(ConfigModelId id) {
+ return chained().resolve(id);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java
new file mode 100644
index 00000000000..8b6ea27ccab
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java
@@ -0,0 +1,113 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.admin;
+
+import com.google.common.collect.ImmutableList;
+import com.yahoo.config.model.ConfigModel;
+import com.yahoo.config.model.ConfigModelContext;
+import com.yahoo.config.model.ConfigModelRepo;
+import com.yahoo.config.model.ApplicationConfigProducerRoot;
+import com.yahoo.config.model.deploy.DeployProperties;
+import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
+import com.yahoo.config.model.builder.xml.ConfigModelId;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.vespa.model.admin.Admin;
+import com.yahoo.vespa.model.builder.xml.dom.DomAdminV2Builder;
+import com.yahoo.vespa.model.builder.xml.dom.DomAdminV4Builder;
+import com.yahoo.vespa.model.container.ContainerModel;
+import org.w3c.dom.Element;
+
+import java.util.*;
+
+/**
+ * Config model adaptor of the Admin class.
+ *
+ * @author lulf
+ * @since 5.1
+ */
+public class AdminModel extends ConfigModel {
+
+ private Admin admin = null;
+ private final Collection<ContainerModel> containerModels;
+
+ /**
+ * Constructs a new config model given a context.
+ *
+ * @param modelContext the model context.
+ */
+ public AdminModel(ConfigModelContext modelContext, Collection<ContainerModel> containerModels) {
+ super(modelContext);
+ this.containerModels = containerModels;
+ }
+
+ public Admin getAdmin() { return admin; }
+
+ private Collection<ContainerModel> getContainerModels() { return containerModels; }
+
+ @Override
+ public void prepare(ConfigModelRepo configModelRepo) {
+ verifyClusterControllersOnlyDefinedForContent(configModelRepo);
+ if (admin == null || admin.getClusterControllers() == null) return;
+ admin.getClusterControllers().prepare();
+ }
+
+ private void verifyClusterControllersOnlyDefinedForContent(ConfigModelRepo configModelRepo) {
+ Admin admin = getAdmin();
+ if (admin == null || admin.getClusterControllers() == null) return;
+ if (configModelRepo.getContent() == null) {
+ throw new IllegalArgumentException("Declaring <clustercontrollers> in <admin> in services.xml will not work when <content> is not defined");
+ }
+ }
+
+ public static class BuilderV2 extends ConfigModelBuilder<AdminModel> {
+
+ public static final List<ConfigModelId> configModelIds =
+ ImmutableList.of(ConfigModelId.fromNameAndVersion("admin", "2.0"),
+ ConfigModelId.fromNameAndVersion("admin", "1.0"));
+
+ public BuilderV2() {
+ super(AdminModel.class);
+ }
+
+ @Override
+ public List<ConfigModelId> handlesElements() { return configModelIds; }
+
+ @Override
+ public void doBuild(AdminModel model, Element adminElement, ConfigModelContext modelContext) {
+ AbstractConfigProducer parent = modelContext.getParentProducer();
+ DeployProperties properties = modelContext.getDeployState().getProperties();
+ DomAdminV2Builder domBuilder = new DomAdminV2Builder(modelContext.getDeployState().getFileRegistry(), properties.multitenant(), properties.configServerSpecs());
+ model.admin = domBuilder.build(parent, adminElement);
+ // TODO: Is required since other models depend on admin.
+ if (parent instanceof ApplicationConfigProducerRoot) {
+ ((ApplicationConfigProducerRoot)parent).setupAdmin(model.admin);
+ }
+ }
+ }
+
+ public static class BuilderV4 extends ConfigModelBuilder<AdminModel> {
+
+ public static final List<ConfigModelId> configModelIds =
+ ImmutableList.of(ConfigModelId.fromNameAndVersion("admin", "3.0"),
+ ConfigModelId.fromNameAndVersion("admin", "4.0"));
+
+ public BuilderV4() {
+ super(AdminModel.class);
+ }
+
+ @Override
+ public List<ConfigModelId> handlesElements() { return configModelIds; }
+
+ @Override
+ public void doBuild(AdminModel model, Element adminElement, ConfigModelContext modelContext) {
+ AbstractConfigProducer parent = modelContext.getParentProducer();
+ DeployProperties properties = modelContext.getDeployState().getProperties();
+ DomAdminV4Builder domBuilder = new DomAdminV4Builder(modelContext, properties.multitenant(), properties.configServerSpecs(), model.getContainerModels());
+ model.admin = domBuilder.build(parent, adminElement);
+ // TODO: Is required since other models depend on admin.
+ if (parent instanceof ApplicationConfigProducerRoot) {
+ ((ApplicationConfigProducerRoot)parent).setupAdmin(model.admin);
+ }
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java
new file mode 100644
index 00000000000..a0b1be20df6
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java
@@ -0,0 +1,119 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.builder.xml;
+
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.config.model.ConfigModel;
+import com.yahoo.config.model.ConfigModelContext;
+import com.yahoo.config.model.ConfigModelInstanceFactory;
+import com.yahoo.config.model.ConfigModelRepo;
+import com.yahoo.config.model.api.ConfigModelPlugin;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import org.w3c.dom.Element;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.List;
+
+/**
+ * Builds a config model using DOM parsers
+ *
+ * @author vegardh
+ * @since 5.1.10
+ */
+public abstract class ConfigModelBuilder<MODEL extends ConfigModel> extends AbstractComponent implements ConfigModelPlugin {
+
+ private Class<MODEL> configModelClass;
+
+ public ConfigModelBuilder(Class<MODEL> configModelClass) {
+ this.configModelClass = configModelClass;
+ }
+
+ /**
+ * Method that must return the XML elements this builder handles. Subclasses must implement this in order to
+ * get called when one of the elements have been encountered when parsing.
+ *
+ * @return A list of elements that this builder handles.
+ */
+ public abstract List<ConfigModelId> handlesElements();
+
+ /**
+ * Convenience hook called from {@link #build}. Implement this method to build a config model.
+ *
+ * @param spec The XML element that this builder should handle.
+ * @param modelContext A model context that contains the application package and other data needed by the
+ * config model constructor.
+ */
+ public abstract void doBuild(MODEL model, Element spec, ConfigModelContext modelContext);
+
+ /**
+ * Builds an instance of this component model.
+ * This calls instantiate(...), instance.setUp(...), doBuild(instance, ...).
+ *
+ * @param deployState a global deployment state used for this model.
+ * @param parent the root config producer this should be added to
+ * @param spec the XML element this is constructed from
+ */
+ public final MODEL build(DeployState deployState, ConfigModelRepo configModelRepo, AbstractConfigProducer parent, Element spec) {
+ ConfigModelContext context = ConfigModelContext.create(deployState, configModelRepo, parent, getIdString(spec));
+ return build(new DefaultModelInstanceFactory(), spec, context);
+ }
+
+ /**
+ * Builds an instance of this component model.
+ * This calls instantiate(...), instance.setUp(...), doBuild(instance, ...).
+ *
+ * @param factory A factory capable of creating models.
+ * @param spec the XML element this is constructed from
+ * @param context A context object containing various data used by builders.
+ */
+ public MODEL build(ConfigModelInstanceFactory<MODEL> factory, Element spec, ConfigModelContext context) {
+ MODEL model = factory.createModel(context);
+ doBuild(model, spec, context);
+ return model;
+ }
+
+ public Class<MODEL> getModelClass() {
+ return configModelClass;
+ }
+
+ private static String getIdString(Element spec) {
+ String idString = XmlHelper.getIdString(spec);
+ if (idString == null || idString.isEmpty()) {
+ idString = spec.getTagName();
+ }
+ return idString;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof ConfigModelBuilder)) {
+ return false;
+ }
+ ConfigModelBuilder otherBuilder = (ConfigModelBuilder) other;
+ List<ConfigModelId> thisIds = this.handlesElements();
+ List<ConfigModelId> otherIds = otherBuilder.handlesElements();
+ if (thisIds.size() != otherIds.size()) {
+ return false;
+ }
+ for (int i = 0; i < thisIds.size(); i++) {
+ if (!thisIds.get(i).equals(otherIds.get(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+
+ private class DefaultModelInstanceFactory implements ConfigModelInstanceFactory<MODEL> {
+ @Override
+ public MODEL createModel(ConfigModelContext context) {
+ try {
+ Constructor<MODEL> constructor = configModelClass.getConstructor(ConfigModelContext.class);
+ return constructor.newInstance(context);
+ } catch (NoSuchMethodException | InvocationTargetException | InstantiationException | IllegalAccessException e) {
+ throw new RuntimeException("Error constructing model '" + configModelClass.getName() + "'", e);
+ }
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java
new file mode 100644
index 00000000000..fca114757ec
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java
@@ -0,0 +1,94 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.builder.xml;
+
+import com.yahoo.component.Version;
+
+/**
+ * A {@link ConfigModelId} describes an element handled by a {@link ConfigModelBuilder}.
+ *
+ * @author lulf
+ * @since 5.1
+ */
+public class ConfigModelId implements Comparable<ConfigModelId> {
+
+ private final String name;
+ private final Version version;
+ private final String stringValue;
+
+ private ConfigModelId(String name, Version version) {
+ this.name = name;
+ this.version = version;
+ this.stringValue = toStringValue();
+ }
+
+ /**
+ * Create id with a name and version
+ * @param tagName Name of the id
+ * @param tagVersion Version of the id
+ * @return A ConfigModelId instance
+ */
+ public static ConfigModelId fromNameAndVersion(String tagName, String tagVersion) {
+ return new ConfigModelId(tagName, Version.fromString(tagVersion));
+ }
+
+ /**
+ * Create id with given name, using default version 1.
+ *
+ * @param tagName Name of the id
+ * @return A ConfigModelId instance
+ */
+ public static ConfigModelId fromName(String tagName) {
+ return new ConfigModelId(tagName, new Version(1));
+ }
+
+ @Override
+ public boolean equals(Object object) {
+ if (!(object instanceof ConfigModelId)) return false;
+ ConfigModelId other = (ConfigModelId)object;
+ return this.name.equals(other.name) && this.version.equals(other.version);
+ }
+
+ @Override
+ public int compareTo(ConfigModelId other) {
+ if (other == this) return 0;
+ int cmp = this.name.compareTo(other.name);
+ if (cmp == 0) {
+ cmp = this.version.compareTo(other.version);
+ }
+ return cmp;
+ }
+
+ @Override
+ public String toString() {
+ return stringValue;
+ }
+
+ @Override
+ public int hashCode() {
+ return stringValue.hashCode();
+ }
+
+ /**
+ * Return the XML element name.
+ * @return the name of the config model
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Return the XML element version.
+ * @return the version of the config model
+ */
+ Version getVersion() {
+ return version;
+ }
+
+ private String toStringValue() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(name);
+ sb.append(".");
+ sb.append(version);
+ return sb.toString();
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java
new file mode 100644
index 00000000000..924d888b0d0
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java
@@ -0,0 +1,135 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.builder.xml;
+
+import com.yahoo.component.ComponentId;
+import com.yahoo.component.ComponentSpecification;
+import com.yahoo.log.LogLevel;
+import com.yahoo.text.XML;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import java.io.IOException;
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
+import java.util.logging.Logger;
+
+
+/**
+ * Static methods for helping dom building
+ *
+ * @author <a href="mailto:bratseth@yahoo-inc.com">Jon Bratseth</a>
+ */
+public final class XmlHelper {
+ private static final Logger log = Logger.getLogger(XmlHelper.class.getPackage().toString());
+
+
+ private static final String idReference = "idref";
+ // Access to this needs to be synchronized (as it is in getDocumentBuilder() below)
+ public static final DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+
+ static {
+ XmlHelper.factory.setNamespaceAware(true);
+ // if use jdom and jaxen this will fail badly:
+ XmlHelper.factory.setXIncludeAware(true);
+ }
+
+ private XmlHelper() {}
+
+ public static String nullIfEmpty(String attribute) {
+ if (attribute.isEmpty())
+ return null;
+ else
+ return attribute;
+ }
+
+ /**
+ * For searchers inside search chains, the id may be both a reference and an id at once, or just a reference.
+ * In other cases, it is clear which one it is from context, so I think the difference is not worth bothering users
+ * with, unless they are XML purists in which case they will have the option of writing this correctly.
+ * - Jon
+ */
+ public static String getIdString(Element element) {
+ String idString = element.getAttribute("id");
+ if (idString == null || idString.trim().equals(""))
+ idString = element.getAttribute(idReference);
+ if (idString == null || idString.trim().equals(""))
+ idString = element.getAttribute("ident");
+ return idString;
+ }
+
+ public static ComponentId getId(Element element) {
+ return new ComponentId(getIdString(element));
+ }
+
+ public static ComponentSpecification getIdRef(Element element) {
+ return new ComponentSpecification(getIdString(element));
+ }
+
+ public static Document getDocument(Reader reader) {
+ Document doc;
+ try {
+ doc = getDocumentBuilder().parse(new InputSource(reader));
+ } catch (SAXException | IOException e) {
+ throw new IllegalArgumentException(e);
+ }
+ return doc;
+ }
+
+ public static List<String> splitAndDiscardEmpty(String field, String regex) {
+ List<String> ret = new ArrayList<>();
+ for (String t : field.split(regex)) {
+ if (!t.isEmpty()) {
+ ret.add(t);
+ }
+ }
+ return ret;
+ }
+
+ public static List<String> spaceSeparatedSymbols(String field) {
+ return splitAndDiscardEmpty(field, " ");
+ }
+
+ public static Collection<String> spaceSeparatedSymbolsFromAttribute(Element spec, String name) {
+ return spaceSeparatedSymbols(spec.getAttribute(name));
+ }
+
+ public static Collection<String> valuesFromElements(Element parent, String elementName) {
+ List<String> symbols = new ArrayList<>();
+ for (Element symbol : XML.getChildren(parent, elementName)) {
+ symbols.add(XML.getValue(symbol).trim());
+ }
+ return symbols;
+ }
+
+ public static boolean isReference(Element element) {
+ return element.hasAttribute(idReference);
+ }
+
+ /**
+ * Creates a new XML document builder.
+ *
+ * @return A new DocumentBuilder instance, or null if we fail to get one.
+ */
+ public static synchronized DocumentBuilder getDocumentBuilder() {
+ try {
+ DocumentBuilder docBuilder = factory.newDocumentBuilder();
+ log.log(LogLevel.DEBUG, "XML parser now operational!");
+ return docBuilder;
+ } catch (ParserConfigurationException e) {
+ log.log(LogLevel.WARNING, "No XML parser available - " + e);
+ return null;
+ }
+ }
+
+ public static Optional<String> getOptionalAttribute(Element element, String name) {
+ return Optional.ofNullable(element.getAttribute(name)).filter(s -> !s.isEmpty());
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/package-info.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/package-info.java
new file mode 100644
index 00000000000..6ce400dc922
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.config.model.builder.xml;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java b/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java
new file mode 100644
index 00000000000..22ea054a27d
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/ConfigDefinitionStore.java
@@ -0,0 +1,15 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.deploy;
+
+import com.yahoo.vespa.config.ConfigDefinition;
+import com.yahoo.vespa.config.ConfigDefinitionKey;
+
+/**
+ * @author lulf
+ * @since 5.1
+ */
+public interface ConfigDefinitionStore {
+
+ ConfigDefinition getConfigDefinition(ConfigDefinitionKey defKey);
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java
new file mode 100644
index 00000000000..3f6ef494a27
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployProperties.java
@@ -0,0 +1,115 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.deploy;
+
+import com.yahoo.config.model.api.ConfigServerSpec;
+import com.yahoo.config.provision.*;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Collection of properties for a deployment.
+ *
+ * @author lulf
+ * @since 5.17
+ */
+public class DeployProperties {
+ private final boolean multitenant;
+ private final ApplicationId applicationId;
+ private final List<ConfigServerSpec> serverSpecs = new ArrayList<>();
+ private final boolean hostedVespa;
+ private final Version vespaVersion;
+ private final Zone zone;
+
+ private DeployProperties(boolean multitenant,
+ ApplicationId applicationId,
+ List<ConfigServerSpec> configServerSpecs,
+ boolean hostedVespa, Version vespaVersion, Zone zone) {
+ this.vespaVersion = vespaVersion;
+ this.zone = zone;
+ this.multitenant = multitenant || hostedVespa || Boolean.getBoolean("multitenant");
+ this.applicationId = applicationId;
+ this.serverSpecs.addAll(configServerSpecs);
+ this.hostedVespa = hostedVespa;
+ }
+
+
+ public boolean multitenant() {
+ return multitenant;
+ }
+
+ public ApplicationId applicationId() {
+ return applicationId;
+ }
+
+ public List<ConfigServerSpec> configServerSpecs() {
+ return serverSpecs;
+ }
+
+ public Quota quota() {
+ return new Quota(Integer.MAX_VALUE);
+ }
+
+ public boolean hostedVespa() {
+ return hostedVespa;
+ }
+
+ public Version vespaVersion() {
+ return vespaVersion;
+ }
+
+ public Zone zone() {
+ return zone;
+ }
+
+ public static class Builder {
+
+ private ApplicationId applicationId = ApplicationId.defaultId();
+ private boolean multitenant = false;
+ private List<ConfigServerSpec> configServerSpecs = new ArrayList<>();
+ private Quota quota = new Quota(Integer.MAX_VALUE);
+ private boolean hostedVespa = false;
+ private Version vespaVersion = Version.fromIntValues(1, 0, 0);
+ private Zone zone = Zone.defaultZone();
+
+ public Builder applicationId(ApplicationId applicationId) {
+ this.applicationId = applicationId;
+ return this;
+ }
+
+ public Builder multitenant(boolean multitenant) {
+ this.multitenant = multitenant;
+ return this;
+ }
+
+ public Builder configServerSpecs(List<ConfigServerSpec> configServerSpecs) {
+ this.configServerSpecs = configServerSpecs;
+ return this;
+ }
+
+ public Builder quota(Quota quota) {
+ this.quota = quota;
+ return this;
+ }
+
+ public Builder vespaVersion(Version version) {
+ this.vespaVersion = version;
+ return this;
+ }
+
+ public Builder hostedVespa(boolean hostedVespa) {
+ this.hostedVespa = hostedVespa;
+ return this;
+ }
+
+ public Builder zone(Zone zone) {
+ this.zone = zone;
+ return this;
+ }
+
+ public DeployProperties build() {
+ return new DeployProperties(multitenant, applicationId, configServerSpecs, hostedVespa, vespaVersion, zone);
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java
new file mode 100644
index 00000000000..5896dc59df2
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java
@@ -0,0 +1,381 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.deploy;
+
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.application.api.FileRegistry;
+import com.yahoo.config.application.api.UnparsedConfigDefinition;
+import com.yahoo.config.codegen.CNode;
+import com.yahoo.config.model.api.ConfigDefinitionRepo;
+import com.yahoo.config.model.api.HostProvisioner;
+import com.yahoo.config.model.api.Model;
+import com.yahoo.config.model.application.provider.BaseDeployLogger;
+import com.yahoo.config.model.application.provider.MockFileRegistry;
+import com.yahoo.config.model.provision.HostsXmlProvisioner;
+import com.yahoo.config.model.provision.SingleNodeProvisioner;
+import com.yahoo.config.model.test.MockApplicationPackage;
+import com.yahoo.config.provision.Rotation;
+import com.yahoo.config.provision.Zone;
+import com.yahoo.io.reader.NamedReader;
+import com.yahoo.log.LogLevel;
+import com.yahoo.searchdefinition.RankProfileRegistry;
+import com.yahoo.searchdefinition.SearchBuilder;
+import com.yahoo.searchdefinition.parser.ParseException;
+import com.yahoo.vespa.config.ConfigDefinition;
+import com.yahoo.vespa.config.ConfigDefinitionBuilder;
+import com.yahoo.vespa.config.ConfigDefinitionKey;
+import com.yahoo.vespa.documentmodel.DocumentModel;
+import com.yahoo.vespa.model.application.validation.ValidationOverrides;
+import com.yahoo.vespa.model.application.validation.xml.ValidationOverridesXMLReader;
+import com.yahoo.vespa.model.container.search.QueryProfiles;
+import com.yahoo.vespa.model.container.search.QueryProfilesBuilder;
+import com.yahoo.vespa.model.container.search.SemanticRuleBuilder;
+import com.yahoo.vespa.model.container.search.SemanticRules;
+import com.yahoo.vespa.model.search.SearchDefinition;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.Reader;
+import java.time.Instant;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.logging.Logger;
+
+/**
+ * Contains various state during deploy that should be reachable to all builders of a {@link com.yahoo.config.model.ConfigModel}
+ *
+ * @author lulf
+ * @since 5.8
+ */
+public class DeployState implements ConfigDefinitionStore {
+
+ private static final Logger log = Logger.getLogger(DeployState.class.getName());
+
+ private final DeployLogger logger;
+ private final FileRegistry fileRegistry;
+ private final DocumentModel documentModel;
+ private final List<SearchDefinition> searchDefinitions;
+ private final ApplicationPackage applicationPackage;
+ private final Optional<ConfigDefinitionRepo> configDefinitionRepo;
+ private final Optional<ApplicationPackage> permanentApplicationPackage;
+ private final Optional<Model> previousModel;
+ private final DeployProperties properties;
+ private final Set<Rotation> rotations;
+ private final Zone zone;
+ private final QueryProfiles queryProfiles;
+ private final SemanticRules semanticRules;
+ private final ValidationOverrides validationOverrides;
+
+ private final HostProvisioner provisioner;
+
+ public static DeployState createTestState() {
+ return new Builder().build();
+ }
+
+ public static DeployState createTestState(ApplicationPackage applicationPackage) {
+ return new Builder().applicationPackage(applicationPackage).build();
+ }
+
+ private DeployState(ApplicationPackage applicationPackage, SearchDocumentModel searchDocumentModel, RankProfileRegistry rankProfileRegistry,
+ FileRegistry fileRegistry, DeployLogger deployLogger, Optional<HostProvisioner> hostProvisioner, DeployProperties properties,
+ Optional<ApplicationPackage> permanentApplicationPackage, Optional<ConfigDefinitionRepo> configDefinitionRepo,
+ java.util.Optional<Model> previousModel, Set<Rotation> rotations, Zone zone, QueryProfiles queryProfiles, SemanticRules semanticRules, Instant now) {
+ this.logger = deployLogger;
+ this.fileRegistry = fileRegistry;
+ this.rankProfileRegistry = rankProfileRegistry;
+ this.applicationPackage = applicationPackage;
+ this.properties = properties;
+ this.previousModel = previousModel;
+ if (hostProvisioner.isPresent()) {
+ this.provisioner = hostProvisioner.get();
+ } else {
+ this.provisioner = getDefaultModelHostProvisioner(applicationPackage);
+ }
+ this.searchDefinitions = searchDocumentModel.getSearchDefinitions();
+ this.documentModel = searchDocumentModel.getDocumentModel();
+ this.permanentApplicationPackage = permanentApplicationPackage;
+ this.configDefinitionRepo = configDefinitionRepo;
+ this.rotations = rotations;
+ this.zone = zone;
+ this.queryProfiles = queryProfiles; // TODO: Remove this by seeing how pagetemplates are propagated
+ this.semanticRules = semanticRules; // TODO: Remove this by seeing how pagetemplates are propagated
+ this.validationOverrides = new ValidationOverridesXMLReader().read(applicationPackage.getValidationOverrides(), now);
+
+ }
+
+ public static HostProvisioner getDefaultModelHostProvisioner(ApplicationPackage applicationPackage) {
+ if (applicationPackage.getHosts() == null) {
+ return new SingleNodeProvisioner();
+ } else {
+ return new HostsXmlProvisioner(applicationPackage.getHosts());
+ }
+ }
+
+ /** Get the global rank profile registry for this application. */
+ public final RankProfileRegistry rankProfileRegistry() { return rankProfileRegistry; }
+
+ /** Returns the validation overrides of this. This is never null */
+ public ValidationOverrides validationOverrides() { return validationOverrides; }
+
+ /**
+ * Returns the config def with the given name and namespace.
+ *
+ * @param defKey The {@link ConfigDefinitionKey} that will uniquely identify a config definition.
+ * @return The definition with a matching name and namespace
+ * @throws java.lang.IllegalArgumentException if def is not found.
+ */
+ public final ConfigDefinition getConfigDefinition(ConfigDefinitionKey defKey) {
+ if (existingConfigDefs == null) {
+ existingConfigDefs = new LinkedHashMap<>();
+ if (configDefinitionRepo.isPresent()) {
+ existingConfigDefs.putAll(createLazyMapping(configDefinitionRepo.get()));
+ }
+ existingConfigDefs.putAll(applicationPackage.getAllExistingConfigDefs());
+ }
+ log.log(LogLevel.DEBUG, "Getting config definition " + defKey);
+ // Fall back to default namespace if not found.
+ if (defKey.getNamespace() == null || defKey.getNamespace().isEmpty()) {
+ defKey = new ConfigDefinitionKey(defKey.getName(), CNode.DEFAULT_NAMESPACE);
+ }
+ ConfigDefinitionKey lookupKey = defKey;
+ // Fall back to just using name
+ if (!existingConfigDefs.containsKey(lookupKey)) {
+
+ int count = 0;
+ for (ConfigDefinitionKey entry : existingConfigDefs.keySet()) {
+ if (entry.getName().equals(defKey.getName())) {
+ count++;
+ }
+ }
+ if (count > 1) {
+ throw new IllegalArgumentException("Using config definition '" + defKey.getName() + "' is ambiguous, there are more than one config definitions with this name, please specify namespace");
+ }
+
+ lookupKey = null;
+ log.log(LogLevel.DEBUG, "Could not find config definition '" + defKey + "', trying with same name in all namespaces");
+ for (ConfigDefinitionKey entry : existingConfigDefs.keySet()) {
+ if (entry.getName().equals(defKey.getName()) && defKey.getNamespace().equals(CNode.DEFAULT_NAMESPACE)) {
+ log.log(LogLevel.INFO, "Could not find config definition '" + defKey + "'" +
+ ", using config definition '" + entry + "' with same name instead (please use new namespace when specifying this config)");
+ lookupKey = entry;
+ break;
+ }
+ }
+ }
+
+ if (lookupKey == null) {
+ throw new IllegalArgumentException("Could not find a config definition with name '" + defKey + "'.");
+ }
+ if (defArchive.get(defKey) != null) {
+ log.log(LogLevel.DEBUG, "Found in archive: " + defKey);
+ return defArchive.get(defKey);
+ }
+
+ log.log(LogLevel.DEBUG, "Retrieving config definition: " + defKey);
+ ConfigDefinition def = existingConfigDefs.get(lookupKey).parse();
+
+ log.log(LogLevel.DEBUG, "Adding " + def + " to archive");
+ defArchive.put(defKey, def);
+ return def;
+ }
+
+ private static Map<ConfigDefinitionKey, UnparsedConfigDefinition> createLazyMapping(final ConfigDefinitionRepo configDefinitionRepo) {
+ Map<ConfigDefinitionKey, UnparsedConfigDefinition> keyToRepo = new LinkedHashMap<>();
+ for (final Map.Entry<ConfigDefinitionKey, com.yahoo.vespa.config.buildergen.ConfigDefinition> defEntry : configDefinitionRepo.getConfigDefinitions().entrySet()) {
+ keyToRepo.put(defEntry.getKey(), new UnparsedConfigDefinition() {
+ @Override
+ public ConfigDefinition parse() {
+ return ConfigDefinitionBuilder.createConfigDefinition(configDefinitionRepo.getConfigDefinitions().get(defEntry.getKey()).getCNode());
+ }
+
+ @Override
+ public String getUnparsedContent() {
+ throw new UnsupportedOperationException("Cannot get unparsed content from " + defEntry.getKey());
+ }
+ });
+ }
+ return keyToRepo;
+ }
+
+ // Global registry of rank profiles.
+ // TODO: I think this can be removed when we remove "<search version=2.0>" and only support content.
+ private final RankProfileRegistry rankProfileRegistry;
+
+ // Mapping from key to something that can create a config definition.
+ private Map<ConfigDefinitionKey, UnparsedConfigDefinition> existingConfigDefs = null;
+
+ // Cache of config defs for all [def,version] combinations looked up so far.
+ private final Map<ConfigDefinitionKey, ConfigDefinition> defArchive = new LinkedHashMap<>();
+
+ public ApplicationPackage getApplicationPackage() {
+ return applicationPackage;
+ }
+ public List<SearchDefinition> getSearchDefinitions() {
+ return searchDefinitions;
+ }
+
+ public DocumentModel getDocumentModel() {
+ return documentModel;
+ }
+
+ public DeployLogger getDeployLogger() {
+ return logger;
+ }
+
+ public FileRegistry getFileRegistry() {
+ return fileRegistry;
+ }
+
+ public HostProvisioner getProvisioner() { return provisioner; }
+
+ public Optional<ApplicationPackage> getPermanentApplicationPackage() {
+ return permanentApplicationPackage;
+ }
+
+ public DeployProperties getProperties() { return properties; }
+
+ public Optional<Model> getPreviousModel() { return previousModel; }
+
+ public boolean isHostedVespa() {
+ return properties.hostedVespa();
+ }
+
+ public Set<Rotation> getRotations() {
+ return this.rotations; // todo: consider returning a copy or immutable view
+ }
+
+ /** Returns the zone in which this is currently running */
+ public Zone zone() { return zone; }
+
+ public QueryProfiles getQueryProfiles() { return queryProfiles; }
+
+ public SemanticRules getSemanticRules() { return semanticRules; }
+
+ public static class Builder {
+
+ private ApplicationPackage applicationPackage = MockApplicationPackage.createEmpty();
+ private FileRegistry fileRegistry = new MockFileRegistry();
+ private DeployLogger logger = new BaseDeployLogger();
+ private Optional<HostProvisioner> hostProvisioner = Optional.empty();
+ private Optional<ApplicationPackage> permanentApplicationPackage = Optional.empty();
+ private DeployProperties properties = new DeployProperties.Builder().build();
+ private Optional<ConfigDefinitionRepo> configDefinitionRepo = Optional.empty();
+ private Optional<Model> previousModel = Optional.empty();
+ private Set<Rotation> rotations = new HashSet<>();
+ private Zone zone = Zone.defaultZone();
+ private Instant now = Instant.now();
+
+ public Builder applicationPackage(ApplicationPackage applicationPackage) {
+ this.applicationPackage = applicationPackage;
+ return this;
+ }
+
+ public Builder fileRegistry(FileRegistry fileRegistry) {
+ this.fileRegistry = fileRegistry;
+ return this;
+ }
+
+ public Builder deployLogger(DeployLogger logger) {
+ this.logger = logger;
+ return this;
+ }
+
+ public Builder modelHostProvisioner(HostProvisioner modelProvisioner) {
+ this.hostProvisioner = Optional.of(modelProvisioner);
+ return this;
+ }
+
+ public Builder permanentApplicationPackage(Optional<ApplicationPackage> permanentApplicationPackage) {
+ this.permanentApplicationPackage = permanentApplicationPackage;
+ return this;
+ }
+
+ public Builder properties(DeployProperties properties) {
+ this.properties = properties;
+ return this;
+ }
+
+ public Builder configDefinitionRepo(ConfigDefinitionRepo configDefinitionRepo) {
+ this.configDefinitionRepo = Optional.of(configDefinitionRepo);
+ return this;
+ }
+
+ public Builder previousModel(Model previousModel) {
+ this.previousModel = Optional.of(previousModel);
+ return this;
+ }
+
+ public Builder rotations(Set<Rotation> rotations) {
+ this.rotations = rotations;
+ return this;
+ }
+
+ public Builder zone(Zone zone) {
+ this.zone = zone;
+ return this;
+ }
+
+ public Builder now(Instant now) {
+ this.now = now;
+ return this;
+ }
+
+ public DeployState build() {
+ RankProfileRegistry rankProfileRegistry = new RankProfileRegistry();
+ QueryProfiles queryProfiles = new QueryProfilesBuilder().build(applicationPackage);
+ SemanticRules semanticRules = new SemanticRuleBuilder().build(applicationPackage);
+ SearchDocumentModel searchDocumentModel = createSearchDocumentModel(rankProfileRegistry, logger, queryProfiles);
+ return new DeployState(applicationPackage, searchDocumentModel, rankProfileRegistry, fileRegistry, logger, hostProvisioner,
+ properties, permanentApplicationPackage, configDefinitionRepo, previousModel, rotations, zone, queryProfiles, semanticRules, now);
+ }
+
+ private SearchDocumentModel createSearchDocumentModel(RankProfileRegistry rankProfileRegistry, DeployLogger logger, QueryProfiles queryProfiles) {
+ Collection<NamedReader> readers = applicationPackage.getSearchDefinitions();
+ Map<String, String> names = new LinkedHashMap<>();
+ SearchBuilder builder = new SearchBuilder(applicationPackage, rankProfileRegistry);
+ for (NamedReader reader : readers) {
+ try {
+ String readerName = reader.getName();
+ String searchName = builder.importReader(reader, readerName, logger);
+ String sdName = stripSuffix(readerName, ApplicationPackage.SD_NAME_SUFFIX);
+ names.put(searchName, sdName);
+ if (!sdName.equals(searchName)) {
+ throw new IllegalArgumentException("Search definition file name ('" + sdName + "') and name of " +
+ "search element ('" + searchName + "') are not equal for file '" + readerName + "'");
+ }
+ } catch (ParseException e) {
+ throw new IllegalArgumentException("Could not parse search definition file '" + getSearchDefinitionRelativePath(reader.getName()) + "': " + e.getMessage(), e);
+ } catch (IOException e) {
+ throw new IllegalArgumentException("Could not read search definition file '" + getSearchDefinitionRelativePath(reader.getName()) + "': " + e.getMessage(), e);
+ } finally {
+ closeIgnoreException(reader.getReader());
+ }
+ }
+ builder.build(logger, queryProfiles);
+ return SearchDocumentModel.fromBuilderAndNames(builder, names);
+ }
+
+ private String getSearchDefinitionRelativePath(String name) {
+ return ApplicationPackage.SEARCH_DEFINITIONS_DIR + File.separator + name;
+ }
+
+ private static String stripSuffix(String nodeName, String postfix) {
+ assert (nodeName.endsWith(postfix));
+ return nodeName.substring(0, nodeName.length() - postfix.length());
+ }
+
+ @SuppressWarnings("EmptyCatchBlock")
+ private static void closeIgnoreException(Reader reader) {
+ try {
+ reader.close();
+ } catch(Exception e) {}
+ }
+ }
+
+}
+
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java b/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java
new file mode 100644
index 00000000000..1fbd72599fe
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/SearchDocumentModel.java
@@ -0,0 +1,53 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.deploy;
+
+import com.yahoo.searchdefinition.SearchBuilder;
+import com.yahoo.vespa.documentmodel.DocumentModel;
+import com.yahoo.vespa.model.search.SearchDefinition;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Internal helper class to retrieve document model and search definitions.
+ *
+ * @author lulf
+ * @since 5.1
+ */
+public class SearchDocumentModel {
+
+ private final DocumentModel documentModel;
+ private final List<SearchDefinition> searchDefinitions;
+
+ public SearchDocumentModel(DocumentModel documentModel, List<SearchDefinition> searchDefinitions) {
+ this.documentModel = documentModel;
+ this.searchDefinitions = searchDefinitions;
+
+ }
+
+ public DocumentModel getDocumentModel() {
+ return documentModel;
+ }
+
+ public List<SearchDefinition> getSearchDefinitions() {
+ return searchDefinitions;
+ }
+
+ public static SearchDocumentModel fromBuilderAndNames(SearchBuilder builder, Map<String, String> names) {
+ List<SearchDefinition> ret = new ArrayList<>();
+ for (com.yahoo.searchdefinition.Search search : builder.getSearchList()) {
+ ret.add(new SearchDefinition(names.get(search.getName()), search));
+ }
+ return new SearchDocumentModel(builder.getModel(), ret);
+ }
+
+ public static SearchDocumentModel fromBuilder(SearchBuilder builder) {
+ List<SearchDefinition> ret = new ArrayList<>();
+ for (com.yahoo.searchdefinition.Search search : builder.getSearchList()) {
+ ret.add(new SearchDefinition(search.getName(), search));
+ }
+ return new SearchDocumentModel(builder.getModel(), ret);
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/package-info.java b/config-model/src/main/java/com/yahoo/config/model/deploy/package-info.java
new file mode 100644
index 00000000000..3feb934615f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.config.model.deploy;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/config/model/graph/ModelGraph.java b/config-model/src/main/java/com/yahoo/config/model/graph/ModelGraph.java
new file mode 100644
index 00000000000..a435733e60f
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/graph/ModelGraph.java
@@ -0,0 +1,80 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.graph;
+
+import com.yahoo.component.ComponentId;
+
+import java.util.*;
+
+/**
+ * A model graph contains the dependency graph of config models.
+ *
+ * @author lulf
+ * @since 5.1
+ */
+public class ModelGraph {
+
+ private final List<ModelNode> modelNodes;
+ private final List<ModelNode> roots;
+ public ModelGraph(List<ModelNode> modelNodes, List<ModelNode> roots) {
+ this.modelNodes = modelNodes;
+ this.roots = roots;
+ }
+
+ /**
+ * Performs a topological sort ot the models stored in this graph. The algorithm is based on Kahn topological sort.
+ *
+ * @return a sorted list of {@link com.yahoo.config.model.graph.ModelNode} in dependency order.
+ */
+ public List<ModelNode> topologicalSort() {
+ DependencyMap dependencyMap = new DependencyMap(modelNodes);
+ Queue<ModelNode> unprocessed = new LinkedList<>();
+ unprocessed.addAll(roots);
+ List<ModelNode> sortedList = new ArrayList<>();
+ while (!unprocessed.isEmpty()) {
+ ModelNode sortedNode = unprocessed.remove();
+ sortedList.add(sortedNode);
+ for (ModelNode node : modelNodes) {
+ if (dependencyMap.dependsOn(node, sortedNode)) {
+ dependencyMap.removeDependency(node, sortedNode);
+ if (!dependencyMap.hasDependencies(node)) {
+ unprocessed.add(node);
+ }
+ }
+ }
+ }
+ for (ModelNode node : modelNodes) {
+ if (dependencyMap.hasDependencies(node)) {
+ throw new IllegalArgumentException("Unable to sort graph because it contains cycles");
+ }
+ }
+ return sortedList;
+ }
+
+ List<ModelNode> getNodes() {
+ return modelNodes;
+ }
+
+ private static class DependencyMap {
+ private final Map<ComponentId, Set<ComponentId>> map = new LinkedHashMap<>();
+ DependencyMap(List<ModelNode> modelNodes) {
+ for (ModelNode node : modelNodes) {
+ Set<ComponentId> ids = new LinkedHashSet<>();
+ ids.addAll(node.listDependencyIds());
+ map.put(node.id, ids);
+ }
+ }
+
+ public boolean dependsOn(ModelNode node, ModelNode sortedNode) {
+ return map.get(node.id).contains(sortedNode.id);
+ }
+
+ public void removeDependency(ModelNode node, ModelNode sortedNode) {
+ map.get(node.id).remove(sortedNode.id);
+ }
+
+ public boolean hasDependencies(ModelNode node) {
+ return !map.get(node.id).isEmpty();
+ }
+ }
+}
+
diff --git a/config-model/src/main/java/com/yahoo/config/model/graph/ModelGraphBuilder.java b/config-model/src/main/java/com/yahoo/config/model/graph/ModelGraphBuilder.java
new file mode 100644
index 00000000000..97d914c6012
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/graph/ModelGraphBuilder.java
@@ -0,0 +1,52 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.graph;
+
+import com.yahoo.config.model.ConfigModel;
+import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Used to add builders and elements in addBuilder, and then build a dependency graph based on the
+ * constructor arguments.
+ *
+ * @author lulf
+ * @since 5.1
+ */
+public class ModelGraphBuilder {
+
+ private final List<ConfigModelBuilder<? extends ConfigModel>> builders = new ArrayList<>();
+
+ /**
+ * Add a {@link com.yahoo.config.model.builder.xml.ConfigModelBuilder} to this graph.
+ *
+ * @param builder The {@link com.yahoo.config.model.builder.xml.ConfigModelBuilder} to add.
+ * @return this for convenience
+ */
+ public ModelGraphBuilder addBuilder(ConfigModelBuilder<? extends ConfigModel> builder) {
+ builders.add(builder);
+ return this;
+ }
+
+ /**
+ * Build a {@link com.yahoo.config.model.graph.ModelGraph} based on the {@link com.yahoo.config.model.builder.xml.ConfigModelBuilder}s
+ * added to this.
+ *
+ * @return A {@link com.yahoo.config.model.graph.ModelGraph} representing the dependency graph.
+ */
+ public ModelGraph build() {
+ List<ModelNode> modelNodes = new ArrayList<>();
+ for (ConfigModelBuilder<? extends ConfigModel> builder : builders) {
+ modelNodes.add(new ModelNode(builder));
+ }
+ List<ModelNode> roots = new ArrayList<>();
+ for (ModelNode modelNode : modelNodes) {
+ int numDependencies = modelNode.addDependenciesFrom(modelNodes);
+ if (numDependencies == 0) {
+ roots.add(modelNode);
+ }
+ }
+ return new ModelGraph(modelNodes, roots);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/graph/ModelNode.java b/config-model/src/main/java/com/yahoo/config/model/graph/ModelNode.java
new file mode 100644
index 00000000000..0cd045b09c8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/graph/ModelNode.java
@@ -0,0 +1,135 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.graph;
+
+import com.google.inject.Inject;
+import com.yahoo.component.ComponentId;
+import com.yahoo.config.model.ConfigModel;
+import com.yahoo.config.model.ConfigModelContext;
+import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
+import com.yahoo.config.model.ConfigModelInstanceFactory;
+
+import java.lang.reflect.*;
+import java.util.*;
+
+/**
+ * Represents a node in the dependency graph, and contains information about a builders dependencies.
+ * Constructor signatures of model classes must have ConfigModelContext as the first argument and
+ * ConfigModel subclasses or Collection of ConfigModels as subsequent arguments.
+ * Only Collection, not Collection subtypes can be used.
+ *
+ * @author lulf
+ * @since 5.1
+ */
+public class ModelNode<MODEL extends ConfigModel> implements ConfigModelInstanceFactory<MODEL> {
+
+ final ComponentId id;
+ public final ConfigModelBuilder<MODEL> builder;
+ final Class<MODEL> clazz;
+ final Constructor<MODEL> constructor;
+ final List<MODEL> instances = new ArrayList<>();
+ private final Map<ComponentId, ModelNode> dependencies = new HashMap<>();
+
+ public ModelNode(ConfigModelBuilder<MODEL> builder) {
+ this.id = builder.getId();
+ this.builder = builder;
+ this.clazz = builder.getModelClass();
+ this.constructor = findConstructor(clazz);
+ }
+
+ private Constructor<MODEL> findConstructor(Class<MODEL> clazz) {
+ for (Constructor<?> ctor : clazz.getDeclaredConstructors()) {
+ if (ctor.getAnnotation(Inject.class) != null) {
+ return (Constructor<MODEL>) ctor;
+ }
+ }
+ return (Constructor<MODEL>) clazz.getDeclaredConstructors()[0];
+ }
+
+ boolean hasDependencies() {
+ return !dependencies.isEmpty();
+ }
+
+ boolean dependsOn(ModelNode node) {
+ return dependencies.containsKey(node.id);
+ }
+
+ /**
+ * This adds dependencies base on constructor arguments in the model classes themselves.
+ * These then have to be created by this mini-di framework and then handed to the builders
+ * that will fill them.
+ *
+ * TODO: This should be changed to model dependencies between model builders instead, such
+ * that they can create their model objects, and eventually make them immutable.
+ */
+ int addDependenciesFrom(List<ModelNode> modelNodes) {
+ int numDependencies = 0;
+ for (Type param : constructor.getGenericParameterTypes()) {
+ for (ModelNode node : modelNodes) {
+ if (param.equals(node.clazz) || isCollectionOf(param, node.clazz)) {
+ addDependency(node);
+ numDependencies++;
+ }
+ }
+ }
+ return numDependencies;
+ }
+
+ private boolean isCollectionOf(Type type, Class<?> nodeClazz) {
+ if (type instanceof ParameterizedType) {
+ ParameterizedType t = (ParameterizedType) type;
+ // Note: IntelliJ says the following cannot be equal but that is wrong
+ return (t.getRawType().equals(java.util.Collection.class) && t.getActualTypeArguments().length == 1 && t.getActualTypeArguments()[0].equals(nodeClazz));
+ }
+ return false;
+ }
+
+ private boolean isCollection(Type type) {
+ if (type instanceof ParameterizedType) {
+ ParameterizedType t = (ParameterizedType) type;
+ // Note: IntelliJ says the following cannot be equal but that is wrong
+ return (t.getRawType().equals(java.util.Collection.class) && t.getActualTypeArguments().length == 1);
+ }
+ return false;
+ }
+
+ private void addDependency(ModelNode node) {
+ dependencies.put(node.id, node);
+ }
+
+ Collection<ComponentId> listDependencyIds() {
+ return dependencies.keySet();
+ }
+
+ @Override
+ public MODEL createModel(ConfigModelContext context) {
+ try {
+ Type [] params = constructor.getGenericParameterTypes();
+ if (params.length < 1 || ! params[0].equals(ConfigModelContext.class)) {
+ throw new IllegalArgumentException("Constructor for " + clazz.getName() + " must have as its first argument a " + ConfigModelContext.class.getName());
+ }
+ Object arguments[] = new Object[params.length];
+ arguments[0] = context;
+ for (int i = 1; i < params.length; i++)
+ arguments[i] = findArgument(params[i]);
+ MODEL instance = constructor.newInstance(arguments);
+ instances.add(instance);
+ return instance;
+ } catch (InvocationTargetException | InstantiationException | IllegalAccessException e) {
+ throw new RuntimeException("Error constructing model '" + clazz.getName() + "'", e);
+ }
+ }
+
+ private Object findArgument(Type param) {
+ for (ModelNode dependency : dependencies.values()) {
+ if (param.equals(dependency.clazz))
+ return dependency.instances.get(0);
+ if (isCollectionOf(param, dependency.clazz))
+ return Collections.unmodifiableCollection(dependency.instances);
+ }
+ // For collections, we don't require that dependency has been added, we just give an empty collection
+ if (isCollection(param))
+ return Collections.emptyList();
+ throw new IllegalArgumentException("Unable to find constructor argument " + param + " for " + clazz.getName());
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/package-info.java b/config-model/src/main/java/com/yahoo/config/model/package-info.java
new file mode 100644
index 00000000000..594f52c0d14
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.config.model;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java
new file mode 100644
index 00000000000..41927bc09a9
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java
@@ -0,0 +1,448 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.producer;
+
+import com.google.common.annotations.Beta;
+import com.yahoo.config.ConfigInstance;
+import com.yahoo.config.subscription.ConfigInstanceUtil;
+import com.yahoo.config.model.ApplicationConfigProducerRoot;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.log.LogLevel;
+import com.yahoo.text.Utf8;
+import com.yahoo.vespa.config.*;
+import com.yahoo.vespa.model.*;
+import com.yahoo.vespa.model.admin.Admin;
+import com.yahoo.vespa.model.admin.MonitoringSystem;
+import com.yahoo.vespa.model.utils.FreezableMap;
+
+import java.io.*;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Type;
+import java.util.*;
+import java.util.logging.Logger;
+
+/**
+ * Superclass for all config producers.
+ * Config producers constructs and returns config instances on request.
+ *
+ * @author gjoranv
+ */
+public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProducer<?>>
+ implements ConfigProducer, ConfigInstance.Producer, Serializable {
+
+ private static final long serialVersionUID = 1L;
+ public static final Logger log = Logger.getLogger(AbstractConfigProducer.class.getPackage().toString());
+ private final String subId;
+ private final boolean hostedVespa;
+ private String configId = null;
+
+ private List<Service> descendantServices = new ArrayList<>();
+
+ private AbstractConfigProducer parent = null;
+
+ private UserConfigRepo userConfigs = new UserConfigRepo();
+
+ private final FreezableMap<String, CHILD> childrenBySubId = new FreezableMap<>(LinkedHashMap.class);
+
+ private static boolean isHostedVespa(AbstractConfigProducer parent) {
+ return (parent != null)
+ && (parent.getRoot() != null)
+ && (parent.getRoot().getDeployState() != null)
+ && parent.getRoot().getDeployState().isHostedVespa();
+ }
+
+ /**
+ * Creates a new AbstractConfigProducer with the given parent and subId.
+ * This constructor will add the resulting producer to the children of parent.
+ *
+ * @param parent The parent of this ConfigProducer
+ * @param subId The fragment of the config id for the producer
+ */
+ public AbstractConfigProducer(AbstractConfigProducer parent, String subId) {
+ this(subId, isHostedVespa(parent));
+
+ if (parent != null) {
+ parent.addChild(this);
+ }
+ }
+
+ protected final void setParent(AbstractConfigProducer parent) { this.parent = parent; }
+ public final String getSubId() { return subId; }
+ public final boolean isHostedVespa() { return hostedVespa; }
+
+ /**
+ * Create an config producer with a configId only. Used e.g. to create root nodes, and producers
+ * that are given children after construction using {@link #addChild(AbstractConfigProducer)}.
+ *
+ * @param subId The sub configId. Note that this can be prefixed when calling addChild with this producer as arg.
+ */
+ public AbstractConfigProducer(String subId) {
+ this(subId, false);
+ }
+
+ private AbstractConfigProducer(String subId, boolean hostedVespa) {
+ if (subId.indexOf('/') != -1) {
+ throw new IllegalArgumentException("A subId might not contain '/' : '" + subId + "'");
+ }
+ this.subId = subId;
+ this.hostedVespa = hostedVespa;
+ }
+
+ /**
+ * Adds a child to this config producer.
+ *
+ * @param child The child config producer to add.
+ */
+ protected void addChild(CHILD child) {
+ if (child == null) {
+ throw new IllegalArgumentException("Trying to add null child for: " + this);
+ }
+ if (child instanceof AbstractConfigProducerRoot) {
+ throw new IllegalArgumentException("Child cannot be a root node: " + child);
+ }
+
+ child.setParent(this);
+ if (childrenBySubId.get(child.getSubId()) != null) {
+ throw new IllegalArgumentException("Multiple services/instances of the id '" + child.getSubId() + "' under the service/instance " +
+ errorMsgClassName() + " '" + subId + "'. (This is commonly caused by service/node index " +
+ "collisions in the config.)." +
+ "\nExisting instance: " + childrenBySubId.get(child.getSubId()) +
+ "\nAttempted to add: " + child);
+ }
+ childrenBySubId.put(child.getSubId(), child);
+
+ if (child instanceof Service) {
+ addDescendantService((Service)child);
+ }
+ }
+
+ public void removeChild(CHILD child) {
+ if (child.getParent() != this)
+ throw new IllegalArgumentException("Could not remove " + child + ": Expected its parent to be " +
+ this + ", but was " + child.getParent());
+
+ if (child instanceof Service)
+ descendantServices.remove(child);
+
+ childrenBySubId.remove(child.getSubId());
+ child.setParent(null);
+ }
+
+ /**
+ * Helper to provide an error message on collisions of sub ids (ignore SimpleConfigProducer, use the parent in that case)
+ */
+ private String errorMsgClassName() {
+ if (getClass().equals(SimpleConfigProducer.class)) return parent.getClass().getSimpleName();
+ return getClass().getSimpleName();
+ }
+
+ /**
+ * Sets the user configs for this producer.
+ *
+ * @param repo User configs repo.
+ */
+ public void setUserConfigs(UserConfigRepo repo) { this.userConfigs = repo; }
+
+ /** Returns the user configs of this */
+ @Override
+ public UserConfigRepo getUserConfigs() { return userConfigs; }
+
+ /**
+ * ConfigProducers that must have a special config id should use
+ * setConfigId() instead of overloading this method. This is
+ * because config IDs must be registered through setConfigId().
+ */
+ public final String getConfigId() {
+ if (configId == null) throw new RuntimeException("The system topology must be frozen first.");
+ return configId;
+ }
+
+ /**
+ * Sets the config id for this producer. Will also add this
+ * service to the root node, so the new config id will be picked
+ * up. Note that this producer will be known with both the old
+ * and the new config id in the root node after using this method.
+ */
+ protected void addConfigId(String id) {
+ if (id == null) throw new NullPointerException("Config ID cannot be null.");
+ getRoot().addDescendant(id, this);
+ if (!isVespa() && (getVespa() != null))
+ getVespa().addDescendant(this);
+ }
+
+ /** Returns this ConfigProducer's children (only 1st level) */
+ public Map<String, CHILD> getChildren() { return Collections.unmodifiableMap(childrenBySubId); }
+
+ @Beta
+ public <J extends AbstractConfigProducer<?>> List<J> getChildrenByTypeRecursive(Class<J> type) {
+ List<J> validChildren = new ArrayList<>();
+
+ if (this.getClass().equals(type)) {
+ validChildren.add(type.cast(this));
+ }
+
+ Map<String, ? extends AbstractConfigProducer<?>> children = this.getChildren();
+ for (AbstractConfigProducer<?> child : children.values()) {
+ validChildren.addAll(child.getChildrenByTypeRecursive(type));
+ }
+
+ return Collections.unmodifiableList(validChildren);
+ }
+
+ /** Returns a list of all the children of this who are instances of Service */
+ public List<Service> getDescendantServices() { return Collections.unmodifiableList(descendantServices); }
+
+ protected void addDescendantService(Service s) { descendantServices.add(s); }
+
+ @Override
+ public final boolean cascadeConfig(ConfigInstance.Builder builder) {
+ boolean found=false;
+ if (parent != null)
+ found = parent.cascadeConfig(builder);
+
+ boolean foundHere = builder.dispatchGetConfig(this);
+ if (log.isLoggable(LogLevel.DEBUG)) {
+ log.log(LogLevel.DEBUG, "cascadeconfig in " + this + ", getting config " +
+ builder.getClass().getDeclaringClass().getName() + " for config id '" + configId + "' found here=" + foundHere);
+ }
+ found = found || foundHere;
+ return found;
+ }
+
+ @Override
+ public final boolean addUserConfig(ConfigInstance.Builder builder) {
+ boolean didApply = false;
+ if (parent != null) {
+ didApply = parent.addUserConfig(builder);
+ }
+
+ if (log.isLoggable(LogLevel.SPAM)) {
+ log.log(LogLevel.SPAM, "User configs is: " + userConfigs.toString());
+ }
+ // TODO: What do we do with md5. Currently ignored for user configs?
+ ConfigDefinitionKey key = new ConfigDefinitionKey(builder.getDefName(), builder.getDefNamespace());
+ if (userConfigs.get(key) != null) {
+ if (log.isLoggable(LogLevel.SPAM)) {
+ log.log(LogLevel.SPAM, "Apply in " + configId);
+ }
+ applyUserConfig(builder, userConfigs.get(key));
+ didApply = true;
+ }
+ return didApply;
+ }
+
+ private void applyUserConfig(ConfigInstance.Builder builder, ConfigPayloadBuilder payloadBuilder) {
+ ConfigInstance.Builder override;
+ if (builder instanceof GenericConfig.GenericConfigBuilder) {
+ // Means that the builder is unknown and that we should try to apply the payload without
+ // the real builder
+ override = getGenericConfigBuilderOverride((GenericConfig.GenericConfigBuilder) builder, payloadBuilder);
+ } else {
+ override = getConfigInstanceBuilderOverride(builder, ConfigPayload.fromBuilder(payloadBuilder));
+ }
+ ConfigInstanceUtil.setValues(builder, override);
+ }
+
+ private ConfigInstance.Builder getGenericConfigBuilderOverride(GenericConfig.GenericConfigBuilder builder, ConfigPayloadBuilder payloadBuilder) {
+ ConfigDefinitionKey key = new ConfigDefinitionKey(builder.getDefName(), builder.getDefNamespace());
+ return new GenericConfig.GenericConfigBuilder(key, payloadBuilder);
+ }
+
+ private ConfigInstance.Builder getConfigInstanceBuilderOverride(ConfigInstance.Builder builder, ConfigPayload payload) {
+ try {
+ ConfigTransformer transformer = new ConfigTransformer(builder.getClass().getEnclosingClass());
+ return transformer.toConfigBuilder(payload);
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("Error applying override to builder", e);
+ }
+ }
+
+ /**
+ * Returns the one and only HostSystem of the root node
+ * Must be overridden by root node.
+ */
+ public HostSystem getHostSystem() {
+ return getRoot().getHostSystem();
+ }
+
+ public AbstractConfigProducerRoot getRoot() {
+ return parent == null ? null : parent.getRoot();
+ }
+
+ /**
+ * Returns the {@link ApplicationConfigProducerRoot} that is the parent of this sub-tree, or null
+ * if this sub-tree has no Vespa parent.
+ */
+ private ApplicationConfigProducerRoot getVespa() {
+ if (isRoot()) return null;
+ return isVespa() ? (ApplicationConfigProducerRoot)this : parent.getVespa();
+ }
+
+ private boolean isRoot() {
+ return parent == null;
+ }
+
+ private boolean isVespa() {
+ return ((this instanceof ApplicationConfigProducerRoot) && parent.isRoot());
+ }
+
+ public AbstractConfigProducer getParent() { return parent; }
+
+ /**
+ * Writes files that need to be written. The files will usually only be
+ * written when the Vespa model is generated through the deploy-application
+ * script.
+ *
+ * TODO: Make sure all implemented ConfigProducers call createConfig()
+ * instead of getConfig() when implementing this method.
+ */
+ public void writeFiles(File directory) throws java.io.IOException {
+ if (!directory.isDirectory() && !directory.mkdirs()) {
+ throw new java.io.IOException("Cannot create directory: "+ directory);
+ }
+ for (Method m : getClass().getMethods()) {
+ try {
+ ConfigInstance.Builder builder = getBuilderIfIsGetConfig(m);
+ if (builder!=null) {
+ writeBuilder(directory, m, builder);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private void writeBuilder(File directory, Method m,
+ ConfigInstance.Builder builder) throws IllegalAccessException,
+ InvocationTargetException, InstantiationException,
+ NoSuchMethodException, IOException {
+ m.invoke(this, builder);
+ Class<?> configInstClass = builder.getClass().getEnclosingClass();
+ ConfigInstance inst = (ConfigInstance) configInstClass.getConstructor(builder.getClass()).newInstance(builder);
+ List<String> payloadList = ConfigInstance.serialize(inst);
+ File outfn = new File(directory, ConfigInstance.getDefName(inst.getClass()) + ".MODEL.cfg");
+ FileOutputStream out = new FileOutputStream(outfn);
+ for (String s : payloadList) {
+ out.write(Utf8.toBytes(s));
+ out.write('\n');
+ }
+ }
+
+ /**
+ * New Builder instance if m is getConfig(SomeConfig.Builder), or null
+ */
+ private ConfigInstance.Builder getBuilderIfIsGetConfig(Method m) throws InstantiationException, IllegalAccessException {
+ if (!"getConfig".equals(m.getName())) return null;
+ Type[] params = m.getParameterTypes();
+ if (params.length!=1) return null;
+ Type param = params[0];
+ if (!(param instanceof Class)) return null;
+ Class<?> paramClass = (Class<?>) param;
+ if (!(ConfigInstance.Builder.class.isAssignableFrom(paramClass))) return null;
+ return (ConfigInstance.Builder) paramClass.newInstance();
+ }
+
+ public void dump(PrintStream out) {
+ for (ConfigProducer c : getChildren().values()) {
+ out.println("id: " + c.getConfigId());
+ if (c.getChildren().size() > 0) {
+ c.dump(out);
+ }
+ }
+ }
+
+ void setupConfigId(String parentConfigId) {
+ if (this instanceof AbstractConfigProducerRoot) {
+ configId = "";
+ } else {
+ configId = parentConfigId + subId;
+ addConfigId(configId);
+ }
+
+ if (this instanceof AbstractConfigProducerRoot || this instanceof ApplicationConfigProducerRoot) {
+ setupChildConfigIds("");
+ } else {
+ setupChildConfigIds(configId + '/');
+ }
+ }
+
+ private static ClassLoader findInheritedClassLoader(Class clazz, String producerName) {
+ Class<?>[] interfazes = clazz.getInterfaces();
+ for (Class interfaze : interfazes) {
+ if (producerName.equals(interfaze.getName())) {
+ return interfaze.getClassLoader();
+ }
+ }
+ if (clazz.getSuperclass() == null)
+ return null;
+ return findInheritedClassLoader(clazz.getSuperclass(), producerName);
+ }
+
+ public ClassLoader getConfigClassLoader(String producerName) {
+ ClassLoader classLoader = findInheritedClassLoader(getClass(), producerName);
+ if (classLoader != null)
+ return classLoader;
+
+ // TODO: Make logic correct, so that the deepest child will be the one winning.
+ for (AbstractConfigProducer child : childrenBySubId.values()) {
+ ClassLoader loader = child.getConfigClassLoader(producerName);
+ if (loader != null) {
+ return loader;
+ }
+ }
+ return null;
+ }
+
+ private void setupChildConfigIds(String currentConfigId) {
+ for (AbstractConfigProducer child : childrenBySubId.values()) {
+ child.setupConfigId(currentConfigId);
+ }
+ }
+
+ void aggregateDescendantServices() {
+ for (AbstractConfigProducer child : childrenBySubId.values()) {
+ child.aggregateDescendantServices();
+ descendantServices.addAll(child.descendantServices);
+ }
+ }
+
+ void freeze() {
+ childrenBySubId.freeze();
+ for (AbstractConfigProducer child : childrenBySubId.values()) {
+ child.freeze();
+ }
+ }
+
+ public void mergeUserConfigs(UserConfigRepo newRepo) {
+ userConfigs.merge(newRepo);
+ }
+
+ @Override
+ public void validate() throws Exception {
+ assert (childrenBySubId.isFrozen());
+
+ for (AbstractConfigProducer child : childrenBySubId.values()) {
+ child.validate();
+ }
+ }
+
+ /** Returns a logger to be used for warnings and messages during initialization, never null */
+ public DeployLogger deployLogger() {
+ return parent.deployLogger();
+ }
+
+ // TODO: Make producers depend on AdminModel instead
+ /** Returns a monitoring service (yamas if that is configured, null otherwise) */
+ protected MonitoringSystem getMonitoringService() {
+ AbstractConfigProducerRoot root = getRoot();
+ Admin admin = (root == null? null : root.getAdmin());
+ if (admin == null) {
+ return null;
+ }
+ if (admin.getYamas() != null) {
+ return admin.getYamas();
+ }
+ return null;
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducerRoot.java
new file mode 100644
index 00000000000..b85e2b8fa97
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducerRoot.java
@@ -0,0 +1,67 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.producer;
+
+import com.yahoo.config.model.ConfigModelRepo;
+import com.yahoo.vespa.model.ConfigProducer;
+import com.yahoo.vespa.model.ConfigProducerRoot;
+import com.yahoo.vespa.model.Service;
+import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * The parent class of classes having the role as the root of a config producer tree.
+ *
+ * @author tonytv
+ */
+public abstract class AbstractConfigProducerRoot extends AbstractConfigProducer<AbstractConfigProducer<?>>
+ implements ConfigProducerRoot {
+
+ /** The ConfigProducers contained in this model indexed by config id */
+ protected final Map<String, ConfigProducer> id2producer = new LinkedHashMap<>();
+
+ public AbstractConfigProducerRoot(String rootConfigId) {
+ super(rootConfigId);
+ }
+
+ public AbstractConfigProducerRoot getRoot() {
+ return this;
+ }
+
+ public abstract FileDistributionConfigProducer getFileDistributionConfigProducer();
+
+ /**
+ * Freezes the parent - child connections of the model
+ * and sets information derived from the topology.
+ */
+ public void freezeModelTopology() {
+ freeze();
+ setupConfigId("");
+ aggregateDescendantServices();
+ }
+
+ public abstract ConfigModelRepo configModelRepo();
+
+ /**
+ * Returns the ConfigProducer with the given id if such configId exists.
+ *
+ * @param configId The configId, e.g. "search.0/tld.0"
+ * @return ConfigProducer with the given configId
+ */
+ public Optional<ConfigProducer> getConfigProducer(String configId) {
+ return Optional.ofNullable(id2producer.get(configId));
+ }
+
+ /**
+ * Returns the Service with the given id if such configId exists and it belongs to a Service ConfigProducer.
+ *
+ * @param configId The configId, e.g. "search.0/tld.0"
+ * @return Service with the given configId
+ */
+ public Optional<Service> getService(String configId) {
+ return getConfigProducer(configId)
+ .filter(Service.class::isInstance)
+ .map(Service.class::cast);
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/UserConfigRepo.java b/config-model/src/main/java/com/yahoo/config/model/producer/UserConfigRepo.java
new file mode 100644
index 00000000000..d32b0c75e69
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/producer/UserConfigRepo.java
@@ -0,0 +1,109 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.producer;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.config.*;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A UserConfigRepo is a repository for user configs, typically for a particular config producer. The repo encapsulates
+ * how the user configs are stored, and defines the methods to retrieve user configs and merge the repo with others.
+ *
+ * @author lulf
+ * @since 5.1
+ */
+public class UserConfigRepo {
+ private final Map<ConfigDefinitionKey, ConfigPayloadBuilder> userConfigsMap;
+
+ public UserConfigRepo() {
+ this.userConfigsMap = new LinkedHashMap<>();
+ }
+
+ @Override
+ public UserConfigRepo clone() {
+ return new UserConfigRepo(copyBuilders(userConfigsMap));
+ }
+
+ /**
+ * Must copy the builder, because the merge method on {@link AbstractConfigProducer} might override the row's builders otherwise
+ */
+ private Map<ConfigDefinitionKey, ConfigPayloadBuilder> copyBuilders(Map<ConfigDefinitionKey, ConfigPayloadBuilder> source) {
+ Map<ConfigDefinitionKey, ConfigPayloadBuilder> ret = new LinkedHashMap<>();
+ for (Map.Entry<ConfigDefinitionKey, ConfigPayloadBuilder> e : source.entrySet()) {
+ ConfigDefinitionKey key = e.getKey();
+ ConfigPayloadBuilder sourceVal = e.getValue();
+ ConfigPayloadBuilder destVal = new ConfigPayloadBuilder(ConfigPayload.fromBuilder(sourceVal));
+ ret.put(key, destVal);
+ }
+ return ret;
+ }
+
+ public UserConfigRepo(Map<ConfigDefinitionKey, ConfigPayloadBuilder> map) {
+ this.userConfigsMap = map;
+ }
+
+ public UserConfigRepo(UserConfigRepo userConfigRepo) {
+ this.userConfigsMap = userConfigRepo.userConfigsMap;
+ }
+
+ public ConfigPayloadBuilder get(ConfigDefinitionKey key) {
+ return userConfigsMap.get(key);
+ }
+
+ public void merge(UserConfigRepo newRepo) {
+ for (Map.Entry<ConfigDefinitionKey, ConfigPayloadBuilder> entry : newRepo.userConfigsMap.entrySet()) {
+ if (entry.getValue() == null) continue;
+
+ ConfigDefinitionKey key = entry.getKey();
+ if (userConfigsMap.containsKey(key)) {
+ ConfigPayloadBuilder lhsBuilder = userConfigsMap.get(key);
+ ConfigPayloadBuilder rhsBuilder = entry.getValue();
+ lhsBuilder.override(rhsBuilder);
+ } else {
+ userConfigsMap.put(key, entry.getValue());
+ }
+ }
+ }
+
+ public boolean isEmpty() {
+ return userConfigsMap.isEmpty();
+ }
+
+ public int size() {
+ return userConfigsMap.size();
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ for (ConfigDefinitionKey key : userConfigsMap.keySet()) {
+ sb.append(key.toString());
+ }
+ return sb.toString();
+ }
+
+ /**
+ * The keys of all the configs contained in this.
+ * @return a set of ConfigDefinitionsKey
+ */
+ public Set<ConfigDefinitionKey> configsProduced() {
+ return userConfigsMap.keySet();
+ }
+
+ /**
+ * Will take the warning messages stored on the payload builders, and apply them to the producer's {@link DeployLogger}
+ * @param producer the producer to apply warnings to
+ */
+ public void applyWarnings(AbstractConfigProducer<?> producer) {
+ for (ConfigPayloadBuilder b : userConfigsMap.values()) {
+ for (String warning : b.warnings()) {
+ producer.deployLogger().log(LogLevel.WARNING, warning);
+ }
+ }
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/package-info.java b/config-model/src/main/java/com/yahoo/config/model/producer/package-info.java
new file mode 100644
index 00000000000..c58a116b196
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/producer/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.config.model.producer;
+
+import com.yahoo.osgi.annotation.ExportPackage; \ No newline at end of file
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/Host.java b/config-model/src/main/java/com/yahoo/config/model/provision/Host.java
new file mode 100644
index 00000000000..9d97308fc76
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/Host.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.provision;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A hostname with zero or more aliases.
+ *
+ * @author musum
+ */
+public class Host {
+
+ private final String hostname;
+ private final List<String> hostAliases;
+
+ public Host(String hostname) {
+ this.hostname = hostname;
+ this.hostAliases = new ArrayList<>();
+ }
+
+ public Host(String hostname, List<String> hostAliases) {
+ this.hostname = hostname;
+ this.hostAliases = hostAliases;
+ }
+
+ public String getHostname() {
+ return hostname;
+ }
+
+ public List<String> getHostAliases() {
+ return hostAliases;
+ }
+
+ @Override
+ public String toString() {
+ return hostname + " (aliases: " + hostAliases + ")";
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java
new file mode 100644
index 00000000000..ece781dd5bd
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java
@@ -0,0 +1,120 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.provision;
+
+import com.yahoo.config.model.builder.xml.XmlHelper;
+import com.yahoo.log.LogLevel;
+import com.yahoo.net.HostName;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.*;
+import java.util.logging.Logger;
+
+/**
+ * TODO: What is this?
+ *
+ * @author musum
+ */
+public class Hosts {
+
+ public static final Logger log = Logger.getLogger(Hosts.class.getPackage().toString());
+
+ private final HashMap<String, Host> hosts = new LinkedHashMap<>();
+ private final Map<String, String> alias2hostname = new LinkedHashMap<>();
+ private final Map<String, Host> alias2host = new LinkedHashMap<>();
+
+ /**
+ * Builds host system from a hosts.xml file
+ *
+ * @param hostsFile a reader for host from application package
+ * @return the HostSystem for this application package
+ */
+ public static Hosts getHosts(Reader hostsFile) {
+ Hosts hosts = new Hosts();
+ Document doc;
+ try {
+ doc = XmlHelper.getDocumentBuilder().parse(new InputSource(hostsFile));
+ } catch (SAXException | IOException e) {
+ throw new IllegalArgumentException(e);
+ }
+ for (Element hostE : XML.getChildren(doc.getDocumentElement(), "host")) {
+ String name = hostE.getAttribute("name");
+ if (name.equals("")) {
+ throw new RuntimeException("Missing 'name' attribute for host.");
+ }
+ if ("localhost".equals(name)) {
+ name = HostName.getLocalhost();
+ }
+ final List<String> hostAliases = VespaDomBuilder.getHostAliases(hostE.getChildNodes());
+ if (hostAliases.isEmpty()) {
+ throw new IllegalArgumentException("No host aliases defined for host '" + name + "'");
+ }
+ Host host = new Host(name, hostAliases);
+ hosts.addHost(host, hostAliases);
+ }
+ log.log(LogLevel.DEBUG, "Created hosts:" + hosts);
+ return hosts;
+ }
+
+ public Collection<Host> getHosts() {
+ return hosts.values();
+ }
+
+ /**
+ * Adds one host to this host system.
+ *
+ * @param host The host to add
+ * @param aliases The aliases for this host.
+ */
+ public void addHost(Host host, List<String> aliases) {
+ hosts.put(host.getHostname(), host);
+ if ((aliases != null) && (aliases.size() > 0)) {
+ addHostAliases(aliases, host);
+ }
+ }
+
+ /**
+ * Add all aliases for one host
+ *
+ * @param hostAliases A list of host aliases
+ * @param host The Host instance to add the alias for
+ */
+ private void addHostAliases(List<String> hostAliases, Host host) {
+ if (hostAliases.size() < 1) {
+ throw new RuntimeException("Host '" + host.getHostname() + "' must have at least one <alias> tag.");
+ }
+ for (String alias : hostAliases) {
+ addHostAlias(alias, host);
+ }
+ }
+
+ /**
+ * Adds an alias for the given host
+ *
+ * @param alias alias (string) for a Host
+ * @param host the {@link Host} to add the alias for
+ */
+ protected void addHostAlias(String alias, Host host) {
+ if (alias2hostname.containsKey(alias)) {
+ throw new RuntimeException("Alias '" + alias + "' must be used for only one host!");
+ }
+ alias2hostname.put(alias, host.getHostname());
+ alias2host.put(alias, host);
+ }
+
+ public Map<String, Host> getAlias2host() {
+ return alias2host;
+ }
+
+ @Override
+ public String toString() {
+ return "Hosts: " + hosts.keySet();
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java
new file mode 100644
index 00000000000..20dc190d8e7
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/HostsXmlProvisioner.java
@@ -0,0 +1,64 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.provision;
+
+import com.yahoo.config.model.api.HostProvisioner;
+import com.yahoo.config.provision.*;
+import com.yahoo.vespa.model.container.Container;
+
+import java.io.Reader;
+import java.util.List;
+
+/**
+ * A host provisioner based on a hosts.xml file.
+ * No state in this provisioner, i.e it does not know anything about the active
+ * application if one exists. Pre-condition: A valid hosts file.
+ *
+ * @author musum
+ * @since 5.11
+ */
+public class HostsXmlProvisioner implements HostProvisioner {
+
+ private final Hosts hosts;
+ public static final String IMPLICIT_ADMIN_HOSTALIAS = "INTERNAL_VESPA_IMPLICIT_ADMIN";
+
+ public HostsXmlProvisioner(Reader hosts) {
+ this.hosts = Hosts.getHosts(hosts);
+ }
+
+ @Override
+ public HostSpec allocateHost(String alias) {
+ /**
+ * Some special rules to allow no admin elements as well
+ * as jdisc element without nodes.
+ */
+ if (alias.equals(IMPLICIT_ADMIN_HOSTALIAS)) {
+ if (hosts.getHosts().size() > 1) {
+ throw new IllegalArgumentException("More than 1 host specified (" + hosts.getHosts().size() + ") and <admin> not specified");
+ } else {
+ return host2HostSpec(getFirstHost());
+ }
+ } else if (alias.equals(Container.SINGLENODE_CONTAINER_SERVICESPEC)) {
+ return host2HostSpec(getFirstHost());
+ }
+ for (Host host : hosts.getHosts()) {
+ if (host.getHostAliases().contains(alias)) {
+ return new HostSpec(host.getHostname(), host.getHostAliases());
+ }
+ }
+ throw new IllegalArgumentException("Unable to find host for alias '" + alias + "'");
+ }
+
+ @Override
+ public List<HostSpec> prepare(ClusterSpec cluster, Capacity quantity, int groups, ProvisionLogger logger) {
+ throw new UnsupportedOperationException();
+ }
+
+ private HostSpec host2HostSpec(Host host) {
+ return new HostSpec(host.getHostname(), host.getHostAliases());
+ }
+
+ private Host getFirstHost() {
+ return hosts.getHosts().iterator().next();
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
new file mode 100644
index 00000000000..c06efece329
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java
@@ -0,0 +1,164 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.provision;
+
+import com.yahoo.collections.ListMap;
+import com.yahoo.collections.Pair;
+import com.yahoo.config.model.api.HostProvisioner;
+import com.yahoo.config.provision.*;
+
+import java.util.*;
+
+/**
+ * In memory host provisioner. NB! ATM cannot be reused after allocate has been called.
+ *
+ * @author musum
+ * @author bratseth
+ */
+public class InMemoryProvisioner implements HostProvisioner {
+
+ /**
+ * If this is true an exception is thrown when all nodes are used.
+ * If false this will simply return nodes best effort, preferring to satisfy the
+ * number of groups requested when possible.
+ */
+ private final boolean failOnOutOfCapacity;
+
+ /** Hosts which should be returned as retired */
+ private final Set<String> retiredHostNames;
+
+ /** Free hosts of each flavor */
+ private final ListMap<String, Host> freeNodes = new ListMap<>();
+ private final Map<String, HostSpec> legacyMapping = new LinkedHashMap<>();
+ private final Map<ClusterSpec, List<HostSpec>> allocations = new LinkedHashMap<>();
+
+ /** Indexes must be unique across all groups in a cluster */
+ private final Map<Pair<ClusterSpec.Type,ClusterSpec.Id>, Integer> nextIndexInCluster = new HashMap<>();
+
+ /** Use this index as start index for all clusters */
+ private final int startIndexForClusters;
+
+ /** Creates this with a number of nodes of the flavor 'default' */
+ public InMemoryProvisioner(int nodeCount) {
+ this(Collections.singletonMap("default", createHostInstances(nodeCount)), true, 0);
+ }
+
+ /** Creates this with a set of host names of the flavor 'default' */
+ public InMemoryProvisioner(boolean failOnOutOfCapacity, String... hosts) {
+ this(Collections.singletonMap("default", toHostInstances(hosts)), failOnOutOfCapacity, 0);
+ }
+
+ /** Creates this with a set of hosts of the flavor 'default' */
+ public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, String ... retiredHostNames) {
+ this(Collections.singletonMap("default", hosts.getHosts()), failOnOutOfCapacity, 0, retiredHostNames);
+ }
+
+ /** Creates this with a set of hosts of the flavor 'default' */
+ public InMemoryProvisioner(Hosts hosts, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) {
+ this(Collections.singletonMap("default", hosts.getHosts()), failOnOutOfCapacity, startIndexForClusters, retiredHostNames);
+ }
+
+ public InMemoryProvisioner(Map<String, Collection<Host>> hosts, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) {
+ this.failOnOutOfCapacity = failOnOutOfCapacity;
+ for (Map.Entry<String, Collection<Host>> hostsOfFlavor : hosts.entrySet())
+ for (Host host : hostsOfFlavor.getValue())
+ freeNodes.put(hostsOfFlavor.getKey(), host);
+ this.retiredHostNames = new HashSet<>(Arrays.asList(retiredHostNames));
+ this.startIndexForClusters = startIndexForClusters;
+ }
+
+ private static Collection<Host> toHostInstances(String[] hostnames) {
+ List<Host> hosts = new ArrayList<>();
+ for (String hostname : hostnames) {
+ hosts.add(new Host(hostname));
+ }
+ return hosts;
+ }
+
+ private static Collection<Host> createHostInstances(int hostCount) {
+ List<Host> hosts = new ArrayList<>();
+ for (int i = 1; i <= hostCount; i++) {
+ hosts.add(new Host("host" + i));
+ }
+ return hosts;
+ }
+
+ @Override
+ public HostSpec allocateHost(String alias) {
+ if (legacyMapping.containsKey(alias)) return legacyMapping.get(alias);
+ List<Host> defaultHosts = freeNodes.get("default");
+ if (defaultHosts.isEmpty()) throw new IllegalArgumentException("No more hosts of default flavor available");
+ Host newHost = freeNodes.removeValue("default", 0);
+ HostSpec hostSpec = new HostSpec(newHost.getHostname(), newHost.getHostAliases());
+ legacyMapping.put(alias, hostSpec);
+ return hostSpec;
+ }
+
+ @Override
+ public List<HostSpec> prepare(ClusterSpec cluster, Capacity requestedCapacity, int groups, ProvisionLogger logger) {
+ if (cluster.group().isPresent() && groups > 1)
+ throw new IllegalArgumentException("Cannot both be specifying a group and ask for groups to be created");
+ if (requestedCapacity.nodeCount() % groups != 0)
+ throw new IllegalArgumentException("Requested " + requestedCapacity.nodeCount() + " nodes in " +
+ groups + " groups, but the node count is not divisible into this number of groups");
+
+ int capacity = failOnOutOfCapacity ? requestedCapacity.nodeCount() :
+ Math.min(requestedCapacity.nodeCount(), freeNodes.get("default").size() + totalAllocatedTo(cluster));
+ if (groups > capacity)
+ groups = capacity;
+
+ String flavor = requestedCapacity.flavor().orElse("default");
+
+ List<HostSpec> allocation = new ArrayList<>();
+ if (groups == 1) {
+ allocation.addAll(allocateHostGroup(cluster, flavor, capacity, startIndexForClusters));
+ }
+ else {
+ for (int i = 0; i < groups; i++) {
+ allocation.addAll(allocateHostGroup(cluster.changeGroup(Optional.of(ClusterSpec.Group.from(String.valueOf(i)))),
+ flavor,
+ capacity / groups,
+ allocation.size()));
+ }
+ }
+ for (ListIterator<HostSpec> i = allocation.listIterator(); i.hasNext(); ) {
+ HostSpec host = i.next();
+ if (retiredHostNames.contains(host.hostname()))
+ i.set(retire(host));
+ }
+ return allocation;
+ }
+
+ private HostSpec retire(HostSpec host) {
+ return new HostSpec(host.hostname(), host.aliases(), host.membership().get().retire());
+ }
+
+ private List<HostSpec> allocateHostGroup(ClusterSpec clusterGroup, String flavor, int nodesInGroup, int startIndex) {
+ List<HostSpec> allocation = allocations.getOrDefault(clusterGroup, new ArrayList<>());
+ allocations.put(clusterGroup, allocation);
+
+ int nextIndex = nextIndexInCluster.getOrDefault(new Pair<>(clusterGroup.type(), clusterGroup.id()), startIndex);
+ while (allocation.size() < nodesInGroup) {
+ if (freeNodes.get(flavor).isEmpty()) throw new IllegalArgumentException("No nodes of flavor '" + flavor + "' available");
+ Host newHost = freeNodes.removeValue(flavor, 0);
+ ClusterMembership membership = ClusterMembership.from(clusterGroup, nextIndex++);
+ allocation.add(new HostSpec(newHost.getHostname(), newHost.getHostAliases(), membership));
+ }
+ nextIndexInCluster.put(new Pair<>(clusterGroup.type(), clusterGroup.id()), nextIndex);
+
+ while (allocation.size() > nodesInGroup)
+ allocation.remove(0);
+
+ return allocation;
+ }
+
+ private int totalAllocatedTo(ClusterSpec cluster) {
+ int count = 0;
+ for (Map.Entry<ClusterSpec, List<HostSpec>> allocation : allocations.entrySet()) {
+ if ( ! allocation.getKey().type().equals(cluster.type())) continue;
+ if ( ! allocation.getKey().id().equals(cluster.id())) continue;
+ count += allocation.getValue().size();
+ }
+ return count;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java
new file mode 100644
index 00000000000..38ed728e4e8
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java
@@ -0,0 +1,49 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.provision;
+
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.api.HostProvisioner;
+import com.yahoo.config.provision.*;
+import com.yahoo.net.HostName;
+import com.yahoo.vespa.model.HostSystem;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.net.UnknownHostException;
+
+/**
+ * A host provisioner used when there is no hosts.xml file (using localhost as the only host)
+ * No state in this provisioner, i.e it does not know anything about the active
+ * application if one exists.
+ *
+ * @author musum
+ * @since 5.11
+ */
+public class SingleNodeProvisioner implements HostProvisioner {
+
+ private final Host host; // the only host in this system
+ private final HostSpec hostSpec;
+ private int counter = 0;
+
+ public SingleNodeProvisioner() {
+ try {
+ host = new Host(HostSystem.lookupCanonicalHostname(HostName.getLocalhost()));
+ } catch (UnknownHostException e) {
+ throw new RuntimeException(e);
+ }
+ this.hostSpec = new HostSpec(host.getHostname(), host.getHostAliases());
+ }
+
+ @Override
+ public HostSpec allocateHost(String alias) {
+ return hostSpec;
+ }
+
+ @Override
+ public List<HostSpec> prepare(ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { // TODO: This should fail if capacity requested is more than 1
+ List<HostSpec> hosts = new ArrayList<>();
+ hosts.add(new HostSpec(host.getHostname(), host.getHostAliases(), ClusterMembership.from(cluster, counter++)));
+ return hosts;
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/package-info.java b/config-model/src/main/java/com/yahoo/config/model/provision/package-info.java
new file mode 100644
index 00000000000..a39e4025efa
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/provision/package-info.java
@@ -0,0 +1,5 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+@ExportPackage
+package com.yahoo.config.model.provision;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/config-model/src/main/java/com/yahoo/config/model/test/ConfigModelTestUtil.java b/config-model/src/main/java/com/yahoo/config/model/test/ConfigModelTestUtil.java
new file mode 100644
index 00000000000..b75147774ab
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/test/ConfigModelTestUtil.java
@@ -0,0 +1,40 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.test;
+
+import com.yahoo.collections.CollectionUtil;
+import com.yahoo.config.model.builder.xml.XmlHelper;
+import org.w3c.dom.Element;
+import org.xml.sax.InputSource;
+
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * @author lulf
+ * @since 5.1
+ */
+public class ConfigModelTestUtil {
+ /**
+ * @param xmlLines XML with " replaced with '
+ */
+ public static Element parse(String... xmlLines) {
+ List<String> lines = new ArrayList<>();
+ lines.add("<?xml version='1.0' encoding='utf-8' ?>");
+ lines.addAll(Arrays.asList(xmlLines));
+
+ try {
+ return XmlHelper.getDocumentBuilder().parse(
+ inputSource((CollectionUtil.mkString(lines, "\n").replace("'", "\""))))
+ .getDocumentElement();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static InputSource inputSource(String str) {
+ return new InputSource(new StringReader(str));
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java
new file mode 100644
index 00000000000..731410c9bf3
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java
@@ -0,0 +1,243 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.test;
+
+import com.yahoo.config.application.api.ComponentInfo;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.application.api.UnparsedConfigDefinition;
+import com.yahoo.config.application.api.ApplicationFile;
+import com.yahoo.config.provision.Version;
+import com.yahoo.io.IOUtils;
+import com.yahoo.path.Path;
+import com.yahoo.io.reader.NamedReader;
+import com.yahoo.searchdefinition.*;
+import com.yahoo.searchdefinition.parser.ParseException;
+import com.yahoo.vespa.config.ConfigDefinitionKey;
+import com.yahoo.config.application.api.ApplicationPackage;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.*;
+
+/**
+ * For testing purposes only
+ *
+ * @author tonytv
+ */
+public class MockApplicationPackage implements ApplicationPackage {
+ public static final String MUSIC_SEARCHDEFINITION = createSearchDefinition("music", "foo");
+ public static final String BOOK_SEARCHDEFINITION = createSearchDefinition("book", "bar");
+
+ private final String hostsS;
+ private final String servicesS;
+ private final List<String> searchDefinitions;
+ private final String searchDefinitionDir;
+ private final Optional<String> deploymentInfo;
+ private final Optional<String> validationOverrides;
+ private final boolean failOnValidateXml;
+
+ private MockApplicationPackage(String hosts, String services, List<String> searchDefinitions, String searchDefinitionDir,
+ String deploymentInfo, String validationOverrides, boolean failOnValidateXml) {
+ this.hostsS = hosts;
+ this.servicesS = services;
+ this.searchDefinitions = searchDefinitions;
+ this.searchDefinitionDir = searchDefinitionDir;
+ this.deploymentInfo = Optional.ofNullable(deploymentInfo);
+ this.validationOverrides = Optional.ofNullable(validationOverrides);
+ this.failOnValidateXml = failOnValidateXml;
+ }
+
+ @Override
+ public String getApplicationName() {
+ return "mock application";
+ }
+
+ @Override
+ public Reader getServices() {
+ return new StringReader(servicesS);
+ }
+
+ @Override
+ public Reader getHosts() {
+ if (hostsS==null) return null;
+ return new StringReader(hostsS);
+ }
+
+ @Override
+ public List<NamedReader> getSearchDefinitions() {
+ ArrayList<NamedReader> readers = new ArrayList<>();
+ SearchBuilder searchBuilder = new SearchBuilder(this, new RankProfileRegistry());
+ for (String sd : searchDefinitions) {
+ try {
+ String name = searchBuilder.importString(sd);
+ readers.add(new NamedReader(name + ApplicationPackage.SD_NAME_SUFFIX, new StringReader(sd)));
+ } catch (ParseException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return readers;
+ }
+
+ @Override
+ public List<NamedReader> searchDefinitionContents() {
+ return new ArrayList<>();
+ }
+
+ @Override
+ public Map<ConfigDefinitionKey, UnparsedConfigDefinition> getAllExistingConfigDefs() {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public List<NamedReader> getFiles(Path dir,String fileSuffix,boolean recurse) {
+ return new ArrayList<>();
+ }
+
+ @Override
+ public ApplicationFile getFile(Path file) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String getHostSource() {
+ return "mock source";
+ }
+
+ @Override
+ public String getServicesSource() {
+ return "mock source";
+ }
+
+ @Override
+ public Optional<Reader> getDeployment() {
+ return deploymentInfo.map(StringReader::new);
+ }
+
+ @Override
+ public Optional<Reader> getValidationOverrides() {
+ return validationOverrides.map(StringReader::new);
+ }
+
+ public List<ComponentInfo> getComponentsInfo(Version vespaVersion) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public Reader getRankingExpression(String name) {
+ File expressionFile = new File(searchDefinitionDir, name);
+ try {
+ return IOUtils.createReader(expressionFile, "utf-8");
+ }
+ catch (IOException e) {
+ throw new IllegalArgumentException("Could not read ranking expression file '" +
+ expressionFile.getAbsolutePath() + "'", e);
+ }
+ }
+
+ public static ApplicationPackage createEmpty() {
+ return new MockApplicationPackage.Builder().withHosts(emptyHosts).withServices(emptyServices).build();
+ }
+
+ public static ApplicationPackage fromSearchDefinitionDirectory(String dir) {
+ return new MockApplicationPackage.Builder()
+ .withEmptyHosts()
+ .withEmptyServices()
+ .withSearchDefinitionDir(dir).build();
+ }
+
+ public static class Builder {
+ private String hosts = null;
+ private String services = null;
+ private List<String> searchDefinitions = Collections.emptyList();
+ private String searchDefinitionDir = null;
+ private String deploymentInfo = null;
+ private String validationOverrides = null;
+ private boolean failOnValidateXml = false;
+
+ public Builder() {
+ }
+
+ public Builder withEmptyHosts() {
+ return this.withHosts(emptyHosts);
+ }
+
+ public Builder withHosts(String hosts) {
+ this.hosts = hosts;
+ return this;
+ }
+
+ public Builder withEmptyServices() {
+ return this.withServices(emptyServices);
+ }
+
+ public Builder withServices(String services) {
+ this.services = services;
+ return this;
+ }
+
+ public Builder withSearchDefinition(String searchDefinition) {
+ this.searchDefinitions = Collections.singletonList(searchDefinition);
+ return this;
+ }
+
+ public Builder withSearchDefinitions(List<String> searchDefinition) {
+ this.searchDefinitions = Collections.unmodifiableList(searchDefinition);
+ return this;
+ }
+
+ public Builder withSearchDefinitionDir(String searchDefinitionDir) {
+ this.searchDefinitionDir = searchDefinitionDir;
+ return this;
+ }
+
+ public Builder withDeploymentInfo(String deploymentInfo) {
+ this.deploymentInfo = deploymentInfo;
+ return this;
+ }
+
+ public Builder withValidationOverrides(String validationOverrides) {
+ this.validationOverrides = validationOverrides;
+ return this;
+ }
+
+ public Builder failOnValidateXml() {
+ this.failOnValidateXml = true;
+ return this;
+ }
+
+ public ApplicationPackage build() {
+ return new MockApplicationPackage(hosts, services, searchDefinitions, searchDefinitionDir,
+ deploymentInfo, validationOverrides, failOnValidateXml);
+ }
+ }
+
+ public static String createSearchDefinition(String name, String fieldName) {
+ return "search " + name + " {" +
+ " document " + name + " {" +
+ " field " + fieldName + " type string {}" +
+ " }" +
+ "}";
+ }
+
+ private static final String emptyServices = "<services version=\"1.0\">" +
+ " <admin version=\"2.0\">" +
+ " <adminserver hostalias=\"node1\" />" +
+ " </admin>" +
+ "</services>";
+
+ private static final String emptyHosts = "<hosts>" +
+ " <host name=\"localhost\">" +
+ " <alias>node1</alias>" +
+ " </host>" +
+ "</hosts>";
+
+ @Override
+ public void validateXML(DeployLogger logger) throws IOException {
+ if (failOnValidateXml) {
+ throw new IllegalArgumentException("Error in application package");
+ } else {
+ throw new UnsupportedOperationException("This application package cannot validate XML");
+ }
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java
new file mode 100644
index 00000000000..fa84cf1c7eb
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java
@@ -0,0 +1,169 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.test;
+
+import com.yahoo.config.ConfigInstance;
+import com.yahoo.config.model.ConfigModelRepo;
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.model.application.provider.BaseDeployLogger;
+import com.yahoo.config.application.api.DeployLogger;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.builder.xml.XmlHelper;
+import com.yahoo.config.model.producer.AbstractConfigProducer;
+import com.yahoo.config.model.producer.AbstractConfigProducerRoot;
+import com.yahoo.text.XML;
+import com.yahoo.vespa.model.ConfigProducer;
+import com.yahoo.vespa.model.HostSystem;
+import com.yahoo.vespa.model.admin.Admin;
+import com.yahoo.vespa.model.builder.xml.dom.DomAdminV2Builder;
+import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer;
+import com.yahoo.vespa.model.filedistribution.FileDistributor;
+import org.w3c.dom.Document;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Set;
+
+
+/**
+ * Use for testing. Use as parent for the config producer(s) you want to test, to test
+ * only a subtree of the producers.
+ *
+ * @author gjoranv
+ */
+public class MockRoot extends AbstractConfigProducerRoot {
+ private static final long serialVersionUID = 1L;
+ public static final String MOCKHOST = "mockhost";
+
+ private HostSystem hostSystem;
+
+ private final DeployState deployState;
+ private FileDistributor fileDistributor;
+ private Admin admin;
+
+ public MockRoot() {
+ this("");
+ }
+
+ public MockRoot(String rootConfigId) {
+ this(rootConfigId, new MockApplicationPackage.Builder().build());
+ }
+
+ public MockRoot(String rootConfigId, ApplicationPackage applicationPackage) {
+ this(rootConfigId, new DeployState.Builder().applicationPackage(applicationPackage).build());
+ }
+
+ public MockRoot(String rootConfigId, DeployState deployState) {
+ super(rootConfigId);
+ hostSystem = new HostSystem(this, "hostsystem", deployState.getProvisioner());
+ this.deployState = deployState;
+ fileDistributor = new FileDistributor(deployState.getFileRegistry());
+ }
+
+ public FileDistributionConfigProducer getFileDistributionConfigProducer() {
+ return null;
+ }
+
+ @Override
+ public ConfigModelRepo configModelRepo() {
+ return new ConfigModelRepo();
+ }
+
+ public Set<String> getConfigIds() {
+ return Collections.unmodifiableSet(id2producer.keySet());
+ }
+
+ @Override
+ public ConfigInstance.Builder getConfig(ConfigInstance.Builder builder, String configId) {
+ ConfigProducer cp = id2producer.get(configId);
+ if (cp == null) return null;
+
+ cp.cascadeConfig(builder);
+ cp.addUserConfig(builder);
+ return builder;
+ }
+
+ @SuppressWarnings("unchecked")
+ public <T extends ConfigInstance> T getConfig(Class<T> configClass, String configId) {
+ try {
+ ConfigInstance.Builder builder = getConfig(getBuilder(configClass).newInstance(), configId);
+ return configClass.getConstructor(builder.getClass()).newInstance(builder);
+ } catch (InstantiationException | NoSuchMethodException | InvocationTargetException | IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public ConfigProducer getProducer(String configId) {
+ return id2producer.get(configId);
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <T extends ConfigInstance> Class<? extends ConfigInstance.Builder> getBuilder(Class<T> configClass) {
+ for (Class<?> memberClass : configClass.getClasses()) {
+ if (memberClass.getSimpleName().equals("Builder"))
+ return (Class<? extends ConfigInstance.Builder>) memberClass;
+ }
+ throw new RuntimeException("Missing builder");
+ }
+
+ @Override
+ public DeployState getDeployState() {
+ return deployState;
+ }
+
+ public FileDistributor getFileDistributor() {
+ return fileDistributor;
+ }
+
+ public HostSystem getHostSystem() {
+ return hostSystem;
+ }
+
+ public void addDescendant(String configId, AbstractConfigProducer descendant) {
+ if (id2producer.containsKey(configId)) {
+ throw new RuntimeException
+ ("Config ID '" + configId + "' cannot be reserved by an instance of class '" +
+ descendant.getClass().getName() + "' since it is already used by an instance of class '" +
+ id2producer.get(configId).getClass().getName() + "'");
+ }
+ id2producer.put(configId, descendant);
+ }
+
+ @Override
+ public void addChild(AbstractConfigProducer abstractConfigProducer) {
+ super.addChild(abstractConfigProducer);
+ }
+
+ public final void setAdmin(String xml) {
+ String servicesXml =
+ "<?xml version='1.0' encoding='utf-8' ?>" +
+ "<services>" + xml + "</services>";
+
+ try {
+ Document doc = XmlHelper.getDocumentBuilder().parse(new InputSource(new StringReader(servicesXml)));
+ setAdmin(new DomAdminV2Builder(deployState.getFileRegistry(), false, new ArrayList<>()).
+ build(this, XML.getChildren(doc.getDocumentElement(), "admin").get(0)));
+ } catch (SAXException | IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public final void setAdmin(Admin admin) {
+ this.admin = admin;
+ }
+
+ @Override
+ public final Admin getAdmin() {
+ return admin;
+ }
+
+ @Override
+ public DeployLogger deployLogger() {
+ return new BaseDeployLogger();
+ }
+
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java b/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java
new file mode 100644
index 00000000000..4518e44d441
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java
@@ -0,0 +1,111 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.test;
+
+import com.google.common.annotations.Beta;
+import com.yahoo.config.model.MapConfigModelRegistry;
+import com.yahoo.config.application.api.ApplicationPackage;
+import com.yahoo.config.model.application.provider.SchemaValidator;
+import com.yahoo.config.model.deploy.DeployState;
+import com.yahoo.config.model.builder.xml.ConfigModelBuilder;
+import com.yahoo.vespa.model.VespaModel;
+import org.xml.sax.SAXException;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Test driver for testing config models. Add custom builders for plugins to be tested. Builds a model from the
+ * xml string and returns a config producer that can be use to test getConfig.
+ *
+ * @author lulf
+ * @since 5.1.20
+ */
+@Beta
+public class TestDriver {
+
+ private final List<ConfigModelBuilder> builders = new ArrayList<>();
+ private final boolean validate;
+
+ public TestDriver(boolean validate) {
+ this.validate = validate;
+ }
+
+ public TestDriver() {
+ this(false);
+ }
+
+ /**
+ * Add a new builder to the tester.
+ *
+ * @param builder builder to add.
+ * @return this for chaining
+ */
+ public TestDriver addBuilder(ConfigModelBuilder builder) {
+ builders.add(builder);
+ return this;
+ }
+
+ /**
+ * Build a model from an XML string. The hosts referenced in services must be set to 'mockhost' when using
+ * this method, as it automatically generates a hosts file for you.
+ *
+ * @param servicesXml The xml for services.xml
+ * @return a producer root capable of answering getConfig requests.
+ */
+ public TestRoot buildModel(String servicesXml) {
+ return buildModel(servicesXml, "<hosts><host name='localhost'><alias>mockhost</alias></host></hosts>");
+ }
+
+ /**
+ * Build a model from an XML string of services and one of hosts.
+ *
+ * @param servicesXml The xml for services.xml
+ * @param hostsXml The xml for hosts.xml
+ * @return a producer root capable of answering getConfig requests.
+ */
+ public TestRoot buildModel(String servicesXml, String hostsXml) {
+ if (!servicesXml.contains("<services")) {
+ servicesXml = "<services version='1.0'>" + servicesXml + "</services>";
+ }
+ return buildModel(new MockApplicationPackage.Builder().withHosts(hostsXml).withServices(servicesXml).build());
+ }
+
+ /**
+ * Build a model from an application package.
+ *
+ * @param applicationPackage Any type of application package.
+ * @return a producer root capable of answering getConfig requests.
+ */
+ public TestRoot buildModel(ApplicationPackage applicationPackage) {
+ return buildModel(new DeployState.Builder().applicationPackage(applicationPackage).build());
+ }
+
+ /**
+ * Build a model given a deploy state.
+ *
+ * @param deployState An instance of {@link com.yahoo.config.model.deploy.DeployState}
+ * @return a producer root capable of answering getConfig requests.
+ */
+ public TestRoot buildModel(DeployState deployState) {
+ MapConfigModelRegistry registry = new MapConfigModelRegistry(builders);
+ try {
+ validate(deployState.getApplicationPackage());
+ return new TestRoot(new VespaModel(registry, deployState));
+ } catch (IOException | SAXException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void validate(ApplicationPackage appPkg) throws IOException {
+ if (!validate) {
+ return;
+ }
+ SchemaValidator validator = SchemaValidator.createTestValidatorHosts();
+ if (appPkg.getHosts() != null) {
+ validator.validate(appPkg.getHosts());
+ }
+ validator = SchemaValidator.createTestValidatorServices();
+ validator.validate(appPkg.getServices());
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java
new file mode 100644
index 00000000000..44491e33479
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java
@@ -0,0 +1,66 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.test;
+
+import com.google.common.annotations.Beta;
+import com.yahoo.config.ConfigInstance;
+import com.yahoo.config.model.ConfigModel;
+import com.yahoo.vespa.model.HostResource;
+import com.yahoo.vespa.model.VespaModel;
+
+import java.util.List;
+
+/**
+ * Test utility class that provides many methods for inspecting the state of a completely built model
+ *
+ * @author lulf
+ * @since 5.1
+ */
+@Beta
+public class TestRoot {
+ private final VespaModel model;
+ TestRoot(VespaModel model) {
+ this.model = model;
+ }
+
+ /**
+ * Get a list of all config models of a particular type.
+ *
+ * @param clazz The class of the models to find.
+ * @return A list of models of given type.
+ */
+ public <MODEL extends ConfigModel> List<MODEL> getConfigModels(Class<MODEL> clazz) {
+ return model.configModelRepo().getModels(clazz);
+ }
+
+ /**
+ * Ask model to populate builder with config for a given config id. This method gives the same config as the
+ * configserver would return to its clients.
+ * @param builder The builder to populate
+ * @param configId The config id of the producer to ask for config.
+ * @return the same builder.
+ */
+ public <BUILDER extends ConfigInstance.Builder> BUILDER getConfig(BUILDER builder, String configId) {
+ return (BUILDER) model.getConfig(builder, configId);
+ }
+
+ /**
+ * Request config of a given type and id. This method gives the same config as the configserver would return to
+ * its clients.
+ *
+ * @param clazz Type of config to request.
+ * @param configId The config id of the producer to ask for config.
+ * @return A config object of the appropriate type with config values set.
+ */
+ public <CONFIGTYPE extends ConfigInstance> CONFIGTYPE getConfig(Class<CONFIGTYPE> clazz, String configId) {
+ return model.getConfig(clazz, configId);
+ }
+
+ /**
+ * Retrieve the hosts available in this model. Useful to verify that hostnames are set correctly etc.
+ *
+ * @return A list of hosts.
+ */
+ public List<HostResource> getHosts() {
+ return model.getHostSystem().getHosts();
+ }
+}
diff --git a/config-model/src/main/java/com/yahoo/config/model/test/package-info.java b/config-model/src/main/java/com/yahoo/config/model/test/package-info.java
new file mode 100644
index 00000000000..6c33505df83
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/config/model/test/package-info.java
@@ -0,0 +1,6 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+//TODO: Temporary export due to standalone container package, remove later.
+@ExportPackage
+package com.yahoo.config.model.test;
+
+import com.yahoo.osgi.annotation.ExportPackage;