diff options
| author | Jon Bratseth <jonbratseth@yahoo.com> | 2016-09-29 23:28:23 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2016-09-29 23:28:23 +0200 |
| commit | 8aeed8886002cc6ad9261da57763766cad5ca7bf (patch) | |
| tree | 814bc1c016c679e67306101e7b409ce7c4d2272b | |
| parent | 151d1cf4c09fc24c39d665a0818c64ff7bf80716 (diff) | |
| parent | 0d27136b2912ce0dd6ca1c6cc5341ebeba71dc12 (diff) | |
Merge pull request #738 from yahoo/bratseth/support-document-api-in-applications
Bratseth/support document api in applications
66 files changed, 506 insertions, 498 deletions
diff --git a/application/src/main/java/com/yahoo/application/Application.java b/application/src/main/java/com/yahoo/application/Application.java index cc1b785ae0b..cfcce72487b 100644 --- a/application/src/main/java/com/yahoo/application/Application.java +++ b/application/src/main/java/com/yahoo/application/Application.java @@ -44,6 +44,13 @@ import java.util.*; @Beta public final class Application implements AutoCloseable { + /** + * This system property is set to "true" upon creation of an Application. + * This is useful for components which are created by dependendy injection which needs to modify + * their behavior to function without reliance on any processes outside the JVM. + */ + public static final String vespaLocalProperty = "vespa.local"; + private final JDisc container; private final List<ContentCluster> contentClusters; private final Path path; @@ -51,6 +58,7 @@ public final class Application implements AutoCloseable { // For internal use only Application(Path path, Networking networking, boolean deletePathWhenClosing) { + System.setProperty(vespaLocalProperty, "true"); this.path = path; this.deletePathWhenClosing = deletePathWhenClosing; contentClusters = ContentCluster.fromPath(path); diff --git a/application/src/main/java/com/yahoo/application/container/JDisc.java b/application/src/main/java/com/yahoo/application/container/JDisc.java index 0c6caf9fdf9..dba16a0e3fe 100644 --- a/application/src/main/java/com/yahoo/application/container/JDisc.java +++ b/application/src/main/java/com/yahoo/application/container/JDisc.java @@ -23,6 +23,7 @@ import com.yahoo.jdisc.test.TestDriver; import com.yahoo.processing.handler.ProcessingHandler; import com.yahoo.search.handler.SearchHandler; +import java.io.File; import java.nio.file.Path; /** @@ -97,7 +98,7 @@ public final class JDisc implements AutoCloseable { * @param networking enabled or disabled * @return a new JDisc instance */ - public static JDisc fromPath(final Path path, Networking networking) { + public static JDisc fromPath(Path path, Networking networking) { return new JDisc(path, false, networking, new ConfigModelRepo()); } @@ -105,7 +106,7 @@ public final class JDisc implements AutoCloseable { * Create a jDisc instance which is given a config model repo (in which (mock) content clusters * can be looked up). */ - public static JDisc fromPath(final Path path, Networking networking, ConfigModelRepo configModelRepo) { + public static JDisc fromPath(Path path, Networking networking, ConfigModelRepo configModelRepo) { return new JDisc(path, false, networking, configModelRepo); } diff --git a/application/src/test/java/com/yahoo/application/ApplicationTest.java b/application/src/test/java/com/yahoo/application/ApplicationTest.java index 6f4e6103743..7b515cb843b 100644 --- a/application/src/test/java/com/yahoo/application/ApplicationTest.java +++ b/application/src/test/java/com/yahoo/application/ApplicationTest.java @@ -28,6 +28,7 @@ import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.DefaultHttpClient; +import org.junit.Ignore; import org.junit.Test; import java.io.BufferedReader; @@ -363,7 +364,7 @@ public class ApplicationTest { assertEquals(200, statusCode); } } - + private static int getFreePort() throws IOException { try (ServerSocket socket = new ServerSocket(0)) { socket.setReuseAddress(true); diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java index 334fda6e6eb..ce63ad23852 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java @@ -35,8 +35,4 @@ public class MockFileRegistry implements FileRegistry { return result; } - @Override - public Set<String> allRelativePaths() { - return Collections.emptySet(); - } } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java index 67a24e0159b..ed4ccf51ff7 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java @@ -83,11 +83,6 @@ public class PreGeneratedFileRegistry implements FileRegistry { } @Override - public Set<String> allRelativePaths() { - return path2Hash.keySet(); - } - - @Override public List<Entry> export() { List<Entry> entries = new ArrayList<>(); for (Map.Entry<String, String> entry : path2Hash.entrySet()) { diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ComponentInfo.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ComponentInfo.java index 42732ddfc47..52aaa148c1b 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ComponentInfo.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ComponentInfo.java @@ -8,6 +8,7 @@ package com.yahoo.config.application.api; * @author tonytv */ public class ComponentInfo { + final String pathRelativeToAppDir; public ComponentInfo(String pathRelativeToAppDir) { @@ -18,4 +19,8 @@ public class ComponentInfo { public String getPathRelativeToAppDir() { return pathRelativeToAppDir; } + + @Override + public String toString() { return "component at '" + pathRelativeToAppDir + "'"; } + } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java b/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java index fe4aab72cb0..8b211d1d400 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java @@ -18,8 +18,6 @@ public interface FileRegistry { */ String fileSourceHost(); - Set<String> allRelativePaths(); - List<Entry> export(); class Entry { diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigServerSpec.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigServerSpec.java index 0a43f190675..499c43906e2 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigServerSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ConfigServerSpec.java @@ -7,8 +7,10 @@ package com.yahoo.config.model.api; * @author tonytv */ public interface ConfigServerSpec { - public String getHostName(); - public int getConfigServerPort(); - public int getHttpPort(); - public int getZooKeeperPort(); + + String getHostName(); + int getConfigServerPort(); + int getHttpPort(); + int getZooKeeperPort(); + } diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java index e6df94c8855..0b0ac77443c 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -265,6 +265,7 @@ public class ApplicationConfigProducerRoot extends AbstractConfigProducer<Abstra } public FileDistributionConfigProducer getFileDistributionConfigProducer() { + if (admin == null) return null; // no admin if standalone return admin.getFileDistributionConfigProducer(); } diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java index 8e1097907f1..8778107cd8a 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java @@ -105,10 +105,10 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce child.setParent(this); if (childrenBySubId.get(child.getSubId()) != null) { throw new IllegalArgumentException("Multiple services/instances of the id '" + child.getSubId() + "' under the service/instance " + - errorMsgClassName() + " '" + subId + "'. (This is commonly caused by service/node index " + - "collisions in the config.)." + - "\nExisting instance: " + childrenBySubId.get(child.getSubId()) + - "\nAttempted to add: " + child); + errorMsgClassName() + " '" + subId + "'. (This is commonly caused by service/node index " + + "collisions in the config.)." + + "\nExisting instance: " + childrenBySubId.get(child.getSubId()) + + "\nAttempted to add: " + child); } childrenBySubId.put(child.getSubId(), child); diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java index 1d5544873d9..fe8b3935fcf 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java @@ -27,10 +27,11 @@ public class SingleNodeProvisioner implements HostProvisioner { public SingleNodeProvisioner() { try { host = new Host(HostSystem.lookupCanonicalHostname(HostName.getLocalhost())); - } catch (UnknownHostException e) { + this.hostSpec = new HostSpec(host.hostname(), host.aliases()); + } + catch (UnknownHostException e) { throw new RuntimeException(e); } - this.hostSpec = new HostSpec(host.hostname(), host.aliases()); } @Override diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java index fa84cf1c7eb..314060e7543 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java @@ -35,7 +35,9 @@ import java.util.Set; * * @author gjoranv */ +// TODO: mockRoot instances can probably be replaced by VespaModel.createIncomplete public class MockRoot extends AbstractConfigProducerRoot { + private static final long serialVersionUID = 1L; public static final String MOCKHOST = "mockhost"; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index d3e922c69dc..2d825e3332d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -5,12 +5,21 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.test.MockRoot; -import com.yahoo.config.provision.*; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.net.HostName; import java.net.InetAddress; import java.net.UnknownHostException; -import java.util.*; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.logging.Level; import java.util.stream.Collectors; @@ -73,6 +82,7 @@ public class HostSystem extends AbstractConfigProducer<Host> { * @return The canonical hostname, or null if unable to resolve. * @throws UnknownHostException if the hostname cannot be resolved */ + // public - This is used by amenders outside this repo public static String lookupCanonicalHostname(String hostname) throws UnknownHostException { return java.net.InetAddress.getByName(hostname).getCanonicalHostName(); } @@ -87,7 +97,7 @@ public class HostSystem extends AbstractConfigProducer<Host> { if (ipAddresses.containsKey(hostname)) return ipAddresses.get(hostname); String ipAddress; - if (hostname.startsWith(MockRoot.MOCKHOST)) { + if (hostname.startsWith(MockRoot.MOCKHOST)) { // TODO: Remove ipAddress = "0.0.0.0"; } else { try { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 9a23be1f5c5..03971df7cd7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -100,6 +100,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri /** The validation overrides of this. This is never null. */ private final ValidationOverrides validationOverrides; + + private final FileDistributor fileDistributor; /** Creates a Vespa Model from internal model types only */ public VespaModel(ApplicationPackage app) throws IOException, SAXException { @@ -130,23 +132,37 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri * @param deployState the global deploy state to use for this model. */ public VespaModel(ConfigModelRegistry configModelRegistry, DeployState deployState) throws IOException, SAXException { + this(configModelRegistry, deployState, true, null); + } + + private VespaModel(ConfigModelRegistry configModelRegistry, DeployState deployState, boolean complete, FileDistributor fileDistributor) throws IOException, SAXException { super("vespamodel"); this.deployState = deployState; this.validationOverrides = deployState.validationOverrides(); configModelRegistry = new VespaConfigModelRegistry(configModelRegistry); VespaModelBuilder builder = new VespaDomBuilder(); root = builder.getRoot(VespaModel.ROOT_CONFIGID, deployState, this); - configModelRepo.readConfigModels(deployState, builder, root, configModelRegistry); - addServiceClusters(deployState.getApplicationPackage(), builder); - setupRouting(); - log.log(LogLevel.DEBUG, "hostsystem=" + getHostSystem()); this.info = Optional.of(createProvisionInfo()); - getAdmin().addPerHostServices(getHostSystem().getHosts(), deployState.getProperties()); - freezeModelTopology(); - root.prepare(configModelRepo); - configModelRepo.prepareConfigModels(); - validateWrapExceptions(); - this.deployState = null; + if (complete) { // create a a completed, frozen model + configModelRepo.readConfigModels(deployState, builder, root, configModelRegistry); + addServiceClusters(deployState.getApplicationPackage(), builder); + setupRouting(); + this.fileDistributor = root.getFileDistributionConfigProducer().getFileDistributor(); + getAdmin().addPerHostServices(getHostSystem().getHosts(), deployState.getProperties()); + freezeModelTopology(); + root.prepare(configModelRepo); + configModelRepo.prepareConfigModels(); + validateWrapExceptions(); + this.deployState = null; + } + else { // create a model with no services instantiated and the given file distributor + this.fileDistributor = fileDistributor; + } + } + + /** Creates a mutable model with no services instantiated */ + public static VespaModel createIncomplete(DeployState deployState) throws IOException, SAXException { + return new VespaModel(new NullConfigModelRegistry(), deployState, false, new FileDistributor(deployState.getFileRegistry())); } private ProvisionInfo createProvisionInfo() { @@ -192,7 +208,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri } public FileDistributor getFileDistributor() { - return root.getFileDistributionConfigProducer().getFileDistributor(); + // return root.getFileDistributionConfigProducer().getFileDistributor(); + return fileDistributor; } /** Returns this models Vespa instance */ @@ -437,9 +454,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri @Override public DeployState getDeployState() { - if (deployState == null) { + if (deployState == null) throw new IllegalStateException("Cannot call getDeployState() once model has been built"); - } return deployState; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java index 38a1e59433f..6949c5adf25 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Admin.java @@ -200,7 +200,7 @@ public class Admin extends AbstractConfigProducer implements Serializable { HostResource deployHost = getHostSystem().getHostByHostname(fileDistributor.fileSourceHost()); if (deployHostIsMissing(deployHost)) { throw new RuntimeException("Could not find host in the application's host system: '" + - fileDistributor.fileSourceHost() + "'. Hostsystem=" + getHostSystem()); + fileDistributor.fileSourceHost() + "'. Hostsystem=" + getHostSystem()); } FileDistributorService fds = new FileDistributorService(fileDistribution, host.getHost().getHostName(), diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java index 11508ba91ed..47332b064da 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/Configserver.java @@ -96,10 +96,12 @@ public class Configserver extends AbstractService { // TODO: Remove this implementation when we are on Hosted Vespa. public static class Spec implements ConfigServerSpec { + private final String hostName; private final int configServerPort; private final int httpPort; private final int zooKeeperPort; + public String getHostName() { return hostName; } @@ -142,4 +144,5 @@ public class Configserver extends AbstractService { this.zooKeeperPort = zooKeeperPort; } } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java index cea325b785f..ff37c0a95fd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomV20ClientsBuilder.java @@ -54,29 +54,17 @@ import java.util.logging.Level; */ public class DomV20ClientsBuilder { - public static final String vespaClientBundleSpecification = "vespaclient-container-plugin"; - // The parent docproc plugin to register data with. private final Clients clients; DomV20ClientsBuilder(Clients clients, String version) { - this.clients = clients; - if (!version.equals("2.0")) { + if ( ! version.equals("2.0")) throw new IllegalArgumentException("Version '" + version + "' of 'clients' not supported."); - } + this.clients = clients; } public void build(Element spec) { - NodeList children = spec.getElementsByTagName("gateways"); - if (children.getLength() > 0 && clients.getConfigProducer()!=null) - clients.getConfigProducer().deployLogger().log(Level.WARNING, "The 'gateways' element is deprecated, and will be disallowed in a " + - "later version of Vespa. Use 'document-api' under 'jdisc' instead, see: " + - ConfigModelUtils.createDocLink("reference/services-jdisc.html")); - for (int i = 0; i < children.getLength(); i++) { - createGateways(clients.getConfigProducer(), (Element) children.item(i), clients); - } - - children = spec.getElementsByTagName("spoolers"); + NodeList children = spec.getElementsByTagName("spoolers"); for (int i = 0; i < children.getLength(); i++) { createSpoolers(clients.getConfigProducer(), (Element) children.item(i), clients); } @@ -87,29 +75,6 @@ public class DomV20ClientsBuilder { } } - static Boolean getBooleanNodeValue(Node node) { - return Boolean.valueOf(node.getFirstChild().getNodeValue()); - } - - static boolean getHttpFileServerEnabled(Element parentHttpFileServer, Element httpFileServer) { - boolean ret=false; - if (parentHttpFileServer != null) { - for (Element child : XML.getChildren(parentHttpFileServer)) { - if ("enabled".equals(child.getNodeName())) { - ret = getBooleanNodeValue(child); - } - } - } - if (httpFileServer != null) { - for (Element child : XML.getChildren(httpFileServer)) { - if ("enabled".equals(child.getNodeName())) { - ret = getBooleanNodeValue(child); - } - } - } - return ret; - } - private void createLoadTypes(Element element, Clients clients) { for (Element e : XML.getChildren(element, "type")) { String priority = e.getAttribute("default-priority"); @@ -118,31 +83,6 @@ public class DomV20ClientsBuilder { } /** - * Creates HttpGateway objects using the given xml Element. - * - * @param pcp AbstractConfigProducer - * @param element The xml Element - */ - private void createGateways(AbstractConfigProducer pcp, Element element, Clients clients) { - String jvmArgs = null; - if (element.hasAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME)) jvmArgs=element.getAttribute(VespaDomBuilder.JVMARGS_ATTRIB_NAME); - - Element gatewaysFeederOptions = findFeederOptions(element); - - HttpGatewayOwner owner = new HttpGatewayOwner(pcp, getFeederConfig(null, gatewaysFeederOptions)); - ContainerCluster cluster = new ContainerHttpGatewayClusterBuilder().build(owner, element); - - int index = 0; - for (Element e : XML.getChildren(element, "gateway")) { - ContainerHttpGateway qrs = new ContainerHttpGatewayBuilder(cluster, index).build(cluster, e); - - if ("".equals(qrs.getJvmArgs()) && jvmArgs!=null) qrs.setJvmArgs(jvmArgs); - index++; - } - clients.setContainerHttpGateways(cluster); - } - - /** * Creates VespaSpooler objects using the given xml Element. */ private void createSpoolers(AbstractConfigProducer pcp, Element element, Clients clients) { @@ -170,13 +110,10 @@ public class DomV20ClientsBuilder { } } - private void createSpoolMasters(SimpleConfigProducer producer, - Element element) { + private void createSpoolMasters(SimpleConfigProducer producer, Element element) { int i=0; - for (Element e : XML.getChildren(element, "spoolmaster")) { - VespaSpoolMaster master = new VespaSpoolMasterBuilder(i).build(producer, e); - i++; - } + for (Element e : XML.getChildren(element, "spoolmaster")) + new VespaSpoolMasterBuilder(i++).build(producer, e); } private SpoolerConfig.Builder getSpoolConfig(Element conf) { @@ -313,133 +250,6 @@ public class DomV20ClientsBuilder { } } - public static class ContainerHttpGatewayClusterBuilder extends DomConfigProducerBuilder<ContainerCluster> { - @Override - protected ContainerCluster doBuild(AbstractConfigProducer parent, - Element spec) { - - ContainerCluster cluster = new ContainerCluster(parent, "gateway", "gateway"); - - SearchChains searchChains = new SearchChains(cluster, "searchchain"); - Set<ComponentSpecification> inherited = new TreeSet<>(); - //inherited.add(new ComponentSpecification("vespa", null, null)); - { - SearchChain mySearchChain = new SearchChain(new ChainSpecification(new ComponentId("vespaget"), - new ChainSpecification.Inheritance(inherited, null), new ArrayList<>(), new TreeSet<>())); - Searcher getComponent = newVespaClientSearcher("com.yahoo.storage.searcher.GetSearcher"); - mySearchChain.addInnerComponent(getComponent); - searchChains.add(mySearchChain); - } - { - SearchChain mySearchChain = new SearchChain(new ChainSpecification(new ComponentId("vespavisit"), - new ChainSpecification.Inheritance(inherited, null), new ArrayList<>(), new TreeSet<>())); - Searcher getComponent = newVespaClientSearcher("com.yahoo.storage.searcher.VisitSearcher"); - mySearchChain.addInnerComponent(getComponent); - searchChains.add(mySearchChain); - } - - ContainerSearch containerSearch = new ContainerSearch(cluster, searchChains, new ContainerSearch.Options()); - cluster.setSearch(containerSearch); - - cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandler", "http://*/feed")); - cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerRemove", "http://*/remove")); - cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerRemoveLocation", "http://*/removelocation")); - cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerGet", "http://*/get")); - cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerVisit", "http://*/visit")); - cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerCompatibility", "http://*/document")); - cluster.addComponent(newVespaClientHandler("com.yahoo.feedhandler.VespaFeedHandlerStatus", "http://*/feedstatus")); - final ProcessingHandler<SearchChains> searchHandler = new ProcessingHandler<>( - cluster.getSearch().getChains(), "com.yahoo.search.handler.SearchHandler"); - searchHandler.addServerBindings("http://*/search/*"); - cluster.addComponent(searchHandler); - - ContainerModelBuilder.addDefaultHandler_legacyBuilder(cluster); - - //BEGIN HACK for docproc chains: - DocprocChains docprocChains = getDocprocChains(cluster, spec); - if (docprocChains != null) { - ContainerDocproc containerDocproc = new ContainerDocproc(cluster, docprocChains); - cluster.setDocproc(containerDocproc); - } - //END HACK - - return cluster; - } - - private Handler newVespaClientHandler(String componentId, String binding) { - Handler<AbstractConfigProducer<?>> handler = new Handler<>(new ComponentModel( - BundleInstantiationSpecification.getFromStrings(componentId, null, vespaClientBundleSpecification), "")); - handler.addServerBindings(binding); - handler.addServerBindings(binding + '/'); - return handler; - } - - private Searcher newVespaClientSearcher(String componentSpec) { - return new Searcher<>(new ChainedComponentModel( - BundleInstantiationSpecification.getFromStrings(componentSpec, null, vespaClientBundleSpecification), - new Dependencies(null, null, null))); - } - - //BEGIN HACK for docproc chains: - private DocprocChains getDocprocChains(AbstractConfigProducer qrs, Element gateways) { - Element clients = (Element) gateways.getParentNode(); - Element services = (Element) clients.getParentNode(); - if (services == null) { - return null; - } - - Element docproc = XML.getChild(services, "docproc"); - if (docproc == null) { - return null; - } - - String version = docproc.getAttribute("version"); - if (version.startsWith("1.")) { - return null; - } else if (version.startsWith("2.")) { - return null; - } else if (version.startsWith("3.")) { - return getDocprocChainsV3(qrs, docproc); - } else { - throw new IllegalArgumentException("Docproc version " + version + " unknown."); - } - } - - private DocprocChains getDocprocChainsV3(AbstractConfigProducer qrs, Element docproc) { - Element docprocChainsElem = XML.getChild(docproc, "docprocchains"); - if (docprocChainsElem == null) { - return null; - } - return new DomDocprocChainsBuilder(null, true).build(qrs, docprocChainsElem); - } - //END HACK - } - - public static class ContainerHttpGatewayBuilder extends DomConfigProducerBuilder<ContainerHttpGateway> { - int index; - ContainerCluster cluster; - - public ContainerHttpGatewayBuilder(ContainerCluster cluster, int index) { - this.index = index; - this.cluster = cluster; - } - - @Override - protected ContainerHttpGateway doBuild(AbstractConfigProducer parent, Element spec) { - // TODO: remove port handling - int port = 19020; - if (spec != null && spec.hasAttribute("baseport")) { - port = Integer.parseInt(spec.getAttribute("baseport")); - } - ContainerHttpGateway httpGateway = new ContainerHttpGateway(cluster, "" + index, port, index); - List<Container> containers = new ArrayList<>(); - containers.add(httpGateway); - - cluster.addContainers(containers); - return httpGateway; - } - } - /** * This class parses the feederoptions xml tag and produces Vespa config output. * @@ -553,4 +363,5 @@ public class DomV20ClientsBuilder { return builder; } } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java index 55cfc8b2fba..c1ad6eead47 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/clients/ContainerDocumentApi.java @@ -26,7 +26,7 @@ import java.util.Set; import java.util.TreeSet; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge * @since 5.1.11 */ public class ContainerDocumentApi implements FeederConfig.Producer { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index e92bffe2542..1351933fbc8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -16,6 +16,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.NodeType; import com.yahoo.container.jdisc.config.MetricDefaultsConfig; +import com.yahoo.path.Path; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.text.XML; import com.yahoo.vespa.defaults.Defaults; @@ -104,7 +105,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { checkVersion(spec); this.log = modelContext.getDeployLogger(); - ContainerCluster cluster = createContainerCluster(spec, modelContext); addClusterContent(cluster, spec, modelContext); addBundlesForPlatformComponents(cluster); @@ -587,9 +587,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private ContainerDocumentApi buildDocumentApi(ContainerCluster cluster, Element spec) { Element documentApiElement = XML.getChild(spec, "document-api"); - if (documentApiElement == null) { - return null; - } + if (documentApiElement == null) return null; ContainerDocumentApi.Options documentApiOptions = DocumentApiOptionsBuilder.build(documentApiElement); return new ContainerDocumentApi(cluster, documentApiOptions); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java index df2090db166..b1ffd55b0f0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/DocumentApiOptionsBuilder.java @@ -12,10 +12,11 @@ import java.util.List; import java.util.logging.Logger; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge * @since 5.1.11 */ public class DocumentApiOptionsBuilder { + private static final Logger log = Logger.getLogger(DocumentApiOptionsBuilder.class.getName()); private static final String[] DEFAULT_BINDINGS = {"http://*/", "https://*/"}; @@ -116,4 +117,5 @@ public class DocumentApiOptionsBuilder { String value = getCleanValue(spec, "abortondocumenterror"); return value == null ? null : Boolean.parseBoolean(value); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java index 630118cc60c..095a5e29450 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributionConfigProducer.java @@ -13,6 +13,7 @@ import java.util.Map; * @author tonytv */ public class FileDistributionConfigProducer extends AbstractConfigProducer { + private final Map<Host, FileDistributorService> fileDistributorServices = new IdentityHashMap<>(); private final FileDistributor fileDistributor; private final FileDistributionOptions options; @@ -56,4 +57,5 @@ public class FileDistributionConfigProducer extends AbstractConfigProducer { return new FileDistributionConfigProducer(ancestor, fileDistributor, options); } } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java index 4dc24618a61..df7b4f58ab5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java @@ -117,4 +117,5 @@ public class FileDistributor { result.addAll(asList(additionalHosts)); return result; } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java index 4ab9ed3af85..84685ecef3d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java @@ -32,8 +32,8 @@ public class FileSender implements Serializable { public static FileReference sendFileToServices(String relativePath, Collection<? extends AbstractService> services) { if (services.isEmpty()) { - throw new IllegalStateException("'sendFileToServices called for empty services!" + - " - This should never happen!"); + throw new IllegalStateException("No service instances. Probably a standalone cluster setting up <nodes> " + + "using 'count' instead of <node> tags."); } FileReference fileref = null; for (AbstractService service : services) { @@ -146,4 +146,5 @@ public class FileSender implements Serializable { } builder.setValue(reference.value()); } + }
\ No newline at end of file diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/FreezableMap.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/FreezableMap.java index a05008cc9a0..211413f9bff 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/utils/FreezableMap.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/FreezableMap.java @@ -4,8 +4,9 @@ package com.yahoo.vespa.model.utils; import java.util.*; /** - * Delegates to a map that can be froozen. + * Delegates to a map that can be frozen. * Not thread safe. + * * @author tonytv */ public class FreezableMap<K, V> implements Map<K, V> { @@ -88,4 +89,5 @@ public class FreezableMap<K, V> implements Map<K, V> { public boolean isFrozen() { return frozen; } + } diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 8f1ed0d874e..a5679151a03 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -161,7 +161,7 @@ ProcessingInContainer = element processing { -# DOCUMENT API/GATEWAY: +# DOCUMENT API: DocumentApi = element document-api { ServerBindings & diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java index 2322726057e..619dd333b51 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java @@ -27,6 +27,7 @@ import com.yahoo.vespa.config.TimingValues; * @since 5.1 */ public class ConfigSubscriber { + private Logger log = Logger.getLogger(getClass().getName()); private State state = State.OPEN; protected List<ConfigHandle<? extends ConfigInstance>> subscriptionHandles = new ArrayList<>(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java index 58d651ae33a..37cea22e420 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java @@ -43,12 +43,4 @@ public class FileDBRegistry implements FileRegistry { return entries; } - @Override - public Set<String> allRelativePaths() { - Set<String> ret = new HashSet<>(); - for (Entry entry : entries) { - ret.add(entry.relativePath); - } - return ret; - } } diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/SortingDegrader.java b/container-search/src/main/java/com/yahoo/search/querytransform/SortingDegrader.java index 0c7871eb6e6..2597e440d17 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/SortingDegrader.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/SortingDegrader.java @@ -69,6 +69,8 @@ public class SortingDegrader extends Searcher { } private void setDegradation(Query query) { + query.trace("Using sorting degrading for performance - totalHits will be wrong. " + + "Turn off with sorting.degrading=false.", 2); Sorting.FieldOrder primarySort = query.getRanking().getSorting().fieldOrders().get(0); // ensured above MatchPhase matchPhase = query.getRanking().getMatchPhase(); diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index a575fbfba2a..9d7139e6226 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -11,7 +11,7 @@ import java.util.ArrayList; import java.util.logging.Logger; /** - * Configures the Vepsa document manager from a document id. + * Configures the Vespa document manager from a config id. * * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> */ diff --git a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java index 53e7ed701a7..b3deae547ab 100644 --- a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java +++ b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java @@ -26,6 +26,7 @@ import static com.yahoo.document.json.JsonTestHelper.inputJson; * @author Vegard Sjonfjell */ public class DocumentUpdateJsonSerializerTest { + final static DocumentTypeManager types = new DocumentTypeManager(); final static JsonFactory parserFactory = new JsonFactory(); final static DocumentType docType = new DocumentType("doctype"); diff --git a/document/src/test/java/com/yahoo/document/json/JsonTestHelper.java b/document/src/test/java/com/yahoo/document/json/JsonTestHelper.java index 45128e6d02a..09e6a74e68a 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonTestHelper.java +++ b/document/src/test/java/com/yahoo/document/json/JsonTestHelper.java @@ -1,5 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.json; + import com.google.common.base.Joiner; import static org.hamcrest.MatcherAssert.assertThat; import static uk.co.datumedge.hamcrest.json.SameJSONAs.sameJSONAs; @@ -8,6 +9,7 @@ import static uk.co.datumedge.hamcrest.json.SameJSONAs.sameJSONAs; * @author Vegard Sjonfjell */ public class JsonTestHelper { + /** * Convenience method to input JSON without escaping double quotes and newlines * Each parameter represents a line of JSON encoded data @@ -23,4 +25,5 @@ public class JsonTestHelper { public static void assertJsonEquals(String inputJson, String expectedJson) { assertThat(inputJson, sameJSONAs(expectedJson)); } + } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java index 0d781e4ca95..bad692f0a0d 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccess.java @@ -41,13 +41,13 @@ import com.yahoo.config.subscription.ConfigSubscriber; * <p>Access to this class is thread-safe.</p> * * @author bratseth - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar Rosenvinge</a> - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Einar Rosenvinge + * @author Simon Thoresen */ public abstract class DocumentAccess { - protected DocumentTypeManager documentMgr; - protected ConfigSubscriber documentTypeManagerConfig; + private final DocumentTypeManager documentTypeManager; + private final ConfigSubscriber documentTypeConfigSubscriber; /** * <p>This is a convenience method to return a document access object with @@ -69,8 +69,14 @@ public abstract class DocumentAccess { */ protected DocumentAccess(DocumentAccessParams params) { super(); - documentMgr = new DocumentTypeManager(); - documentTypeManagerConfig = DocumentTypeManagerConfigurer.configure(documentMgr, params.getDocumentManagerConfigId()); + if (params.documentmanagerConfig().isPresent()) { // our config has been injected into the creator + documentTypeManager = new DocumentTypeManager(params.documentmanagerConfig().get()); + documentTypeConfigSubscriber = null; + } + else { // fallback to old style subscription + documentTypeManager = new DocumentTypeManager(); + documentTypeConfigSubscriber = DocumentTypeManagerConfigurer.configure(documentTypeManager, params.getDocumentManagerConfigId()); + } } /** @@ -154,11 +160,15 @@ public abstract class DocumentAccess { public abstract SubscriptionSession openSubscription(SubscriptionParameters parameters); /** - * <p>Shuts down the underlying sessions used by this DocumentAccess; + * Shuts down the underlying sessions used by this DocumentAccess; * subsequent use of this DocumentAccess will throw unspecified exceptions, - * depending on implementation.</p> + * depending on implementation. + * Classes overriding this must call super.shutdown(). */ - public abstract void shutdown(); + public void shutdown() { + if (documentTypeConfigSubscriber != null) + documentTypeConfigSubscriber.close(); + } /** * <p>Returns the {@link DocumentTypeManager} used by this @@ -167,6 +177,6 @@ public abstract class DocumentAccess { * @return The document type manager. */ public DocumentTypeManager getDocumentTypeManager() { - return documentMgr; + return documentTypeManager; } } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccessParams.java b/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccessParams.java index 57cfdbd32e1..701fafbab06 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccessParams.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/DocumentAccessParams.java @@ -1,6 +1,10 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.documentapi;
+import com.yahoo.document.config.DocumentmanagerConfig;
+
+import java.util.Optional;
+
/**
* Superclass of the classes which contains the parameters for creating or opening a document access.
*
@@ -8,26 +12,27 @@ package com.yahoo.documentapi; */
public class DocumentAccessParams {
- // The id to resolve to document manager config.
+ /** The id to resolve to document manager config. Not needed if the config is passed here */
private String documentManagerConfigId = "client";
- /**
- * Returns the config id that the document manager should subscribe to.
- *
- * @return The config id.
- */
- public String getDocumentManagerConfigId() {
- return documentManagerConfigId;
- }
+ /** The document manager config, or empty if not provided (in which case a subscription must be created) */
+ private Optional<DocumentmanagerConfig> documentmanagerConfig = Optional.empty();
+
+ /** Returns the config id that the document manager should subscribe to. */
+ public String getDocumentManagerConfigId() { return documentManagerConfigId; }
+
+ /** Returns the document manager config to use, or empty if it it necessary to subscribe to get it */
+ public Optional<DocumentmanagerConfig> documentmanagerConfig() { return documentmanagerConfig; }
- /**
- * Sets the config id that the document manager should subscribe to.
- *
- * @param configId The config id.
- * @return This, to allow chaining.
- */
+ /** Sets the config id that the document manager should subscribe to. */
public DocumentAccessParams setDocumentManagerConfigId(String configId) {
documentManagerConfigId = configId;
return this;
}
+
+ public DocumentAccessParams setDocumentmanagerConfig(DocumentmanagerConfig documentmanagerConfig) {
+ this.documentmanagerConfig = Optional.of(documentmanagerConfig);
+ return this;
+ }
+
}
\ No newline at end of file diff --git a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalDocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalDocumentAccess.java index edcefe9447d..ab1b5e7cdd6 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/local/LocalDocumentAccess.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/local/LocalDocumentAccess.java @@ -22,13 +22,6 @@ public class LocalDocumentAccess extends DocumentAccess { } @Override - public void shutdown() { - if (documentTypeManagerConfig != null) { - documentTypeManagerConfig.close(); - } - } - - @Override public SyncSession createSyncSession(SyncParameters parameters) { return new LocalSyncSession(this); } @@ -57,4 +50,5 @@ public class LocalDocumentAccess extends DocumentAccess { public SubscriptionSession openSubscription(SubscriptionParameters parameters) { throw new UnsupportedOperationException("Not supported yet"); } + } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java index 818bc204784..0a57a700276 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java @@ -7,8 +7,11 @@ import com.yahoo.document.select.parser.ParseException; import com.yahoo.documentapi.*; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.messagebus.MessageBus; +import com.yahoo.messagebus.NetworkMessageBus; import com.yahoo.messagebus.RPCMessageBus; import com.yahoo.messagebus.network.Network; +import com.yahoo.messagebus.network.local.LocalNetwork; +import com.yahoo.messagebus.network.local.LocalWire; import com.yahoo.messagebus.routing.RoutingTable; import java.util.concurrent.Executors; @@ -17,16 +20,18 @@ import java.util.concurrent.ScheduledExecutorService; /** * This class implements the {@link DocumentAccess} interface using message bus for communication. * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar Rosenvinge</a> + * @author Einar Rosenvinge * @author bratseth */ public class MessageBusDocumentAccess extends DocumentAccess { - private final RPCMessageBus bus; + private final NetworkMessageBus bus; + private final MessageBusParams params; // TODO: make pool size configurable? ScheduledExecutorService is not dynamic - private final ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool( - Runtime.getRuntime().availableProcessors(), ThreadFactoryFactory.getDaemonThreadFactory("mbus.access.scheduler")); + private final ScheduledExecutorService scheduledExecutorService = + Executors.newScheduledThreadPool(Runtime.getRuntime().availableProcessors(), + ThreadFactoryFactory.getDaemonThreadFactory("mbus.access.scheduler")); /** * Creates a new document access using default values for all parameters. @@ -46,32 +51,38 @@ public class MessageBusDocumentAccess extends DocumentAccess { try { com.yahoo.messagebus.MessageBusParams mbusParams = new com.yahoo.messagebus.MessageBusParams(params.getMessageBusParams()); mbusParams.addProtocol(new DocumentProtocol(getDocumentTypeManager(), params.getProtocolConfigId(), params.getLoadTypes())); - bus = new RPCMessageBus(mbusParams, - params.getRPCNetworkParams(), - params.getRoutingConfigId()); + if (System.getProperty("vespa.local", "false").equals("true")) { // set by Application when running locally + LocalNetwork network = new LocalNetwork(); + bus = new NetworkMessageBus(network, new MessageBus(network, mbusParams)); + } + else { + bus = new RPCMessageBus(mbusParams, params.getRPCNetworkParams(), params.getRoutingConfigId()); + } } catch (Exception e) { throw new DocumentAccessException(e); } } + + private MessageBus messageBus() { + return bus.getMessageBus(); + } @Override public void shutdown() { + super.shutdown(); bus.destroy(); - if (documentTypeManagerConfig != null) { - documentTypeManagerConfig.close(); - } scheduledExecutorService.shutdownNow(); } @Override public MessageBusSyncSession createSyncSession(SyncParameters parameters) { - return new MessageBusSyncSession(parameters, bus.getMessageBus(), this.params); + return new MessageBusSyncSession(parameters, messageBus(), this.params); } @Override public MessageBusAsyncSession createAsyncSession(AsyncParameters parameters) { - return new MessageBusAsyncSession(parameters, bus.getMessageBus(), this.params); + return new MessageBusAsyncSession(parameters, messageBus(), this.params); } @Override @@ -101,34 +112,20 @@ public class MessageBusDocumentAccess extends DocumentAccess { throw new UnsupportedOperationException("Subscriptions not supported."); } - /** - * Returns the internal message bus object so that clients can use it directly. - * - * @return The internal message bus. - */ - public MessageBus getMessageBus() { - return bus.getMessageBus(); - } + /** Returns the internal message bus object so that clients can use it directly. */ + public MessageBus getMessageBus() { return messageBus(); } /** * Returns the network layer of the internal message bus object so that clients can use it directly. This may seem * abit arbitrary, but the fact is that the RPCNetwork actually implements the IMirror API as well as exposing the * SystemState object. - * - * @return The network layer. */ - public Network getNetwork() { - return bus.getRPCNetwork(); - } + public Network getNetwork() { return bus.getNetwork(); } /** * Returns the parameter object that controls the underlying message bus. Changes to these parameters do not affect * previously created sessions. - * - * @return The parameter object. */ - public MessageBusParams getParams() { - return params; - } + public MessageBusParams getParams() { return params; } } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java index cb453559ab1..b9129bf3b85 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java @@ -23,6 +23,7 @@ import java.util.TreeMap; * load types yourself with addType(). */ public class LoadTypeSet { + class DualMap { Map<String, LoadType> nameMap = new TreeMap<String, LoadType>(); Map<Integer, LoadType> idMap = new HashMap<Integer, LoadType>(); @@ -49,6 +50,10 @@ public class LoadTypeSet { configure(new ConfigGetter<>(LoadTypeConfig.class).getConfig(configId)); } + public LoadTypeSet(LoadTypeConfig loadTypeConfig) { + configure(loadTypeConfig); + } + public Map<String, LoadType> getNameMap() { return map.nameMap; } diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/OsgiFramework.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/OsgiFramework.java index 615b36fef1f..78b97caf57b 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/OsgiFramework.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/OsgiFramework.java @@ -38,7 +38,7 @@ public interface OsgiFramework { * or the caller does not have the appropriate permissions, or the system {@link * BundleContext} is no longer valid. */ - public List<Bundle> installBundle(String bundleLocation) throws BundleException; + List<Bundle> installBundle(String bundleLocation) throws BundleException; /** * <p>Starts the given {@link Bundle}s. The parameter <tt>privileged</tt> tells the framework whether or not @@ -53,13 +53,13 @@ public interface OsgiFramework { * @throws SecurityException If the caller does not have the appropriate permissions. * @throws IllegalStateException If this bundle has been uninstalled or this bundle tries to change its own state. */ - public void startBundles(List<Bundle> bundles, boolean privileged) throws BundleException; + void startBundles(List<Bundle> bundles, boolean privileged) throws BundleException; /** * <p>This method <em>synchronously</em> refreshes all bundles currently loaded. Once this method returns, the * class loaders of all bundles will reflect on the current set of loaded bundles.</p> */ - public void refreshPackages(); + void refreshPackages(); /** * <p>Returns the BundleContext of this framework's system bundle. The returned BundleContext can be used by the @@ -70,7 +70,7 @@ public interface OsgiFramework { * @throws SecurityException If the caller does not have the appropriate permissions. * @since 2.0 */ - public BundleContext bundleContext(); + BundleContext bundleContext(); /** * <p>Returns an iterable collection of all installed bundles. This method returns a list of all bundles installed @@ -79,7 +79,7 @@ public interface OsgiFramework { * * @return An iterable collection of Bundle objects, one object per installed bundle. */ - public List<Bundle> bundles(); + List<Bundle> bundles(); /** * <p>This method starts the framework instance. Before this method is called, any call to {@link @@ -87,7 +87,7 @@ public interface OsgiFramework { * * @throws BundleException If any error occurs. */ - public void start() throws BundleException; + void start() throws BundleException; /** * <p>This method <em>synchronously</em> shuts down the framework. It must be called at the end of a session in @@ -95,5 +95,6 @@ public interface OsgiFramework { * * @throws BundleException If any error occurs. */ - public void stop() throws BundleException; + void stop() throws BundleException; + } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java index 729bef7985f..cf5beb4a903 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java @@ -22,33 +22,34 @@ import java.util.logging.Logger; * and forward messages.</p> * * <p>There are three types of sessions:</p> - * <ul><li>{@link SourceSession Source sessions} sends messages and receives - * replies</li> - * <li>{@link IntermediateSession Intermediate sessions} receives messages on - * their way to their final destination, and may decide to forward the messages - * or reply directly.</li> - * <li>{@link DestinationSession Destination sessions} are the final recipient - * of messages, and are expected to reply to every one of them, but may not - * forward messages.</li></ul> + * <ul> + * <li>{@link SourceSession Source sessions} sends messages and receives replies</li> + * <li>{@link IntermediateSession Intermediate sessions} receives messages on + * their way to their final destination, and may decide to forward the messages or reply directly. + * <li>{@link DestinationSession Destination sessions} are the final recipient + * of messages, and are expected to reply to every one of them, but may not forward messages. + * </ul> * * <p>A message bus is configured with a {@link Protocol protocol}. This table * enumerates the permissible routes from intermediates to destinations and the * messaging semantics of each hop.</p> * - * <p>The responsibilities of a message bus are:</p> - * <ul> <li>Assign a route to every send message from its routing table</li> - * <li>Deliver every message it <i>accepts</i> to the next hop on its route on a - * best effort basis, <i>or</i> deliver a <i>failure reply</i>.</li> - * <li>Deliver replies back to message sources through all the intermediate - * hops.</li></ul> + * The responsibilities of a message bus are: + * <ul> + * <li>Assign a route to every send message from its routing table + * <li>Deliver every message it <i>accepts</i> to the next hop on its route + * <i>or</i> deliver a <i>failure reply</i>. + * <li>Deliver replies back to message sources through all the intermediate hops. + * </ul> * - * <p>A runtime will typically</p> - * <ul><li>Create a message bus implementation and set properties on this - * implementation once.</li> - * <li>Create sessions using that message bus many places.</li></ul> + * A runtime will typically + * <ul> + * <li>Create a message bus implementation and set properties on this implementation once. + * <li>Create sessions using that message bus many places.</li> + * </ul> * - * @author btratseth - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author bratseth + * @author Simon Thoresen */ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, ReplyHandler { @@ -101,9 +102,8 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, // Attach and start network. this.net = net; net.attach(this); - if (!net.waitUntilReady(120)) { + if ( ! net.waitUntilReady(120)) throw new IllegalStateException("Network failed to become ready in time."); - } // Start messenger. msn = new Messenger(); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/NetworkMessageBus.java b/messagebus/src/main/java/com/yahoo/messagebus/NetworkMessageBus.java new file mode 100644 index 00000000000..24e177f1fbf --- /dev/null +++ b/messagebus/src/main/java/com/yahoo/messagebus/NetworkMessageBus.java @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.messagebus; + +import com.yahoo.messagebus.network.Network; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * The combination of a messagebus and a network over which it may send data. + * + * @author bratseth + */ +public class NetworkMessageBus { + + private final Network network; + private final MessageBus messageBus; + + private final AtomicBoolean destroyed = new AtomicBoolean(false); + + public NetworkMessageBus(Network network, MessageBus messageBus) { + this.network = network; + this.messageBus = messageBus; + } + + /** Returns the contained message bus object */ + public MessageBus getMessageBus() { return messageBus; } + + /** Returns the network of this as a Network */ + public Network getNetwork() { return network; } + + /** + * Irreversibly destroys the content of this. + * + * @return whether this destroyed anything, or if it was already destroyed + */ + public boolean destroy() { + if ( destroyed.getAndSet(true)) return false; + + getMessageBus().destroy(); + return true; + } + +} diff --git a/messagebus/src/main/java/com/yahoo/messagebus/RPCMessageBus.java b/messagebus/src/main/java/com/yahoo/messagebus/RPCMessageBus.java index d767e197b11..cfa50a35549 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/RPCMessageBus.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/RPCMessageBus.java @@ -3,6 +3,7 @@ package com.yahoo.messagebus; import com.yahoo.log.LogLevel; import com.yahoo.messagebus.network.Identity; +import com.yahoo.messagebus.network.Network; import com.yahoo.messagebus.network.rpc.RPCNetwork; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; @@ -17,12 +18,9 @@ import java.util.logging.Logger; * * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> */ -public class RPCMessageBus { +public class RPCMessageBus extends NetworkMessageBus { private static final Logger log = Logger.getLogger(RPCMessageBus.class.getName()); - private final AtomicBoolean destroyed = new AtomicBoolean(false); - private final MessageBus mbus; - private final RPCNetwork net; private final ConfigAgent configAgent; /** @@ -33,9 +31,16 @@ public class RPCMessageBus { * @param routingCfgId The config id for message bus routing specs. */ public RPCMessageBus(MessageBusParams mbusParams, RPCNetworkParams rpcParams, String routingCfgId) { - net = new RPCNetwork(rpcParams); - mbus = new MessageBus(net, mbusParams); - configAgent = new ConfigAgent(routingCfgId != null ? routingCfgId : "client", mbus); + this(mbusParams, new RPCNetwork(rpcParams), routingCfgId); + } + + private RPCMessageBus(MessageBusParams mbusParams, RPCNetwork network, String routingCfgId) { + this(new MessageBus(network, mbusParams), network, routingCfgId); + } + + private RPCMessageBus(MessageBus messageBus, RPCNetwork network, String routingCfgId) { + super(network, messageBus); + configAgent = new ConfigAgent(routingCfgId != null ? routingCfgId : "client", messageBus); configAgent.subscribe(); } @@ -80,33 +85,17 @@ public class RPCMessageBus { * Sets the destroyed flag to true. The very first time this method is called, it cleans up all its dependencies. * Even if you retain a reference to this object, all of its content is allowed to be garbage collected. * - * @return True if content existed and was destroyed. + * @return true if content existed and was destroyed. */ + @Override public boolean destroy() { - if (!destroyed.getAndSet(true)) { + boolean destroyed = super.destroy(); + if (destroyed) configAgent.shutdown(); - mbus.destroy(); - return true; - } - return false; + return destroyed; } - /** - * Returns the contained message bus object. - * - * @return Message bus. - */ - public MessageBus getMessageBus() { - return mbus; - } - - /** - * Returns the contained rpc network object. - * - * @return RPC network. - */ - public RPCNetwork getRPCNetwork() { - return net; - } + /** Returns the network of this as a RPCNetwork */ + public RPCNetwork getRPCNetwork() { return (RPCNetwork)getNetwork(); } } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/Network.java b/messagebus/src/main/java/com/yahoo/messagebus/network/Network.java index cd3b3286778..b0bbe4266c4 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/Network.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/Network.java @@ -21,28 +21,28 @@ public interface Network { * @param seconds The timeout. * @return True if ready. */ - public boolean waitUntilReady(double seconds); + boolean waitUntilReady(double seconds); /** * Attach the network layer to the given owner * * @param owner owner of the network */ - public void attach(NetworkOwner owner); + void attach(NetworkOwner owner); /** * Register a session name with the network layer. This will make the session visible to other nodes. * * @param session the session name */ - public void registerSession(String session); + void registerSession(String session); /** * Unregister a session name with the network layer. This will make the session unavailable for other nodes. * * @param session session name */ - public void unregisterSession(String session); + void unregisterSession(String session); /** * Resolves the service address of the recipient referenced by the given routing node. If a recipient can not be @@ -52,7 +52,7 @@ public interface Network { * @param recipient The node whose service address to allocate. * @return True if a service address was allocated. */ - public boolean allocServiceAddress(RoutingNode recipient); + boolean allocServiceAddress(RoutingNode recipient); /** * Frees the service address from the given routing node. This allows the network layer to track and close @@ -60,7 +60,7 @@ public interface Network { * * @param recipient The node whose service address to free. */ - public void freeServiceAddress(RoutingNode recipient); + void freeServiceAddress(RoutingNode recipient); /** * Send a message to the given recipients. A {@link RoutingNode} contains all the necessary context for sending. @@ -68,7 +68,7 @@ public interface Network { * @param msg The message to send. * @param recipients A list of routing leaf nodes resolved for the message. */ - public void send(Message msg, List<RoutingNode> recipients); + void send(Message msg, List<RoutingNode> recipients); /** * Synchronize with internal threads. This method will handshake with all internal threads. This has the implicit @@ -76,12 +76,12 @@ public interface Network { * that would make the thread wait for itself... forever. This method is typically used to untangle during session * shutdown. */ - public void sync(); + void sync(); /** * Shuts down the network. This is a blocking call that waits for all scheduled tasks to complete. */ - public void shutdown(); + void shutdown(); /** * Returns a string that represents the connection specs of this network. It is in not a complete address since it @@ -89,12 +89,13 @@ public interface Network { * * @return The connection string. */ - public String getConnectionSpec(); + String getConnectionSpec(); /** * Returns a reference to a name server mirror. * * @return The mirror object. */ - public IMirror getMirror(); + IMirror getMirror(); + } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/local/LocalNetwork.java b/messagebus/src/main/java/com/yahoo/messagebus/network/local/LocalNetwork.java index ffcb853a0a7..78cf352cfbf 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/local/LocalNetwork.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/local/LocalNetwork.java @@ -23,7 +23,7 @@ import java.util.concurrent.Executors; import static com.yahoo.messagebus.ErrorCode.NO_ADDRESS_FOR_SERVICE; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + * @author Simon Thoresen Hult */ public class LocalNetwork implements Network { @@ -32,35 +32,39 @@ public class LocalNetwork implements Network { private final String hostId; private volatile NetworkOwner owner; - public LocalNetwork(final LocalWire wire) { + public LocalNetwork() { + this(new LocalWire()); + } + + public LocalNetwork(LocalWire wire) { this.wire = wire; this.hostId = wire.newHostId(); } @Override - public boolean waitUntilReady(final double seconds) { + public boolean waitUntilReady(double seconds) { return true; } @Override - public void attach(final NetworkOwner owner) { + public void attach(NetworkOwner owner) { this.owner = owner; } @Override - public void registerSession(final String session) { + public void registerSession(String session) { wire.registerService(hostId + "/" + session, this); } @Override - public void unregisterSession(final String session) { + public void unregisterSession(String session) { wire.unregisterService(hostId + "/" + session); } @Override - public boolean allocServiceAddress(final RoutingNode recipient) { - final String service = recipient.getRoute().getHop(0).getServiceName(); - final ServiceAddress address = wire.resolveServiceAddress(service); + public boolean allocServiceAddress(RoutingNode recipient) { + String service = recipient.getRoute().getHop(0).getServiceName(); + ServiceAddress address = wire.resolveServiceAddress(service); if (address == null) { recipient.setError(new Error(NO_ADDRESS_FOR_SERVICE, "No address for service '" + service + "'.")); return false; @@ -70,24 +74,24 @@ public class LocalNetwork implements Network { } @Override - public void freeServiceAddress(final RoutingNode recipient) { + public void freeServiceAddress(RoutingNode recipient) { recipient.setServiceAddress(null); } @Override - public void send(final Message msg, final List<RoutingNode> recipients) { - for (final RoutingNode recipient : recipients) { + public void send(Message msg, List<RoutingNode> recipients) { + for (RoutingNode recipient : recipients) { new MessageEnvelope(this, msg, recipient).send(); } } - private void receiveLater(final MessageEnvelope envelope) { - final byte[] payload = envelope.sender.encode(envelope.msg.getProtocol(), envelope.msg); + private void receiveLater(MessageEnvelope envelope) { + byte[] payload = envelope.sender.encode(envelope.msg.getProtocol(), envelope.msg); executor.execute(new Runnable() { @Override public void run() { - final Message msg = decode(envelope.msg.getProtocol(), payload, Message.class); + Message msg = decode(envelope.msg.getProtocol(), payload, Message.class); msg.getTrace().setLevel(envelope.msg.getTrace().getLevel()); msg.setRoute(envelope.msg.getRoute()).getRoute().removeHop(0); msg.setRetryEnabled(envelope.msg.getRetryEnabled()); @@ -96,7 +100,7 @@ public class LocalNetwork implements Network { msg.pushHandler(new ReplyHandler() { @Override - public void handleReply(final Reply reply) { + public void handleReply(Reply reply) { new ReplyEnvelope(LocalNetwork.this, envelope, reply).send(); } }); @@ -106,17 +110,17 @@ public class LocalNetwork implements Network { }); } - private void receiveLater(final ReplyEnvelope envelope) { - final byte[] payload = envelope.sender.encode(envelope.reply.getProtocol(), envelope.reply); + private void receiveLater(ReplyEnvelope envelope) { + byte[] payload = envelope.sender.encode(envelope.reply.getProtocol(), envelope.reply); executor.execute(new Runnable() { @Override public void run() { - final Reply reply = decode(envelope.reply.getProtocol(), payload, Reply.class); + Reply reply = decode(envelope.reply.getProtocol(), payload, Reply.class); reply.setRetryDelay(envelope.reply.getRetryDelay()); reply.getTrace().getRoot().addChild(TraceNode.decode(envelope.reply.getTrace().getRoot().encode())); for (int i = 0, len = envelope.reply.getNumErrors(); i < len; ++i) { - final Error error = envelope.reply.getError(i); + Error error = envelope.reply.getError(i); reply.addError(new Error(error.getCode(), error.getMessage(), error.getService() != null ? error.getService() : envelope.sender.hostId)); @@ -126,7 +130,7 @@ public class LocalNetwork implements Network { }); } - private byte[] encode(final Utf8String protocolName, final Routable toEncode) { + private byte[] encode(Utf8String protocolName, Routable toEncode) { if (toEncode.getType() == 0) { return new byte[0]; } @@ -134,7 +138,7 @@ public class LocalNetwork implements Network { } @SuppressWarnings("unchecked") - private <T extends Routable> T decode(final Utf8String protocolName, final byte[] toDecode, final Class<T> clazz) { + private <T extends Routable> T decode(Utf8String protocolName, byte[] toDecode, Class<T> clazz) { if (toDecode.length == 0) { return clazz.cast(new EmptyReply()); } @@ -167,15 +171,14 @@ public class LocalNetwork implements Network { final Message msg; final RoutingNode recipient; - MessageEnvelope(final LocalNetwork sender, final Message msg, final RoutingNode recipient) { + MessageEnvelope(LocalNetwork sender, Message msg, RoutingNode recipient) { this.sender = sender; this.msg = msg; this.recipient = recipient; } void send() { - LocalServiceAddress.class.cast(recipient.getServiceAddress()) - .getNetwork().receiveLater(this); + LocalServiceAddress.class.cast(recipient.getServiceAddress()).getNetwork().receiveLater(this); } } @@ -185,7 +188,7 @@ public class LocalNetwork implements Network { final MessageEnvelope parent; final Reply reply; - ReplyEnvelope(final LocalNetwork sender, final MessageEnvelope parent, final Reply reply) { + ReplyEnvelope(LocalNetwork sender, MessageEnvelope parent, Reply reply) { this.sender = sender; this.parent = parent; this.reply = reply; @@ -195,4 +198,5 @@ public class LocalNetwork implements Network { parent.sender.receiveLater(this); } } + } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/local/LocalWire.java b/messagebus/src/main/java/com/yahoo/messagebus/network/local/LocalWire.java index 84ca8c64bc0..5c9035a5f99 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/local/LocalWire.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/local/LocalWire.java @@ -11,7 +11,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Pattern; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + * @author Simon Thoresen Hult */ public class LocalWire implements IMirror { @@ -19,19 +19,19 @@ public class LocalWire implements IMirror { private final AtomicInteger updateCnt = new AtomicInteger(); private final ConcurrentHashMap<String, LocalNetwork> services = new ConcurrentHashMap<>(); - public void registerService(final String serviceName, final LocalNetwork owner) { + public void registerService(String serviceName, LocalNetwork owner) { if (services.putIfAbsent(serviceName, owner) != null) { throw new IllegalStateException(); } updateCnt.incrementAndGet(); } - public void unregisterService(final String serviceName) { + public void unregisterService(String serviceName) { services.remove(serviceName); updateCnt.incrementAndGet(); } - public LocalServiceAddress resolveServiceAddress(final String serviceName) { + public LocalServiceAddress resolveServiceAddress(String serviceName) { final LocalNetwork owner = services.get(serviceName); return owner != null ? new LocalServiceAddress(serviceName, owner) : null; } @@ -41,10 +41,10 @@ public class LocalWire implements IMirror { } @Override - public Mirror.Entry[] lookup(final String pattern) { - final List<Mirror.Entry> out = new ArrayList<>(); - final Pattern regex = Pattern.compile(pattern.replace("*", "[a-zA-Z0-9_-]+")); - for (final String key : services.keySet()) { + public Mirror.Entry[] lookup(String pattern) { + List<Mirror.Entry> out = new ArrayList<>(); + Pattern regex = Pattern.compile(pattern.replace("*", "[a-zA-Z0-9_-]+")); + for (String key : services.keySet()) { if (regex.matcher(key).matches()) { out.add(new Mirror.Entry(key, key)); } @@ -56,4 +56,5 @@ public class LocalWire implements IMirror { public int updates() { return updateCnt.get(); } + } diff --git a/standalone-container/src/main/scala/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.scala b/standalone-container/src/main/scala/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.scala index 6d45e6fa8a1..5a41462cb48 100644 --- a/standalone-container/src/main/scala/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.scala +++ b/standalone-container/src/main/scala/com/yahoo/application/container/impl/ClassLoaderOsgiFramework.scala @@ -17,6 +17,8 @@ import org.osgi.framework.wiring._ import org.osgi.resource.{Wire, Capability, Requirement} /** + * A (mock) OSGI implementation which loads classes from the system classpath + * * @author tonytv */ final class ClassLoaderOsgiFramework extends OsgiFramework { @@ -197,4 +199,5 @@ final class ClassLoaderOsgiFramework extends OsgiFramework { override def createFilter(filter: String) = throw new UnsupportedOperationException } + } diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala index 3f31dcd67ca..fc57353c194 100644 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala +++ b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala @@ -10,6 +10,7 @@ import java.io.{IOException, File} import com.yahoo.config.model.test.MockRoot import com.yahoo.config.model.application.provider._ import com.yahoo.vespa.defaults.Defaults +import com.yahoo.vespa.model.VespaModel import com.yahoo.vespa.model.container.xml.{ConfigServerContainerModelBuilder, ManhattanContainerModelBuilder, ContainerModelBuilder} import org.w3c.dom.Element import com.yahoo.config.model.builder.xml.XmlHelper @@ -164,7 +165,7 @@ object StandaloneContainerApplication { fileRegistry: FileRegistry, preprocessedApplicationDir: File, networkingOption: Networking, - configModelRepo: ConfigModelRepo = new ConfigModelRepo): (MockRoot, Container) = { + configModelRepo: ConfigModelRepo = new ConfigModelRepo): (VespaModel, Container) = { val logger = new BaseDeployLogger val rawApplicationPackage = new FilesApplicationPackage.Builder(applicationPath.toFile).includeSourceFiles(true).preprocessedDir(preprocessedApplicationDir).build() // TODO: Needed until we get rid of semantic rules @@ -179,12 +180,13 @@ object StandaloneContainerApplication { configDefinitionRepo(configDefinitionRepo). build() - val root = new MockRoot("", deployState) + val root = VespaModel.createIncomplete(deployState) val vespaRoot = new ApplicationConfigProducerRoot(root, "vespa", deployState.getDocumentModel, deployState.getProperties.vespaVersion(), deployState.getProperties.applicationId()) + val spec = containerRootElement(applicationPackage) val containerModel = newContainerModelBuilder(networkingOption).build(deployState, configModelRepo, vespaRoot, spec) @@ -192,11 +194,16 @@ object StandaloneContainerApplication { containerModel.initialize(configModelRepo) val container = first(containerModel.getCluster().getContainers) + // TODO: If we can do the mutations below on the builder, we can separate out model finalization from the + // VespaModel constructor, such that the above and below code to finalize the container can be + // replaced by root.finalize(); + // Always disable rpc server for standalone container. This server will soon be removed anyway. container.setRpcServerEnabled(false) container.setHttpServerEnabled(networkingOption == Networking.enable) initializeContainer(container, spec) + root.freezeModelTopology() (root, container) } diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneSubscriberFactory.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneSubscriberFactory.scala index 432e5b82946..4ac88eaafae 100644 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneSubscriberFactory.scala +++ b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneSubscriberFactory.scala @@ -4,6 +4,7 @@ package com.yahoo.container.standalone import com.yahoo.config.model.test.MockRoot import com.yahoo.config.{ConfigBuilder, ConfigInstance} import com.yahoo.container.di.ConfigKeyT +import com.yahoo.vespa.model.VespaModel import scala.collection.JavaConversions._ import scala.collection.JavaConverters._ import com.yahoo.vespa.config.ConfigKey @@ -14,7 +15,7 @@ import StandaloneSubscriberFactory._ * @author tonytv * @author gjoranv */ -class StandaloneSubscriberFactory(root: MockRoot) extends SubscriberFactory { +class StandaloneSubscriberFactory(root: VespaModel) extends SubscriberFactory { class StandaloneSubscriber(configKeys: Set[ConfigKeyT]) extends Subscriber { override def configChanged = generation == 0 diff --git a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java index 484d4c4d50e..f847f05108e 100644 --- a/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java +++ b/standalone-container/src/test/java/com/yahoo/container/standalone/StandaloneContainerActivatorTest.java @@ -24,7 +24,7 @@ import static org.hamcrest.collection.IsEmptyCollection.empty; import static org.junit.Assert.assertThat; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge * @since 5.22.0 */ public class StandaloneContainerActivatorTest { diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala index 7152c0c0af1..f0c2ce6fa0d 100644 --- a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala +++ b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainer.scala @@ -32,7 +32,7 @@ object StandaloneContainer { } } - def withContainerModel[T](containerNode: Node)(f: MockRoot => T) { + def withContainerModel[T](containerNode: Node)(f: VespaModel => T) { withTempDirectory { applicationPath => createServicesXml(applicationPath, containerNode) diff --git a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala index 41026e1c263..2705322ab32 100644 --- a/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala +++ b/standalone-container/src/test/scala/com/yahoo/container/standalone/StandaloneContainerTest.scala @@ -58,7 +58,7 @@ class StandaloneContainerTest { </services> StandaloneContainer.withContainerModel(servicesXml) { root => - assertNotNull(root.getProducer("container-1/standalone")) + assertTrue(root.getConfigProducer("container-1/standalone").isPresent) } } @@ -72,10 +72,10 @@ class StandaloneContainerTest { </jdisc> StandaloneContainer.withContainerModel(xml) { root => - val container = root.getProducer("jdisc/standalone").asInstanceOf[AbstractService] + val container = root.getConfigProducer("jdisc/standalone").get().asInstanceOf[AbstractService] println("portCnt: " + container.getPortCount) println("numPorts: " + container.getNumPortsAllocated) - assertThat(container.getNumPortsAllocated, is(1)) + assertEquals(1, container.getNumPortsAllocated) } } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java index 4889d064387..6a7797c20a7 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java @@ -23,6 +23,7 @@ import com.yahoo.document.restapi.RestUri; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusParams; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.vespa.config.content.LoadTypeConfig; import com.yahoo.vespaxmlparser.VespaXMLFeedReader; import java.io.IOException; @@ -55,10 +56,12 @@ public class RestApi extends LoggingRequestHandler { private AtomicInteger threadsAvailableForApi = new AtomicInteger(20 /*max concurrent requests */); @Inject - public RestApi(Executor executor, AccessLog accessLog, DocumentmanagerConfig documentManagerConfig) { + public RestApi(Executor executor, AccessLog accessLog, DocumentmanagerConfig documentManagerConfig, + LoadTypeConfig loadTypeConfig) { super(executor, accessLog); - final LoadTypeSet loadTypes = new LoadTypeSet("client"); - this.operationHandler = new OperationHandlerImpl(new MessageBusDocumentAccess(new MessageBusParams(loadTypes))); + MessageBusParams params = new MessageBusParams(new LoadTypeSet(loadTypeConfig)); + params.setDocumentmanagerConfig(documentManagerConfig); + this.operationHandler = new OperationHandlerImpl(new MessageBusDocumentAccess(params)); this.singleDocumentParser = new SingleDocumentParser(new DocumentTypeManager(documentManagerConfig)); } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java index 14b2d86ae75..87a7ebe9e49 100755 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemove.java @@ -3,9 +3,12 @@ package com.yahoo.feedhandler; import com.google.inject.Inject; import com.yahoo.clientmetrics.RouteMetricSet; +import com.yahoo.cloud.config.ClusterListConfig; +import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.document.DocumentId; +import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.feedapi.FeedContext; import com.yahoo.feedapi.MessagePropertyProcessor; import com.yahoo.feedapi.SingleSender; @@ -20,9 +23,14 @@ import java.util.concurrent.Executor; public class VespaFeedHandlerRemove extends VespaFeedHandlerBase { @Inject - public VespaFeedHandlerRemove(FeederConfig feederConfig, - LoadTypeConfig loadTypeConfig, Executor executor, Metric metric) throws Exception { - super(feederConfig, loadTypeConfig, executor, metric); + public VespaFeedHandlerRemove(FeederConfig feederConfig, + LoadTypeConfig loadTypeConfig, + DocumentmanagerConfig documentmanagerConfig, + SlobroksConfig slobroksConfig, + ClusterListConfig clusterListConfig, + Executor executor, + Metric metric) throws Exception { + super(feederConfig, loadTypeConfig, documentmanagerConfig, slobroksConfig, clusterListConfig, executor, metric); } VespaFeedHandlerRemove(FeedContext context, Executor executor) throws Exception { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java index 3b2f82c865e..04d22386bfb 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerRemoveLocation.java @@ -3,8 +3,11 @@ package com.yahoo.feedhandler; import com.google.inject.Inject; import com.yahoo.clientmetrics.RouteMetricSet; +import com.yahoo.cloud.config.ClusterListConfig; +import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.messagebus.protocol.RemoveLocationMessage; import com.yahoo.feedapi.FeedContext; import com.yahoo.feedapi.MessagePropertyProcessor; @@ -19,9 +22,13 @@ import java.util.concurrent.Executor; public class VespaFeedHandlerRemoveLocation extends VespaFeedHandlerBase { @Inject - public VespaFeedHandlerRemoveLocation(FeederConfig feederConfig, LoadTypeConfig loadTypeConfig, Executor executor, - Metric metric) throws Exception { - super(feederConfig, loadTypeConfig, executor, metric); + public VespaFeedHandlerRemoveLocation(FeederConfig feederConfig, + LoadTypeConfig loadTypeConfig, + DocumentmanagerConfig documentmanagerConfig, + SlobroksConfig slobroksConfig, + ClusterListConfig clusterListConfig, + Executor executor, Metric metric) throws Exception { + super(feederConfig, loadTypeConfig, documentmanagerConfig, slobroksConfig, clusterListConfig, executor, metric); } VespaFeedHandlerRemoveLocation(FeedContext context, Executor executor) throws Exception { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java index 77930ae5a94..ed80443f970 100755 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerStatus.java @@ -3,9 +3,12 @@ package com.yahoo.feedhandler; import java.util.concurrent.Executor; +import com.yahoo.cloud.config.ClusterListConfig; +import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.vespa.config.content.LoadTypeConfig; import com.yahoo.feedapi.FeedContext; import com.yahoo.metrics.MetricManager; @@ -16,8 +19,14 @@ public class VespaFeedHandlerStatus extends ThreadedHttpRequestHandler { private MetricManager manager; - public VespaFeedHandlerStatus(FeederConfig feederConfig, LoadTypeConfig loadTypeConfig, Executor executor) { - this(FeedContext.getInstance(feederConfig, loadTypeConfig, new NullFeedMetric()), true, true, executor); + public VespaFeedHandlerStatus(FeederConfig feederConfig, + LoadTypeConfig loadTypeConfig, + DocumentmanagerConfig documentmanagerConfig, + SlobroksConfig slobroksConfig, + ClusterListConfig clusterListConfig, + Executor executor) { + this(FeedContext.getInstance(feederConfig, loadTypeConfig, documentmanagerConfig, slobroksConfig, + clusterListConfig, new NullFeedMetric()), true, true, executor); } VespaFeedHandlerStatus(FeedContext context, boolean doLog, boolean makeSnapshots, Executor executor) { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/GetSearcher.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/GetSearcher.java index 661fcac6a64..cf42bce9c1c 100755 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/GetSearcher.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/GetSearcher.java @@ -2,7 +2,10 @@ package com.yahoo.storage.searcher; import com.google.inject.Inject; +import com.yahoo.cloud.config.ClusterListConfig; +import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.feedhandler.NullFeedMetric; import com.yahoo.processing.request.CompoundName; import com.yahoo.vespa.config.content.LoadTypeConfig; @@ -169,9 +172,13 @@ public class GetSearcher extends Searcher { } @Inject - public GetSearcher(FeederConfig feederConfig, LoadTypeConfig loadTypeConfig) throws Exception { - this(FeedContext.getInstance(feederConfig, loadTypeConfig, new NullFeedMetric()), - (long)(feederConfig.timeout() * 1000)); + public GetSearcher(FeederConfig feederConfig, + LoadTypeConfig loadTypeConfig, + DocumentmanagerConfig documentmanagerConfig, + SlobroksConfig slobroksConfig, + ClusterListConfig clusterListConfig) throws Exception { + this(FeedContext.getInstance(feederConfig, loadTypeConfig, documentmanagerConfig, slobroksConfig, + clusterListConfig, new NullFeedMetric()), (long)(feederConfig.timeout() * 1000)); } GetSearcher(FeedContext context) throws Exception { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/VisitSearcher.java b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/VisitSearcher.java index 621ffcefbe1..2d7e5fbc338 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/VisitSearcher.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/storage/searcher/VisitSearcher.java @@ -1,6 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.storage.searcher; +import com.yahoo.cloud.config.ClusterListConfig; +import com.yahoo.cloud.config.SlobroksConfig; +import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.feedhandler.NullFeedMetric; import com.yahoo.vespa.config.content.LoadTypeConfig; import com.yahoo.component.ComponentId; @@ -30,8 +33,13 @@ public class VisitSearcher extends Searcher { public static final String VISITOR_CONTINUATION_TOKEN_FIELDNAME = "visitorContinuationToken"; FeedContext context; - public VisitSearcher(FeederConfig feederConfig, LoadTypeConfig loadTypeConfig) throws Exception { - this(FeedContext.getInstance(feederConfig, loadTypeConfig, new NullFeedMetric())); + public VisitSearcher(FeederConfig feederConfig, + LoadTypeConfig loadTypeConfig, + DocumentmanagerConfig documentmanagerConfig, + SlobroksConfig slobroksConfig, + ClusterListConfig clusterListConfig) throws Exception { + this(FeedContext.getInstance(feederConfig, loadTypeConfig, documentmanagerConfig, + slobroksConfig, clusterListConfig, new NullFeedMetric())); } VisitSearcher(FeedContext context) throws Exception { diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/DocumentApiApplicationtest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/DocumentApiApplicationtest.java new file mode 100644 index 00000000000..42ec41bd107 --- /dev/null +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/DocumentApiApplicationtest.java @@ -0,0 +1,23 @@ +package com.yahoo.document.restapi; + +import com.yahoo.application.Application; +import com.yahoo.application.Networking; +import org.junit.Test; + +/** + * @author bratseth + */ +public class DocumentApiApplicationTest { + + /** Test that it is possible to instantiate an Application with a document-api */ + @Test + public void application_with_document_api() { + String services = + "<container version='1.0'>" + + " <document-api/>" + + "</container>"; + try (Application application = Application.fromServicesXml(services, Networking.enable)) { + } + } + +} diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java index c4e9e27ca75..95a48ab41fe 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/RestApiTest.java @@ -33,11 +33,12 @@ import static org.hamcrest.core.StringStartsWith.startsWith; import static org.junit.Assert.assertThat; public class RestApiTest { + Application application; @Before public void setup() throws Exception { - application = Application.fromApplicationPackage(Paths.get("src/test/application"), Networking.enable); + application = Application.fromApplicationPackage(Paths.get("src/test/rest-api-application"), Networking.enable); } @After @@ -249,7 +250,7 @@ public class RestApiTest { public void testbasicEncodingV2() throws Exception { Request request = new Request("http://localhost:" + getFirstListenPort() + get_enc_test_uri_v2); HttpGet get = new HttpGet(request.getUri()); - final String rest = doRest(get); + String rest = doRest(get); assertThat(rest, containsString(get_enc_response_part1_v2)); assertThat(rest, containsString(get_enc_response_part2)); } @@ -264,7 +265,7 @@ public class RestApiTest { public void testbasicVisit() throws Exception { Request request = new Request("http://localhost:" + getFirstListenPort() + visit_test_uri); HttpGet get = new HttpGet(request.getUri()); - final String rest = doRest(get); + String rest = doRest(get); assertThat(rest, containsString(visit_response_part1)); assertThat(rest, containsString(visit_response_part2)); assertThat(rest, containsString(visit_response_part3)); @@ -276,9 +277,9 @@ public class RestApiTest { @Test public void testBadVisit() throws Exception { - final Request request = new Request("http://localhost:" + getFirstListenPort() + visit_test_bad_uri); + Request request = new Request("http://localhost:" + getFirstListenPort() + visit_test_bad_uri); HttpGet get = new HttpGet(request.getUri()); - final String rest = doRest(get); + String rest = doRest(get); assertThat(rest, containsString(visit_test_bad_response)); } diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java index 820f7f56e2f..4b1c69c73e7 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/storage/searcher/VisitorSearcherTestCase.java @@ -60,8 +60,7 @@ public class VisitorSearcherTestCase { public VisitSearcher create() throws Exception { ClusterListConfig.Storage.Builder storageCluster = new ClusterListConfig.Storage.Builder().configid("storage/cluster.foobar").name("foobar"); ClusterListConfig clusterListCfg = new ClusterListConfig(new ClusterListConfig.Builder().storage(storageCluster)); - ClusterList clusterList = new ClusterList(); - clusterList.configure(clusterListCfg); + ClusterList clusterList = new ClusterList(clusterListCfg); return new VisitSearcher(new FeedContext( new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(458).route("riksveg18").retryenabled(true)), new LoadTypeConfig(new LoadTypeConfig.Builder())), @@ -139,15 +138,13 @@ public class VisitorSearcherTestCase { ClusterListConfig.Storage.Builder storageCluster1 = new ClusterListConfig.Storage.Builder().configid("storage/cluster.foo").name("foo"); ClusterListConfig.Storage.Builder storageCluster2 = new ClusterListConfig.Storage.Builder().configid("storage/cluster.bar").name("bar"); ClusterListConfig clusterListCfg = new ClusterListConfig(new ClusterListConfig.Builder().storage(Arrays.asList(storageCluster1, storageCluster2))); - ClusterList clusterList = new ClusterList(); - clusterList.configure(clusterListCfg); + ClusterList clusterList = new ClusterList(clusterListCfg); VisitSearcher searcher = new VisitSearcher(new FeedContext( new MessagePropertyProcessor(new FeederConfig(new FeederConfig.Builder().timeout(100).route("whatever").retryenabled(true)), new LoadTypeConfig(new LoadTypeConfig.Builder())), factory, docMan, clusterList, new NullFeedMetric())); - searcher.getVisitorParameters( - newQuery("visit?visit.selection=id.user=1234"), null); + searcher.getVisitorParameters(newQuery("visit?visit.selection=id.user=1234"), null); } @Test diff --git a/vespaclient-container-plugin/src/test/application/services.xml b/vespaclient-container-plugin/src/test/rest-api-application/services.xml index df178e109c3..df178e109c3 100644 --- a/vespaclient-container-plugin/src/test/application/services.xml +++ b/vespaclient-container-plugin/src/test/rest-api-application/services.xml diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/FeedContext.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/FeedContext.java index a26064cd98b..c08d70b02f4 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/FeedContext.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/FeedContext.java @@ -1,6 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.feedapi; +import com.yahoo.cloud.config.ClusterListConfig; +import com.yahoo.cloud.config.SlobroksConfig; +import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.config.content.LoadTypeConfig; import com.yahoo.document.DocumentTypeManager; @@ -87,16 +90,21 @@ public class FeedContext { return docTypeManager; } - public static FeedContext getInstance(FeederConfig feederConfig, LoadTypeConfig loadTypeConfig, Metric metric) { + public static FeedContext getInstance(FeederConfig feederConfig, + LoadTypeConfig loadTypeConfig, + DocumentmanagerConfig documentmanagerConfig, + SlobroksConfig slobroksConfig, + ClusterListConfig clusterListConfig, + Metric metric) { synchronized (sync) { try { if (instance == null) { MessagePropertyProcessor proc = new MessagePropertyProcessor(feederConfig, loadTypeConfig); - MessageBusSessionFactory mbusFactory = new MessageBusSessionFactory(proc); + MessageBusSessionFactory mbusFactory = new MessageBusSessionFactory(proc, documentmanagerConfig, slobroksConfig); instance = new FeedContext(proc, mbusFactory, mbusFactory.getAccess().getDocumentTypeManager(), - new ClusterList("client"), metric); + new ClusterList(clusterListConfig), metric); } else { instance.getPropertyProcessor().configure(feederConfig, loadTypeConfig); } diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/FeederOptions.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/FeederOptions.java index 2894993b983..1546d605f02 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/FeederOptions.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/FeederOptions.java @@ -252,18 +252,6 @@ public class FeederOptions { return params; } - public MessageBusParams toMessageBusParams() { - MessageBusParams mbusParams = new MessageBusParams(); - if (retryEnabled) { - RetryTransientErrorsPolicy retryPolicy = new RetryTransientErrorsPolicy(); - retryPolicy.setBaseDelay(retryDelay); - mbusParams.setRetryPolicy(retryPolicy); - } else { - mbusParams.setRetryPolicy(null); - } - return mbusParams; - } - public RPCNetworkParams getNetworkParams() { try { RPCNetworkParams networkParams = new RPCNetworkParams(); diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessageBusSessionFactory.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessageBusSessionFactory.java index 8021ea86783..d670ceb4e77 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessageBusSessionFactory.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessageBusSessionFactory.java @@ -1,6 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.feedapi; +import com.yahoo.cloud.config.SlobroksConfig; +import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.VisitorParameters; import com.yahoo.documentapi.VisitorSession; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; @@ -12,6 +14,7 @@ import com.yahoo.jdisc.Metric; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.ReplyHandler; import com.yahoo.messagebus.SourceSession; +import com.yahoo.messagebus.network.rpc.RPCNetworkParams; import java.util.Collections; @@ -27,12 +30,24 @@ public class MessageBusSessionFactory implements SessionFactory { String NUM_UPDATES = "num_updates"; } + @SuppressWarnings("unused") // used from extensions public MessageBusSessionFactory(MessagePropertyProcessor processor) { + this(processor, null, null); + } + + public MessageBusSessionFactory(MessagePropertyProcessor processor, + DocumentmanagerConfig documentmanagerConfig, + SlobroksConfig slobroksConfig) { this.processor = processor; MessageBusParams params = new MessageBusParams(processor.getLoadTypes()); params.setTraceLevel(processor.getFeederOptions().getTraceLevel()); - params.setRPCNetworkParams(processor.getFeederOptions().getNetworkParams()); + RPCNetworkParams rpcNetworkParams = processor.getFeederOptions().getNetworkParams(); + if (slobroksConfig != null) // not set: will subscribe + rpcNetworkParams.setSlobroksConfig(slobroksConfig); + params.setRPCNetworkParams(rpcNetworkParams); params.setDocumentManagerConfigId("client"); + if (documentmanagerConfig != null) // not set: will subscribe + params.setDocumentmanagerConfig(documentmanagerConfig); access = new MessageBusDocumentAccess(params); } diff --git a/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java b/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java index 6e3facbdc98..08e1ca0482f 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java @@ -3,8 +3,11 @@ package com.yahoo.feedhandler; import com.google.inject.Inject; import com.yahoo.clientmetrics.RouteMetricSet; +import com.yahoo.cloud.config.ClusterListConfig; +import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.feedapi.DocprocMessageProcessor; import com.yahoo.feedapi.FeedContext; import com.yahoo.feedapi.Feeder; @@ -30,9 +33,14 @@ public final class VespaFeedHandler extends VespaFeedHandlerBase { public static final String JSON_INPUT = "jsonInput"; @Inject - public VespaFeedHandler(FeederConfig feederConfig, LoadTypeConfig loadTypeConfig, Executor executor, + public VespaFeedHandler(FeederConfig feederConfig, + LoadTypeConfig loadTypeConfig, + DocumentmanagerConfig documentmanagerConfig, + SlobroksConfig slobroksConfig, + ClusterListConfig clusterListConfig, + Executor executor, Metric metric) throws Exception { - super(feederConfig, loadTypeConfig, executor, metric); + super(feederConfig, loadTypeConfig, documentmanagerConfig, slobroksConfig, clusterListConfig, executor, metric); } VespaFeedHandler(FeedContext context, Executor executor) throws Exception { diff --git a/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerBase.java b/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerBase.java index fa1e6854593..6b4810f1ac4 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerBase.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerBase.java @@ -3,11 +3,14 @@ package com.yahoo.feedhandler; import com.google.inject.Inject; import com.yahoo.clientmetrics.ClientMetrics; +import com.yahoo.cloud.config.ClusterListConfig; +import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.docproc.DocprocService; import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.feedapi.FeedContext; import com.yahoo.feedapi.MessagePropertyProcessor; import com.yahoo.feedapi.SharedSender; @@ -29,9 +32,14 @@ public abstract class VespaFeedHandlerBase extends ThreadedHttpRequestHandler { @Inject public VespaFeedHandlerBase(FeederConfig feederConfig, LoadTypeConfig loadTypeConfig, + DocumentmanagerConfig documentmanagerConfig, + SlobroksConfig slobroksConfig, + ClusterListConfig clusterListConfig, Executor executor, Metric metric) throws Exception { - this(FeedContext.getInstance(feederConfig, loadTypeConfig, metric), executor, (long)feederConfig.timeout() * 1000); + this(FeedContext.getInstance(feederConfig, loadTypeConfig, documentmanagerConfig, + slobroksConfig, clusterListConfig, metric), + executor, (long)feederConfig.timeout() * 1000); } public VespaFeedHandlerBase(FeedContext context, Executor executor) throws Exception { diff --git a/vespaclient-core/src/main/java/com/yahoo/vespaclient/ClusterList.java b/vespaclient-core/src/main/java/com/yahoo/vespaclient/ClusterList.java index 3ea3bb5cb9d..de3a85d9a9e 100644 --- a/vespaclient-core/src/main/java/com/yahoo/vespaclient/ClusterList.java +++ b/vespaclient-core/src/main/java/com/yahoo/vespaclient/ClusterList.java @@ -5,36 +5,38 @@ import com.yahoo.cloud.config.ClusterListConfig; import com.yahoo.config.subscription.ConfigGetter; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +/** A list of content clusters, either obtained from a list, a given config or by self-subscribing */ public class ClusterList { - List<ClusterDef> storageClusters = new ArrayList<ClusterDef>(); + + List<ClusterDef> contentClusters = new ArrayList<>(); public ClusterList() { - this(null); + this(Collections.emptyList()); + } + + public ClusterList(List<ClusterDef> contentClusters) { + this.contentClusters = contentClusters; } public ClusterList(String configId) { - if (configId != null) { - configure(new ConfigGetter<>(ClusterListConfig.class).getConfig(configId)); - } + configure(new ConfigGetter<>(ClusterListConfig.class).getConfig(configId)); } - - public List<ClusterDef> getStorageClusters() { - return storageClusters; + + public ClusterList(ClusterListConfig config) { + configure(config); } - public void configure(ClusterListConfig cfg) { - storageClusters.clear(); - for (int i = 0; i < cfg.storage().size(); i++) { - storageClusters.add(new ClusterDef(cfg.storage(i).name(), - cfg.storage(i).configid())); - } + private void configure(ClusterListConfig config) { + contentClusters.clear(); // TODO: Create a new + for (int i = 0; i < config.storage().size(); i++) + contentClusters.add(new ClusterDef(config.storage(i).name(), config.storage(i).configid())); } - public static ClusterList createMockedList(List<ClusterDef> clusters) { - ClusterList list = new ClusterList(null); - list.storageClusters = clusters; - return list; + public List<ClusterDef> getStorageClusters() { + return contentClusters; // TODO: Use immutable list } + } diff --git a/vespajlib/src/main/java/com/yahoo/net/HostName.java b/vespajlib/src/main/java/com/yahoo/net/HostName.java index 9dff33e1f5f..4e791ca117a 100644 --- a/vespajlib/src/main/java/com/yahoo/net/HostName.java +++ b/vespajlib/src/main/java/com/yahoo/net/HostName.java @@ -15,12 +15,11 @@ public class HostName { private static String myHost = null; /** - * Static method that returns the name of localhost using shell - * command "hostname". + * Static method that returns the name of localhost using shell command "hostname". + * If you need a guaranteed resolvable name see LinuxINetAddress. * * @return the name of localhost. * @throws RuntimeException if executing the command 'hostname' fails. - * @see LinuxInetAddress if you need a host name/address which is reachable */ public static synchronized String getLocalhost() { if (myHost == null) { @@ -38,4 +37,5 @@ public class HostName { } return myHost; } + } |