diff options
24 files changed, 554 insertions, 42 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 4539f71490f..968e690fac3 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -54,8 +54,6 @@ import java.util.logging.Logger; */ public class DeployState implements ConfigDefinitionStore { - private static final Logger log = Logger.getLogger(DeployState.class.getName()); - private final DeployLogger logger; private final FileRegistry fileRegistry; private final DocumentModel documentModel; diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java index 98b7856612b..17bde94b53f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java @@ -69,6 +69,8 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce protected final void setParent(AbstractConfigProducer parent) { this.parent = parent; } public final String getSubId() { return subId; } + + /** Whether this is hosted Vespa: NOTE: This cannot be trusted to be correct :-/ */ public final boolean isHostedVespa() { return hostedVespa; } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java index e24fae3e3b5..d13272d3b0a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java @@ -181,6 +181,7 @@ public class VespaDomBuilder extends VespaModelBuilder { /** * Allocates a host to the service using host file or create service spec for provisioner to use later * Pre-condition: producerSpec is non-null + * * @param service the service to allocate a host for * @param hostSystem a {@link HostSystem} * @param producerSpec xml element for the service diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 7697d1123ff..96e5b05aacf 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -104,7 +104,7 @@ import static com.yahoo.container.core.BundleLoaderProperties.DISK_BUNDLE_PREFIX /** * @author gjoranv - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge * @author tonytv */ public final class ContainerCluster @@ -133,6 +133,7 @@ public final class ContainerCluster ServletPathsConfig.Producer, RoutingProviderConfig.Producer, ConfigserverConfig.Producer { + /** * URI prefix used for internal, usually programmatic, APIs. URIs using this * prefix should never considered available for direct use by customers, and diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index e3debe6ce20..c39b3247be1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -375,15 +375,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { cluster.addContainers(Collections.singleton(container)); } - private void addNodesFromXml(ContainerCluster cluster, Element spec, ConfigModelContext context) { - Element nodesElement = XML.getChild(spec, "nodes"); + private void addNodesFromXml(ContainerCluster cluster, Element containerElement, ConfigModelContext context) { + Element nodesElement = XML.getChild(containerElement, "nodes"); if (nodesElement == null) { // default single node on localhost - Container container = new Container(cluster, "container.0", 0); - HostResource host = allocateSingleNodeHost(cluster, log); - container.setHostResource(host); - if ( ! container.isInitialized() ) // TODO: Fold this into initService - container.initService(); - cluster.addContainers(Collections.singleton(container)); + Container node = new Container(cluster, "container.0", 0); + HostResource host = allocateSingleNodeHost(cluster, log, containerElement, context); + node.setHostResource(host); + if ( ! node.isInitialized() ) // TODO: Fold this into initService + node.initService(); + cluster.addContainers(Collections.singleton(node)); } else { List<Container> nodes = createNodes(cluster, nodesElement, context); @@ -440,10 +440,17 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } } - private HostResource allocateSingleNodeHost(ContainerCluster cluster, DeployLogger logger) { - if (cluster.isHostedVespa()) { - ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(cluster.getName()), Optional.empty()); - return cluster.getHostSystem().allocateHosts(clusterSpec, Capacity.fromNodeCount(1), 1, logger).keySet().iterator().next(); + /** Creates a single host when there is no nodes tag */ + private HostResource allocateSingleNodeHost(ContainerCluster cluster, DeployLogger logger, Element containerElement, ConfigModelContext context) { + if (cluster.getRoot().getDeployState().isHosted()) { + Optional<HostResource> singleContentHost = getHostResourceFromContentClusters(cluster, containerElement, context); + if (singleContentHost.isPresent()) { // there is a content cluster; put the container on its first node + return singleContentHost.get(); + } + else { // request 1 node + ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from(cluster.getName()), Optional.empty()); + return cluster.getHostSystem().allocateHosts(clusterSpec, Capacity.fromNodeCount(1), 1, logger).keySet().iterator().next(); + } } else { return cluster.getHostSystem().getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC); } @@ -492,6 +499,33 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return createNodesFromHosts(hosts, cluster); } + /** + * This is used in case we are on hosted Vespa and no nodes tag is supplied: + * If there are content clusters this will pick the first host in the first cluster as the container node. + * If there are no content clusters this will return empty (such that the node can be created by the container here). + */ + private Optional<HostResource> getHostResourceFromContentClusters(ContainerCluster cluster, Element containersElement, ConfigModelContext context) { + Optional<Element> services = servicesRootOf(containersElement); + if ( ! services.isPresent()) + return Optional.empty(); + List<Element> contentServices = XML.getChildren(services.get(), "content"); + if ( contentServices.isEmpty() ) return Optional.empty(); + Element contentNodesElementOrNull = XML.getChild(contentServices.get(0), "nodes"); + + NodesSpecification nodesSpec; + if (contentNodesElementOrNull == null) + nodesSpec = NodesSpecification.nonDedicated(1); + else + nodesSpec = NodesSpecification.from(new ModelElement(contentNodesElementOrNull)); + + Map<HostResource, ClusterMembership> hosts = + StorageGroup.provisionHosts(nodesSpec, + contentServices.get(0).getAttribute("id"), + cluster.getRoot().getHostSystem(), + context.getDeployLogger()); + return Optional.of(hosts.keySet().iterator().next()); + } + /** Returns the services element above the given Element, or empty if there is no services element */ private Optional<Element> servicesRootOf(Element element) { Node parent = element.getParentNode(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index 367b6c03968..2b23939d8ed 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.model.HostSystem; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.builder.xml.dom.NodesSpecification; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; +import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.content.engines.PersistenceEngine; @@ -252,13 +253,25 @@ public class StorageGroup { for (XmlNodeBuilder nodeBuilder : nodeBuilders) { storageGroup.nodes.add(nodeBuilder.build(owner, storageGroup)); } - + + if ( ! parent.isPresent() && subGroups.isEmpty() && nodeBuilders.isEmpty()) // no nodes or groups: create single node + storageGroup.nodes.add(buildSingleNode(owner)); + if ( ! parent.isPresent()) owner.redundancy().setTotalNodes(storageGroup.countNodes()); return storageGroup; } + private StorageNode buildSingleNode(ContentCluster parent) { + int distributionKey = 0; + StorageNode sNode = new StorageNode(parent.getStorageNodes(), 1.0, distributionKey , false); + sNode.setHostResource(parent.getHostSystem().getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC)); + PersistenceEngine provider = parent.getPersistence().create(sNode, storageGroup, null); + new Distributor(parent.getDistributorNodes(), distributionKey, null, provider); + return sNode; + } + /** * Builds a storage group for a hosted environment * @@ -340,6 +353,7 @@ public class StorageGroup { } private static class XmlNodeBuilder { + private final ModelElement clusterElement; private final ModelElement element; @@ -364,6 +378,7 @@ public class StorageGroup { * <li>only group is present: This is a nonleaf group</li> * <li>only nodes is present: This is the implicitly specified toplevel leaf group, or a set of groups * specified using a group count attribute. + * <li>Neither element is present: Create a single node. * </ul> */ private GroupBuilder collectGroup(Optional<ModelElement> groupElement, Optional<ModelElement> nodesElement, String name, String index) { @@ -384,13 +399,17 @@ public class StorageGroup { explicitNodes.addAll(collectExplicitNodes(groupElement)); explicitNodes.addAll(collectExplicitNodes(nodesElement)); - if (subGroups.size() > 0 && explicitNodes.size() > 0) - throw new IllegalArgumentException("A group can contain either nodes or groups, but not both."); - - Optional<NodesSpecification> nodeRequirement = - nodesElement.isPresent() && nodesElement.get().getStringAttribute("count") != null ? Optional.of(NodesSpecification.from(nodesElement.get())) : Optional.empty(); - if (nodeRequirement.isPresent() && subGroups.size() > 0) + if (subGroups.size() > 0 && nodesElement.isPresent()) throw new IllegalArgumentException("A group can contain either explicit subgroups or a nodes specification, but not both."); + + Optional<NodesSpecification> nodeRequirement; + if (nodesElement.isPresent() && nodesElement.get().getStringAttribute("count") != null ) // request these nodes + nodeRequirement = Optional.of(NodesSpecification.from(nodesElement.get())); + else if (! nodesElement.isPresent() && subGroups.isEmpty() && owner.getRoot().getDeployState().isHosted()) // request one node + nodeRequirement = Optional.of(NodesSpecification.nonDedicated(1)); + else // Nodes or groups explicitly listed, and/opr not hosted - resolve in GroupBuilder + nodeRequirement = Optional.empty(); + return new GroupBuilder(group, subGroups, explicitNodes, nodeRequirement, deployLogger); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/engines/PersistenceEngine.java b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/PersistenceEngine.java index 29ed02ab2b6..a2ec6a57346 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/engines/PersistenceEngine.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/engines/PersistenceEngine.java @@ -17,8 +17,9 @@ public abstract class PersistenceEngine extends AbstractConfigProducer implement /** * Creates a config producer for the engines provider at a given node. */ - public static interface PersistenceFactory { - public PersistenceEngine create(StorageNode storageNode, StorageGroup parentGroup, ModelElement storageNodeElement); + public interface PersistenceFactory { + + PersistenceEngine create(StorageNode storageNode, StorageGroup parentGroup, ModelElement storageNodeElement); /** * If a write request succeeds on some nodes and fails on others, causing request to @@ -26,7 +27,7 @@ public abstract class PersistenceEngine extends AbstractConfigProducer implement * reverts are supported. (Typically require backend to keep multiple entries of the * same document identifier persisted at the same time) */ - public boolean supportRevert(); + boolean supportRevert(); /** * Multi level splitting can increase split performance a lot where documents have been @@ -34,8 +35,10 @@ public abstract class PersistenceEngine extends AbstractConfigProducer implement * is cheap. Backends where split is cheaper than fetching document identifiers will * not want to enable multi level splitting. */ - public boolean enableMultiLevelSplitting(); + boolean enableMultiLevelSplitting(); + + ContentCluster.DistributionMode getDefaultDistributionMode(); - public ContentCluster.DistributionMode getDefaultDistributionMode(); } + } diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc index a1fef689253..4155123b09a 100644 --- a/config-model/src/main/resources/schema/content.rnc +++ b/config-model/src/main/resources/schema/content.rnc @@ -112,7 +112,8 @@ Content = element content { # Here you can add document definitions that you also want to handle. # Search might want to know of them in advance. Documents? & - (ContentNodes | TopGroup) & + ContentNodes? & + TopGroup? & Controllers? } diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 1d0b4609ec3..b29ec5f8d0e 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -1060,8 +1060,7 @@ public class ModelProvisioningTest { public void testUsingHostaliasWithProvisioner() { String services = "<?xml version='1.0' encoding='utf-8' ?>\n" + - "<services>\n" + - "\n" + + "<services>" + "<admin version='2.0'>" + " <adminserver hostalias='node1'/>\n"+ "</admin>\n" + @@ -1074,9 +1073,8 @@ public class ModelProvisioningTest { " </nodes>" + "</jdisc>" + "</services>"; - int numberOfHosts = 1; VespaModelTester tester = new VespaModelTester(); - tester.addHosts(numberOfHosts); + tester.addHosts(1); VespaModel model = tester.createModel(services, true); assertEquals(1, model.getRoot().getHostSystem().getHosts().size()); assertEquals(1, model.getAdmin().getSlobroks().size()); @@ -1098,6 +1096,100 @@ public class ModelProvisioningTest { assertThat(model.getContainerClusters().size(), is(1)); } + @Test + public void testNoNodeTagMeans1Node() { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <jdisc id='foo' version='1.0'>" + + " <search/>" + + " <document-api/>" + + " </jdisc>" + + " <content version='1.0' id='bar'>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " </content>" + + "</services>"; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(1); + VespaModel model = tester.createModel(services, true); + assertEquals(1, model.getRoot().getHostSystem().getHosts().size()); + assertEquals(1, model.getAdmin().getSlobroks().size()); + assertEquals(1, model.getContainerClusters().get("foo").getContainers().size()); + assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes()); + } + + @Test + public void testNoNodeTagMeans1NodeNoContent() { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <jdisc id='foo' version='1.0'>" + + " <search/>" + + " <document-api/>" + + " </jdisc>" + + "</services>"; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(1); + VespaModel model = tester.createModel(services, true); + assertEquals(1, model.getRoot().getHostSystem().getHosts().size()); + assertEquals(1, model.getAdmin().getSlobroks().size()); + assertEquals(1, model.getContainerClusters().get("foo").getContainers().size()); + } + + @Test + public void testNoNodeTagMeans1NodeNonHosted() { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <jdisc id='foo' version='1.0'>" + + " <search/>" + + " <document-api/>" + + " </jdisc>" + + " <content version='1.0' id='bar'>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " </content>" + + "</services>"; + VespaModelTester tester = new VespaModelTester(); + tester.setHosted(false); + tester.addHosts(1); + VespaModel model = tester.createModel(services, true); + assertEquals(1, model.getRoot().getHostSystem().getHosts().size()); + assertEquals(1, model.getAdmin().getSlobroks().size()); + assertEquals(1, model.getContainerClusters().get("foo").getContainers().size()); + assertEquals(1, model.getContentClusters().get("bar").getRootGroup().recursiveGetNodes().size()); + } + + @Test + public void testSingleNodeNonHosted() { + String services = + "<?xml version='1.0' encoding='utf-8' ?>\n" + + "<services>" + + " <jdisc id='foo' version='1.0'>" + + " <search/>" + + " <document-api/>" + + " <nodes><node hostalias='foo'/></nodes>"+ + " </jdisc>" + + " <content version='1.0' id='bar'>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes><node hostalias='foo' distribution-key='0'/></nodes>"+ + " </content>" + + "</services>"; + VespaModelTester tester = new VespaModelTester(); + tester.setHosted(false); + tester.addHosts(1); + VespaModel model = tester.createModel(services, true); + assertEquals(1, model.getRoot().getHostSystem().getHosts().size()); + assertEquals(1, model.getAdmin().getSlobroks().size()); + assertEquals(1, model.getContainerClusters().get("foo").getContainers().size()); + assertEquals(1, model.getContentClusters().get("bar").getRootGroup().countNodes()); + } + /** Recreate the combination used in some factory tests */ @Test public void testMultitenantButNotHosted() { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java index d886b0feee3..e74713ce9fa 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DistributorTest.java @@ -24,7 +24,7 @@ public class DistributorTest { ContentCluster parseCluster(String xml) { try { - final List<String> searchDefs = ApplicationPackageUtils.generateSearchDefinitions("music", "movies", "bunnies"); + List<String> searchDefs = ApplicationPackageUtils.generateSearchDefinitions("music", "movies", "bunnies"); MockRoot root = ContentClusterUtils.createMockRoot(searchDefs); return ContentClusterUtils.createCluster(xml, root); } catch (Exception e) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index 0c57d036998..bedbe71e389 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -4,11 +4,13 @@ package com.yahoo.vespa.model.test; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.ConfigModelRegistry; import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.api.HostProvisioner; import com.yahoo.config.model.deploy.DeployProperties; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.provision.Host; import com.yahoo.config.model.provision.Hosts; import com.yahoo.config.model.provision.InMemoryProvisioner; +import com.yahoo.config.model.provision.SingleNodeProvisioner; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; @@ -36,6 +38,8 @@ import java.util.Map; public class VespaModelTester { private final ConfigModelRegistry configModelRegistry; + + private boolean hosted = true; private Map<String, Collection<Host>> hosts = new HashMap<>(); public VespaModelTester() { @@ -57,6 +61,9 @@ public class VespaModelTester { return new Hosts(hosts); } + /** Sets whether this sets up a model for a hosted system. Default: true */ + public void setHosted(boolean hosted) { this.hosted = hosted; } + /** Creates a model which uses 0 as start index and fails on out of capacity */ public VespaModel createModel(String services, String ... retiredHostNames) { return createModel(services, true, retiredHostNames); @@ -76,10 +83,15 @@ public class VespaModelTester { public VespaModel createModel(String services, boolean failOnOutOfCapacity, int startIndexForClusters, String ... retiredHostNames) { VespaModelCreatorWithMockPkg modelCreatorWithMockPkg = new VespaModelCreatorWithMockPkg(null, services, ApplicationPackageUtils.generateSearchDefinition("type1")); ApplicationPackage appPkg = modelCreatorWithMockPkg.appPkg; + + HostProvisioner provisioner = hosted ? + new InMemoryProvisioner(hosts, failOnOutOfCapacity, startIndexForClusters, retiredHostNames) : + new SingleNodeProvisioner(); + DeployState deployState = new DeployState.Builder() .applicationPackage(appPkg) - .modelHostProvisioner(new InMemoryProvisioner(hosts, failOnOutOfCapacity, startIndexForClusters, retiredHostNames)) - .properties((new DeployProperties.Builder()).hostedVespa(true).build()).build(); + .modelHostProvisioner(provisioner) + .properties((new DeployProperties.Builder()).hostedVespa(hosted).build()).build(); return modelCreatorWithMockPkg.create(false, deployState, configModelRegistry); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/IdentifierTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/IdentifierTestCase.java index 69840bff19e..cd1cfec85b2 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/IdentifierTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/parser/IdentifierTestCase.java @@ -9,17 +9,16 @@ import java.util.List; import static org.junit.Assert.assertEquals; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ public class IdentifierTestCase { @Test public void requireThatThereAreNoReservedWords() throws ParseException { List<String> tokens = Arrays.asList("attribute", - "base64_decode", - "base64_encode", + "base64decode", + "base64encode", "clear_state", - "compact_phrase", "create_if_non_existent", "echo", "exact", @@ -35,7 +34,7 @@ public class IdentifierTestCase { "index", "join", "linguistics", - "lower_case", + "lowercase", "ngram", "normalize", "now", @@ -47,7 +46,7 @@ public class IdentifierTestCase { "remove_ctrl_chars", "remove_if_zero", "remove_so_si", - "select_field", + "select_input", "set_language", "set_var", "split", @@ -71,4 +70,5 @@ public class IdentifierTestCase { assertEquals(str, parser.identifier()); } } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/package-info.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/package-info.java index f85d8ae2924..5859a3a5bee 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/package-info.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/package-info.java @@ -4,4 +4,4 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.osgi.annotation.ExportPackage; -/** Implements the provisioning API to perform node provisioning form a node repository */
\ No newline at end of file +/** Implements the provisioning API to perform node provisioning from a node repository */
\ No newline at end of file diff --git a/sample-apps/boolean-search/README.md b/sample-apps/boolean-search/README.md new file mode 100644 index 00000000000..a13f54b7fb5 --- /dev/null +++ b/sample-apps/boolean-search/README.md @@ -0,0 +1,20 @@ +Boolean Search +================== + +Boolean Search and how to feed and query is described in +[boolean search](https://git.corp.yahoo.com/pages/vespa/documentation/documentation/boolean-search.html). + +Adding boolean search to an application is easy. Just add a field of +type predicate to the .sd-file. (Remember to set the arity parameter.) + + +### Feed and search +1. **Feed** the data that is to be searched: + ```sh + curl -X POST --data-binary @adsdata.xml <endpoint url>/document + ``` + +2. **Search** using yql expressions, e.g. `select * from sources * where predicate(target, {"name":"Wile E. Coyote"},{});` + ```sh +curl "<endpoint url>/search/?query=sddocname:ad&yql=select%20*%20from%20sources%20*%20where%20predicate(target%2C%20%7B%22name%22%3A%22Wile%20E.%20Coyote%22%7D%2C%7B%7D)%3B" + ``` diff --git a/sample-apps/boolean-search/adsdata.xml b/sample-apps/boolean-search/adsdata.xml new file mode 100644 index 00000000000..70c7519d902 --- /dev/null +++ b/sample-apps/boolean-search/adsdata.xml @@ -0,0 +1,24 @@ +<?xml version="1.0" encoding="utf-8"?> +<vespafeed> + + <document type="ad" documentid="id:sampleapp:ad::http://example.com/sled"> + <title>ACME Rocket Sled</title> + <target>name in ['Wile E. Coyote'] or age in [14..25]</target> + </document> + + <document type="ad" documentid="id:sampleapp:ad::http://example.com/boulder"> + <title>ACME Dehydrated Boulders</title> + <target>name in ['Wile E. Coyote'] or age in [10..20]</target> + </document> + + <document type="ad" documentid="id:sampleapp:ad::http://example.com/tornado"> + <title>ACME Do-It Yourself Tornado Kit</title> + <target>name in ['Wile E. Coyote'] or age in [20..40]</target> + </document> + + <document type="ad" documentid="id:sampleapp:ad::http://example.com/seed"> + <title>Bird seeds</title> + <target>name in ['Road Runner']</target> + </document> + +</vespafeed> diff --git a/sample-apps/boolean-search/pom.xml b/sample-apps/boolean-search/pom.xml new file mode 100644 index 00000000000..5c6ad07fd28 --- /dev/null +++ b/sample-apps/boolean-search/pom.xml @@ -0,0 +1,82 @@ +<?xml version="1.0"?> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <groupId>com.yahoo.example</groupId> + <artifactId>boolean-application</artifactId> + <packaging>container-plugin</packaging> + <version>1.0.0</version> + <name>application</name> + + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <test.hide>true</test.hide> + <vespa_version>6-SNAPSHOT</vespa_version> + </properties> + + + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.11</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>application</artifactId> <!-- Is this needed? --> + <version>${vespa_version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container</artifactId> <!-- not container-dev --> + <version>${vespa_version}</version> + <scope>provided</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <optimize>true</optimize> + <showDeprecation>true</showDeprecation> + <showWarnings>true</showWarnings> + <source>1.8</source> + <target>1.8</target> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>2.13</version> + <configuration> + <systemPropertyVariables> + <isMavenSurefirePlugin>true</isMavenSurefirePlugin> + </systemPropertyVariables> + <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> + </configuration> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-application-maven-plugin</artifactId> <!-- Zip the application package --> + <version>${vespa_version}</version> + <executions> + <execution> + <goals> + <goal>packageApplication</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + </plugin> + </plugins> + </build> + +</project> diff --git a/sample-apps/boolean-search/src/main/application/deployment.xml b/sample-apps/boolean-search/src/main/application/deployment.xml new file mode 100644 index 00000000000..454e2cb861d --- /dev/null +++ b/sample-apps/boolean-search/src/main/application/deployment.xml @@ -0,0 +1,4 @@ +<deployment version='1.0'> + <test /> + <staging /> +</deployment> diff --git a/sample-apps/boolean-search/src/main/application/searchdefinitions/ad.sd b/sample-apps/boolean-search/src/main/application/searchdefinitions/ad.sd new file mode 100644 index 00000000000..704c9712bcb --- /dev/null +++ b/sample-apps/boolean-search/src/main/application/searchdefinitions/ad.sd @@ -0,0 +1,22 @@ +search ad { + + document ad { + + field title type string { + indexing: index | summary + } + + field target type predicate { + indexing: attribute | summary + index { + arity: 8 + } + } + } + + rank-profile default { + summary-features: subqueries(target).lsb subqueries(target).msb + } + +} + diff --git a/sample-apps/boolean-search/src/main/application/services.xml b/sample-apps/boolean-search/src/main/application/services.xml new file mode 100644 index 00000000000..e19a46b055a --- /dev/null +++ b/sample-apps/boolean-search/src/main/application/services.xml @@ -0,0 +1,34 @@ +<?xml version='1.0' encoding='UTF-8'?> +<services version='1.0'> + + <admin version='3.0'> + <nodes count="1" /> + </admin> + + <jdisc version='1.0' id='default'> + <search> + <config name='container.search.legacy-emulation'> + <string-backed-feature-data>false</string-backed-feature-data> + <string-backed-structured-data>false</string-backed-structured-data> + </config> + <chain id="default" inherits="vespa"> + <searcher id="com.yahoo.example.SubqueriesSearcher" bundle="boolean-application"/> + </chain> + </search> + <document-api /> + <nodes count="1" /> + </jdisc> + + <content version="1.0" id="adserver"> + <redundancy>1</redundancy> + <documents> + <document type="ad" mode="index" /> + </documents> + <nodes count="1" /> + <engine> + <proton> + <searchable-copies>1</searchable-copies> + </proton> + </engine> + </content> +</services> diff --git a/sample-apps/boolean-search/src/main/java/com/yahoo/example/SubqueriesSearcher.java b/sample-apps/boolean-search/src/main/java/com/yahoo/example/SubqueriesSearcher.java new file mode 100644 index 00000000000..0243cd4e431 --- /dev/null +++ b/sample-apps/boolean-search/src/main/java/com/yahoo/example/SubqueriesSearcher.java @@ -0,0 +1,52 @@ +package com.yahoo.example; + +import com.yahoo.data.access.Inspectable; +import com.yahoo.data.access.Inspector; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +/** + * A searcher that reads "subqueries" information for field "target", + * and creates a new field "subqueries" which holds a 64-bit subquery bitmap. + * + * @author Joe Developer + */ +public class SubqueriesSearcher extends Searcher { + public SubqueriesSearcher() { + } + + public Result search(Query query, Execution execution) { + Result result = execution.search(query); + execution.fill(result); + for (Hit hit : result.hits().asList()) { + if (hit.isMeta()) continue; + simplifySubqueriesFor("target", hit); + hit.removeField("summaryfeatures"); + } + return result; + } + + /** + * Reads summaryfeatures for subqueries(field) and adds a + * new field "subqueries(field)" with a 64-bit subquery bitmap + * @param field Field to read subqueries for + * @param hit Hit to read summaryfeatures from and update with the new field + */ + private void simplifySubqueriesFor(String field, Hit hit) { + Object o = hit.getField("summaryfeatures"); + if (o instanceof Inspectable) { + String subqueriesName = "subqueries(" + field + ")"; + + Inspectable summaryfeatures = (Inspectable) o; + Inspector obj = summaryfeatures.inspect(); + long lsb = obj.field(subqueriesName).asLong(0); // The .lsb suffix is optional, so read both with and without. + lsb |= obj.field(subqueriesName + ".lsb").asLong(0); + long msb = obj.field(subqueriesName + ".msb").asLong(0); + + hit.setField(subqueriesName, msb << 32 | lsb); + } + } +} diff --git a/sample-apps/boolean-search/src/test/application/services.xml b/sample-apps/boolean-search/src/test/application/services.xml new file mode 100644 index 00000000000..1444a26c59f --- /dev/null +++ b/sample-apps/boolean-search/src/test/application/services.xml @@ -0,0 +1,19 @@ +<?xml version='1.0' encoding='UTF-8'?> +<services version='1.0'> + <admin version='3.0'> + <nodes count="1" /> + </admin> + + <!-- duplication of src/main/application due to Ticket 6904654 --> + <jdisc version='1.0' id='default'> + <search> + <chain id="default" inherits="vespa"> + <searcher id="com.yahoo.example.SubqueriesSearcher" bundle="boolean-application"> + </searcher> + <searcher id="com.yahoo.example.MockBackend" bundle="boolean-application"> + </searcher> + </chain> + </search> + <nodes count="1" /> + </jdisc> +</services> diff --git a/sample-apps/boolean-search/src/test/java/com/yahoo/example/MockBackend.java b/sample-apps/boolean-search/src/test/java/com/yahoo/example/MockBackend.java new file mode 100644 index 00000000000..0c92eec1c2f --- /dev/null +++ b/sample-apps/boolean-search/src/test/java/com/yahoo/example/MockBackend.java @@ -0,0 +1,31 @@ +package com.yahoo.example; + +import com.yahoo.data.access.simple.Value; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.result.Hit; +import com.yahoo.search.searchchain.Execution; + +/** + * @author Joe Developer +*/ +public class MockBackend extends Searcher { + + public MockBackend() { + } + + public @Override + Result search(Query query,Execution execution) { + Result result=new Result(query); + for (int i = 0; i < 3; ++i) { + Hit hit = new Hit("mock-hit:" + i); + Value.ObjectValue summaryfeatures = new Value.ObjectValue(); + summaryfeatures.put("subqueries(target).lsb", new Value.LongValue(0x3)); + summaryfeatures.put("subqueries(target).msb", new Value.LongValue(0x1)); + hit.setField("summaryfeatures", summaryfeatures); + result.hits().add(hit); + } + return result; + } +} diff --git a/sample-apps/boolean-search/src/test/java/com/yahoo/example/SubqueriesSearcherTest.java b/sample-apps/boolean-search/src/test/java/com/yahoo/example/SubqueriesSearcherTest.java new file mode 100644 index 00000000000..ea29533081a --- /dev/null +++ b/sample-apps/boolean-search/src/test/java/com/yahoo/example/SubqueriesSearcherTest.java @@ -0,0 +1,37 @@ +package com.yahoo.example; + +import com.yahoo.application.Application; +import com.yahoo.application.Networking; +import com.yahoo.application.container.Search; +import com.yahoo.component.ComponentSpecification; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import org.junit.Test; + +import java.nio.file.Paths; + +import static org.junit.Assert.assertEquals; + +/** + * @author Joe Developer + */ +public class SubqueriesSearcherTest { + @Test + public void hit_is_added() throws Exception { + try (Application app = Application.fromApplicationPackage( + Paths.get("src/test/application"), + Networking.disable)) + { + Search search = app.getJDisc("jdisc").search(); + Result result = search.process(ComponentSpecification.fromString("default"), new Query("?query=ignored")); + + assertEquals(3, result.hits().size()); + Hit hit = result.hits().get(0); + + assertEquals(null, hit.getField("summaryfeatures")); // Summaryfeatures was removed by searcher + assertEquals(0x100000003L, hit.getField("subqueries(target)")); + } + } + +} diff --git a/sample-apps/boolean-search/src/test/resources/adsdata.xml b/sample-apps/boolean-search/src/test/resources/adsdata.xml new file mode 100644 index 00000000000..e4c23ae983a --- /dev/null +++ b/sample-apps/boolean-search/src/test/resources/adsdata.xml @@ -0,0 +1,24 @@ +<?xml version="1.0" encoding="utf-8"?> +<vespafeed> + + <document type="ad" documentid="id:sampleapp:ad::http://example.com/sled"> + <title>ACME Rocket Sled</title> + <target>name in ['Wile E. Coyote'] or age in [14..25]</target> + </document> + + <document type="ad" documentid="id:sampleapp:ad::http://example.com/boulder"> + <title>ACME Dehydrated Boulders</title> + <target>name in ['Wile E. Coyote'] or age in [10..20]</target> + </document> + + <document type="ad" documentid="id:sampleapp:ad::http://example.com/tornado"> + <title>ACME Do-It Yourself Tornado Kit</title> + <target>name in ['Wile E. Coyote'] or age in [20..40]</target> + </document> + + <document type="ad" documentid="id:sampleapp:ad::http://example.com/seed"> + <title>Bird seeds</title> + <target>name in ['Road Runner']</target> + </document> + +</vespafeed> |