diff options
author | Jon Bratseth <bratseth@oath.com> | 2018-09-28 12:01:47 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@oath.com> | 2018-09-28 12:01:47 +0200 |
commit | a0494eede7b7fa9e9c0abbef1b263d6a11de5c64 (patch) | |
tree | c0f390c31de0cd2ddb27e181a3ea100a36fe9940 | |
parent | 2aa1a397cf63b7db49866056341311a8e501f270 (diff) | |
parent | 157c8b77cfe432d0476f91eb87780e6b1fcd499f (diff) |
Merge with master
342 files changed, 3019 insertions, 3859 deletions
diff --git a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java index d68b36e063c..16accb368fd 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application; +import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.log.LogLevel; @@ -10,8 +11,16 @@ import org.w3c.dom.Element; import org.w3c.dom.NamedNodeMap; import javax.xml.transform.TransformerException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.logging.Logger; +import java.util.stream.Collectors; /** * Handles overrides in a XML document according to the rules defined for multi environment application packages. @@ -23,8 +32,7 @@ import java.util.logging.Logger; * 3. When multiple XML elements with the same name is specified (i.e. when specifying search or docproc chains), * the id attribute of the element is used together with the element name when applying directives * - * @author lulf - * @since 5.22 + * @author Ulf Lilleengen */ class OverrideProcessor implements PreProcessor { @@ -71,15 +79,13 @@ class OverrideProcessor implements PreProcessor { } private Context getParentContext(Element parent, Context context) { - Optional<Environment> environment = context.environment; - RegionName region = context.region; - if ( ! environment.isPresent()) { - environment = getEnvironment(parent); - } - if (region.isDefault()) { - region = getRegion(parent); - } - return Context.create(environment, region); + Set<Environment> environments = context.environments; + Set<RegionName> regions = context.regions; + if (environments.isEmpty()) + environments = getEnvironments(parent); + if (regions.isEmpty()) + regions = getRegions(parent); + return Context.create(environments, regions); } /** @@ -100,13 +106,13 @@ class OverrideProcessor implements PreProcessor { */ private void checkConsistentInheritance(List<Element> children, Context context) { for (Element child : children) { - Optional<Environment> env = getEnvironment(child); - RegionName reg = getRegion(child); - if (env.isPresent() && context.environment.isPresent() && !env.equals(context.environment)) { - throw new IllegalArgumentException("Environment in child (" + env.get() + ") differs from that inherited from parent (" + context.environment + ") at " + child); + Set<Environment> environments = getEnvironments(child); + Set<RegionName> regions = getRegions(child); + if ( ! environments.isEmpty() && ! context.environments.isEmpty() && !environments.equals(context.environments)) { + throw new IllegalArgumentException("Environments in child (" + environments + ") differs from that inherited from parent (" + context.environments + ") at " + child); } - if (!reg.isDefault() && !context.region.isDefault() && !reg.equals(context.region)) { - throw new IllegalArgumentException("Region in child (" + reg + ") differs from that inherited from parent (" + context.region + ") at " + child); + if ( ! regions.isEmpty() && ! context.regions.isEmpty() && ! regions.equals(context.regions)) { + throw new IllegalArgumentException("Regions in child (" + regions + ") differs from that inherited from parent (" + context.regions + ") at " + child); } } } @@ -118,22 +124,24 @@ class OverrideProcessor implements PreProcessor { Iterator<Element> elemIt = children.iterator(); while (elemIt.hasNext()) { Element child = elemIt.next(); - if ( ! matches(getEnvironment(child), getRegion(child))) { + if ( ! matches(getEnvironments(child), getRegions(child))) { parent.removeChild(child); elemIt.remove(); } } } - private boolean matches(Optional<Environment> elementEnvironment, RegionName elementRegion) { - if (elementEnvironment.isPresent()) { // match environment - if (! environment.equals(elementEnvironment.get())) return false; + private boolean matches(Set<Environment> elementEnvironments, Set<RegionName> elementRegions) { + if ( ! elementEnvironments.isEmpty()) { // match environment + if ( ! elementEnvironments.contains(environment)) return false; } - if ( ! elementRegion.isDefault()) { // match region - if ( ! region.equals(elementRegion)) return false; - // match region but no environment in prod only to avoid a region attribute overriding capacity policies outside prod - if ( ! elementEnvironment.isPresent() && ! environment.equals(Environment.prod)) return false; + if ( ! elementRegions.isEmpty()) { // match region + // match region in prod only + if ( environment.equals(Environment.prod) && ! elementRegions.contains(region)) return false; + + // explicit region implies prod + if ( ! environment.equals(Environment.prod) && elementEnvironments.isEmpty() ) return false; } return true; @@ -174,11 +182,11 @@ class OverrideProcessor implements PreProcessor { private int getNumberOfOverrides(Element child, Context context) { int currentMatch = 0; - Optional<Environment> elementEnvironment = hasEnvironment(child) ? getEnvironment(child) : context.environment; - RegionName elementRegion = hasRegion(child) ? getRegion(child) : context.region; - if (elementEnvironment.isPresent() && elementEnvironment.get().equals(environment)) + Set<Environment> elementEnvironments = hasEnvironment(child) ? getEnvironments(child) : context.environments; + Set<RegionName> elementRegions = hasRegion(child) ? getRegions(child) : context.regions; + if ( ! elementEnvironments.isEmpty() && elementEnvironments.contains(environment)) currentMatch++; - if ( ! elementRegion.isDefault() && elementRegion.equals(region)) + if ( ! elementRegions.isEmpty() && elementRegions.contains(region)) currentMatch++; return currentMatch; } @@ -220,20 +228,16 @@ class OverrideProcessor implements PreProcessor { return element.hasAttributeNS(XmlPreProcessor.deployNamespaceUri, ATTR_ENV); } - private Optional<Environment> getEnvironment(Element element) { + private Set<Environment> getEnvironments(Element element) { String env = element.getAttributeNS(XmlPreProcessor.deployNamespaceUri, ATTR_ENV); - if (env == null || env.isEmpty()) { - return Optional.empty(); - } - return Optional.of(Environment.from(env)); + if (env == null || env.isEmpty()) return Collections.emptySet(); + return Arrays.stream(env.split(" ")).map(Environment::from).collect(Collectors.toSet()); } - private RegionName getRegion(Element element) { + private Set<RegionName> getRegions(Element element) { String reg = element.getAttributeNS(XmlPreProcessor.deployNamespaceUri, ATTR_REG); - if (reg == null || reg.isEmpty()) { - return RegionName.defaultName(); - } - return RegionName.from(reg); + if (reg == null || reg.isEmpty()) return Collections.emptySet(); + return Arrays.stream(reg.split(" ")).map(RegionName::from).collect(Collectors.toSet()); } private Map<String, List<Element>> elementsByTagNameAndId(List<Element> children) { @@ -287,21 +291,21 @@ class OverrideProcessor implements PreProcessor { */ private static final class Context { - final Optional<Environment> environment; + final ImmutableSet<Environment> environments; - final RegionName region; + final ImmutableSet<RegionName> regions; - private Context(Optional<Environment> environment, RegionName region) { - this.environment = environment; - this.region = region; + private Context(Set<Environment> environments, Set<RegionName> regions) { + this.environments = ImmutableSet.copyOf(environments); + this.regions = ImmutableSet.copyOf(regions); } static Context empty() { - return new Context(Optional.empty(), RegionName.defaultName()); + return new Context(ImmutableSet.of(), ImmutableSet.of()); } - public static Context create(Optional<Environment> environment, RegionName region) { - return new Context(environment, region); + public static Context create(Set<Environment> environments, Set<RegionName> regions) { + return new Context(environments, regions); } } diff --git a/config-application-package/src/main/java/com/yahoo/config/application/PreProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/PreProcessor.java index d331c4432bc..f207a07d3be 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/PreProcessor.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/PreProcessor.java @@ -9,9 +9,10 @@ import java.io.IOException; /** * Performs pre-processing of XML document and returns new document that has been processed. * - * @author lulf - * @since 5.21 + * @author Ulf Lilleengen */ public interface PreProcessor { - public Document process(Document input) throws IOException, TransformerException; + + Document process(Document input) throws IOException, TransformerException; + } diff --git a/config-application-package/src/main/java/com/yahoo/config/application/PropertiesProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/PropertiesProcessor.java index 6e45460a4c4..65120c3677c 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/PropertiesProcessor.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/PropertiesProcessor.java @@ -16,7 +16,6 @@ import java.util.logging.Logger; * Handles getting properties from services.xml and replacing references to properties with their real values * * @author hmusum - * @since 5.22 */ class PropertiesProcessor implements PreProcessor { private final static Logger log = Logger.getLogger(PropertiesProcessor.class.getName()); @@ -82,8 +81,8 @@ class PropertiesProcessor implements PreProcessor { } private String replaceValue(String propertyValue) { - /* Use a list with keys sorted by length (longest key first) - Needed for replacing values where you have overlapping keys */ + // Use a list with keys sorted by length (longest key first) + // Needed for replacing values where you have overlapping keys ArrayList<String> keys = new ArrayList<>(properties.keySet()); Collections.sort(keys, Collections.reverseOrder(Comparator.comparing(String::length))); diff --git a/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java index ceef0a6730e..0bb160319c0 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/XmlPreProcessor.java @@ -22,7 +22,6 @@ import java.util.List; * and create a new Document which is based on the supplied environment and region * * @author hmusum - * @since 5.22 */ public class XmlPreProcessor { @@ -51,7 +50,7 @@ public class XmlPreProcessor { public Document run() throws ParserConfigurationException, IOException, SAXException, TransformerException { DocumentBuilder docBuilder = Xml.getPreprocessDocumentBuilder(); - final Document document = docBuilder.parse(new InputSource(xmlInput)); + Document document = docBuilder.parse(new InputSource(xmlInput)); return execute(document); } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 7ca9bcf48f3..1724bad765f 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -305,6 +305,7 @@ public class FilesApplicationPackage implements ApplicationPackage { /** * Verify that two sets of search definitions are disjoint (TODO: everything except error message is very generic). + * * @param fileSds Set of search definitions from file * @param bundleSds Set of search definitions from bundles */ diff --git a/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java index d3c2b672ee5..a456924673d 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/IncludeProcessorTest.java @@ -13,44 +13,48 @@ import java.io.*; import java.nio.file.NoSuchFileException; /** - * @author lulf - * @since 5.22 + * @author Ulf Lilleengen */ public class IncludeProcessorTest { @Test - public void testInclude() throws IOException, SAXException, XMLStreamException, ParserConfigurationException, TransformerException { + public void testInclude() throws IOException, SAXException, ParserConfigurationException, TransformerException { File app = new File("src/test/resources/multienvapp"); DocumentBuilder docBuilder = Xml.getPreprocessDocumentBuilder(); - String expected = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?><services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" + + String expected = + "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" + " <preprocess:properties>\n" + " <qrs.port>4099</qrs.port>\n" + " <qrs.port>5000</qrs.port>\n" + " </preprocess:properties>\n" + " <preprocess:properties deploy:environment='prod'>\n" + " <qrs.port deploy:region='us-west'>5001</qrs.port>" + - " <qrs.port deploy:region='us-east'>5002</qrs.port>" + + " <qrs.port deploy:region='us-east us-central'>5002</qrs.port>" + " </preprocess:properties>\n" + " <admin version=\"2.0\">\n" + " <adminserver hostalias=\"node0\"/>\n" + " </admin>\n" + - " <admin deploy:environment=\"prod\" version=\"2.0\">\n" + + " <admin deploy:environment=\"staging prod\" deploy:region=\"us-east us-central\" version=\"2.0\">\n" + " <adminserver hostalias=\"node1\"/>\n" + " </admin>\n" + " <content id=\"foo\" version=\"1.0\">\n" + " <redundancy>1</redundancy><documents>\n" + " <document mode=\"index\" type=\"music.sd\"/>\n" + - "</documents><nodes>\n" + + " </documents><nodes>\n" + " <node distribution-key=\"0\" hostalias=\"node0\"/>\n" + - "</nodes><nodes deploy:environment=\"prod\">\n" + + " </nodes>" + + " <nodes deploy:environment=\"prod\">\n" + " <node distribution-key=\"0\" hostalias=\"node0\"/>\n" + " <node distribution-key=\"1\" hostalias=\"node1\"/>\n" + - "</nodes><nodes deploy:environment=\"prod\" deploy:region=\"us-west\">\n" + + " </nodes>" + + " <nodes deploy:environment=\"prod\" deploy:region=\"us-west\">\n" + " <node distribution-key=\"0\" hostalias=\"node0\"/>\n" + " <node distribution-key=\"1\" hostalias=\"node1\"/>\n" + " <node distribution-key=\"2\" hostalias=\"node2\"/>\n" + - "</nodes></content>\n" + + " </nodes>" + + "</content>\n" + "<jdisc id=\"stateless\" version=\"1.0\">\n" + " <search deploy:environment=\"prod\">\n" + " <chain id=\"common\">\n" + @@ -68,7 +72,7 @@ public class IncludeProcessorTest { " </nodes>\n" + "</jdisc></services>"; - Document doc = (new IncludeProcessor(app)).process(docBuilder.parse(Xml.getServices(app))); + Document doc = new IncludeProcessor(app).process(docBuilder.parse(Xml.getServices(app))); // System.out.println(Xml.documentAsString(doc)); TestBase.assertDocument(expected, doc); } @@ -77,7 +81,7 @@ public class IncludeProcessorTest { public void testRequiredIncludeIsDefault() throws ParserConfigurationException, IOException, SAXException, TransformerException { File app = new File("src/test/resources/multienvapp_failrequired"); DocumentBuilder docBuilder = Xml.getPreprocessDocumentBuilder(); - (new IncludeProcessor(app)).process(docBuilder.parse(Xml.getServices(app))); + new IncludeProcessor(app).process(docBuilder.parse(Xml.getServices(app))); } } diff --git a/config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java b/config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java index b20437bc259..3827fe2ad42 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/XmlPreprocessorTest.java @@ -23,8 +23,10 @@ public class XmlPreprocessorTest { private static final File services = new File(appDir, "services.xml"); @Test - public void testPreProcessing() throws IOException, SAXException, XMLStreamException, ParserConfigurationException, TransformerException { - String expectedDev = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?><services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" + + public void testPreProcessing() throws IOException, SAXException, ParserConfigurationException, TransformerException { + String expectedDev = + "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" + " <admin version=\"2.0\">\n" + " <adminserver hostalias=\"node0\"/>\n" + " </admin>\n" + @@ -46,14 +48,40 @@ public class XmlPreprocessorTest { " </nodes>\n" + " </jdisc>\n" + "</services>"; + TestBase.assertDocument(expectedDev, new XmlPreProcessor(appDir, services, Environment.dev, RegionName.from("default")).run()); - Document docDev = (new XmlPreProcessor(appDir, services, Environment.dev, RegionName.from("default")).run()); - TestBase.assertDocument(expectedDev, docDev); - + String expectedStaging = + "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" + + " <admin version=\"2.0\">\n" + + " <adminserver hostalias=\"node1\"/>\n" + // Difference from dev: node1 + " </admin>\n" + + " <content id=\"foo\" version=\"1.0\">\n" + + " <redundancy>1</redundancy>\n" + + " <documents>\n" + + " <document mode=\"index\" type=\"music.sd\"/>\n" + + " </documents>\n" + + " <nodes>\n" + + " <node distribution-key=\"0\" hostalias=\"node0\"/>\n" + + " </nodes>\n" + + " </content>\n" + + " <jdisc id=\"stateless\" version=\"1.0\">\n" + + " <search/>\n" + + " <component bundle=\"foobundle\" class=\"MyFoo\" id=\"foo\"/>\n" + + "" + // Difference from dev: no TestBar + " <nodes>\n" + + " <node hostalias=\"node0\" baseport=\"5000\"/>\n" + + " </nodes>\n" + + " </jdisc>\n" + + "</services>"; + // System.out.println(Xml.documentAsString(new XmlPreProcessor(appDir, services, Environment.staging, RegionName.from("default")).run())); + TestBase.assertDocument(expectedStaging, new XmlPreProcessor(appDir, services, Environment.staging, RegionName.from("default")).run()); - String expectedUsWest = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?><services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" + + String expectedUsWest = + "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" + " <admin version=\"2.0\">\n" + - " <adminserver hostalias=\"node1\"/>\n" + + " <adminserver hostalias=\"node0\"/>\n" + " </admin>\n" + " <content id=\"foo\" version=\"1.0\">\n" + " <redundancy>1</redundancy>\n" + @@ -81,12 +109,11 @@ public class XmlPreprocessorTest { " </nodes>\n" + " </jdisc>\n" + "</services>"; + TestBase.assertDocument(expectedUsWest, new XmlPreProcessor(appDir, services, Environment.prod, RegionName.from("us-west")).run()); - Document docUsWest = (new XmlPreProcessor(appDir, services, Environment.prod, RegionName.from("us-west"))).run(); - // System.out.println(Xml.documentAsString(docUsWest)); - TestBase.assertDocument(expectedUsWest, docUsWest); - - String expectedUsEast = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?><services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" + + String expectedUsEastAndCentral = + "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">\n" + " <admin version=\"2.0\">\n" + " <adminserver hostalias=\"node1\"/>\n" + " </admin>\n" + @@ -115,14 +142,16 @@ public class XmlPreprocessorTest { " </nodes>\n" + " </jdisc>\n" + "</services>"; - - Document docUsEast = (new XmlPreProcessor(appDir, services, Environment.prod, RegionName.from("us-east"))).run(); - TestBase.assertDocument(expectedUsEast, docUsEast); + TestBase.assertDocument(expectedUsEastAndCentral, + new XmlPreProcessor(appDir, services, Environment.prod, RegionName.from("us-east")).run()); + TestBase.assertDocument(expectedUsEastAndCentral, + new XmlPreProcessor(appDir, services, Environment.prod, RegionName.from("us-central")).run()); } @Test - public void testPropertiesWithOverlappingNames() throws IOException, SAXException, XMLStreamException, ParserConfigurationException, TransformerException { - String input = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + + public void testPropertiesWithOverlappingNames() throws IOException, SAXException, ParserConfigurationException, TransformerException { + String input = + "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">" + " <preprocess:properties>" + " <sherpa.host>gamma-usnc1.dht.yahoo.com</sherpa.host>" + @@ -146,7 +175,8 @@ public class XmlPreprocessorTest { " </admin>" + "</services>"; - String expectedProd = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + + String expectedProd = + "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>" + "<services xmlns:deploy=\"vespa\" xmlns:preprocess=\"properties\" version=\"1.0\">" + " <config name='a'>" + " <a>36000</a>" + diff --git a/config-application-package/src/test/resources/multienvapp/services.xml b/config-application-package/src/test/resources/multienvapp/services.xml index 3d4a2087c57..d0f43f0b025 100644 --- a/config-application-package/src/test/resources/multienvapp/services.xml +++ b/config-application-package/src/test/resources/multienvapp/services.xml @@ -1,5 +1,6 @@ <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> <services version='1.0' xmlns:deploy="vespa" xmlns:preprocess="properties"> + <preprocess:properties> <qrs.port>4099</qrs.port> <qrs.port>5000</qrs.port> @@ -7,17 +8,23 @@ <preprocess:properties deploy:environment='prod'> <qrs.port deploy:region='us-west'>5001</qrs.port> - <qrs.port deploy:region='us-east'>5002</qrs.port> + <qrs.port deploy:region='us-east us-central'>5002</qrs.port> </preprocess:properties> + <admin version='2.0'> <adminserver hostalias='node0'/> </admin> - <admin version='2.0' deploy:environment='prod'> + + <admin version='2.0' deploy:environment='staging prod' deploy:region='us-east us-central'> <adminserver hostalias='node1'/> </admin> + <preprocess:include file='jdisc.xml'/> + <content version='1.0' id='foo'> <preprocess:include file='content/content_foo.xml'/> </content> + <preprocess:include file='doesnotexist.xml' required='false' /> + </services> diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java index 6c7396d607b..dc6c17e425e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java @@ -41,17 +41,12 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro derive(search); } - private boolean hasFullIndexingDocprocRights(ImmutableSDField field) { - Attribute self = field.getAttributes().get(field.getName()); - return (!field.isExtraField() || ((self != null) && self.isMutable())); - } - @Override protected void derive(ImmutableSDField field, Search search) { if (field.isImportedField()) { return; } - if (hasFullIndexingDocprocRights(field)) { + if (field.hasFullIndexingDocprocRights()) { docFields.add(field.getName()); } if (field.usesStructOrMap() && @@ -93,7 +88,7 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro if (modifiesSelf(exp)) { later.add(exp); } else { - ilscriptBuilder.content(exp.toString()); + ilscriptBuilder.content(exp.toString()); } fieldFetcher.visit(exp); touchedFields.addAll(fieldFetcher.touchedFields()); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java index 125e6b976b5..ff11c2fdf22 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java @@ -169,4 +169,9 @@ public class ImmutableImportedSDField implements ImmutableSDField { return new UnsupportedOperationException("'" + aspect + "' is not meaningful or relevant for an imported field."); } + @Override + public boolean hasFullIndexingDocprocRights() { + return importedField.targetField().hasFullIndexingDocprocRights(); + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java index 70553d4b57c..6fe8a4da92b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java @@ -81,4 +81,5 @@ public interface ImmutableSDField { */ Field asField(); + boolean hasFullIndexingDocprocRights(); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index 0590027a402..e310b6b6858 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -799,4 +799,10 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, return aliasToName; } + @Override + public boolean hasFullIndexingDocprocRights() { + Attribute self = getAttributes().get(getName()); + return (!isExtraField() || ((self != null) && self.isMutable())); + } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java index aeab2bb6638..b0129c0a836 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java @@ -97,13 +97,11 @@ public class IndexingInputs extends Processor { @Override protected void doVisit(Expression exp) { if ( ! (exp instanceof InputExpression)) return; - - SDDocumentType docType = search.getDocument(); String inputField = ((InputExpression)exp).getFieldName(); - if (docType.getField(inputField) != null) return; + if (search.getField(inputField).hasFullIndexingDocprocRights()) return; fail(search, field, "Indexing script refers to field '" + inputField + "' which does not exist " + - "in document type '" + docType.getName() + "'."); + "in document type '" + search.getDocument().getName() + "', and is not a mutable attribute."); } } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java index 15d295736c1..1af2a979cb4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java @@ -31,6 +31,7 @@ public class Processing { UriHack::new, LiteralBoost::new, TagType::new, + ValidateFieldTypesDocumentsOnly::new, IndexingInputs::new, OptimizeIlscript::new, ValidateFieldWithIndexSettingsCreatesIndex::new, diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java index 21b7f1d2675..1cf3d61d8f1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java @@ -29,16 +29,24 @@ public class ValidateFieldTypes extends Processor { @Override public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; + if (!validate) return; String searchName = search.getName(); Map<String, DataType> seenFields = new HashMap<>(); + verifySearchAndDocFields(searchName, seenFields); + verifySummaryFields(searchName, seenFields); + } + + final protected void verifySearchAndDocFields(String searchName, Map<String, DataType> seenFields) { search.allFields().forEach(field -> { checkFieldType(searchName, "index field", field.getName(), field.getDataType(), seenFields); for (Map.Entry<String, Attribute> entry : field.getAttributes().entrySet()) { checkFieldType(searchName, "attribute", entry.getKey(), entry.getValue().getDataType(), seenFields); } }); + + } + final protected void verifySummaryFields(String searchName, Map<String, DataType> seenFields) { for (DocumentSummary summary : search.getSummaries().values()) { for (SummaryField field : summary.getSummaryFields()) { checkFieldType(searchName, "summary field", field.getName(), field.getDataType(), seenFields); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesDocumentsOnly.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesDocumentsOnly.java new file mode 100644 index 00000000000..ff3e638fff1 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesDocumentsOnly.java @@ -0,0 +1,25 @@ +package com.yahoo.searchdefinition.processing; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.DataType; +import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.searchdefinition.Search; +import com.yahoo.vespa.model.container.search.QueryProfiles; + +import java.util.HashMap; +import java.util.Map; + +public class ValidateFieldTypesDocumentsOnly extends ValidateFieldTypes { + public ValidateFieldTypesDocumentsOnly(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + super(search, deployLogger, rankProfileRegistry, queryProfiles); + } + + @Override + public void process(boolean validate, boolean documentsOnly) { + if ( ! validate) return; + + String searchName = search.getName(); + Map<String, DataType> seenFields = new HashMap<>(); + verifySearchAndDocFields(searchName, seenFields); + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 73d77406700..8e7c20d4234 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -349,6 +349,7 @@ public class VespaMetricSet { metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.count.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.count.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.count.rate")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.createiterator.sum.count.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.visit.sum.count.rate")); metrics.add(new Metric("vds.filestor.alldisks.queuesize.average","diskqueuesize")); metrics.add(new Metric("vds.filestor.alldisks.averagequeuewait.sum.average","diskqueuewait")); @@ -356,11 +357,14 @@ public class VespaMetricSet { metrics.add(new Metric("vds.visitor.allthreads.queuesize.count.average")); metrics.add(new Metric("vds.visitor.allthreads.completed.sum.average")); metrics.add(new Metric("vds.visitor.allthreads.created.sum.rate","visit")); + metrics.add(new Metric("vds.visitor.allthreads.averagemessagesendtime.sum.average")); + metrics.add(new Metric("vds.visitor.allthreads.averageprocessingtime.sum.average")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.put.sum.latency.average")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.remove.sum.latency.average")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.get.sum.latency.average")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.update.sum.latency.average")); + metrics.add(new Metric("vds.filestor.alldisks.allthreads.createiterator.sum.latency.average")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.visit.sum.latency.average")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.splitbuckets.count.rate")); metrics.add(new Metric("vds.filestor.alldisks.allthreads.joinbuckets.count.rate")); diff --git a/config-model/src/test/examples/duplicatenamesindoc.sd b/config-model/src/test/examples/duplicatenamesindoc.sd deleted file mode 100644 index 3c55efe953d..00000000000 --- a/config-model/src/test/examples/duplicatenamesindoc.sd +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search duplicatenamesindoc { -document { - field foo type int { - indexing: attribute - } - - field fOo type string { - indexing: index - } -} -} diff --git a/config-model/src/test/examples/duplicatenamesinsearchdifferenttype.sd b/config-model/src/test/examples/duplicatenamesinsearchdifferenttype.sd deleted file mode 100644 index 502a4bb7aff..00000000000 --- a/config-model/src/test/examples/duplicatenamesinsearchdifferenttype.sd +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search duplicatenamesinsearch { -document { - field grpphotoids64 type string { } -} - -field grpphotoids64 type array<long> { - indexing: input grpphotoids64 | split " " | for_each { - base64decode } | attribute - } - -} diff --git a/config-model/src/test/examples/incorrectsummarytypes.sd b/config-model/src/test/examples/incorrectsummarytypes.sd deleted file mode 100644 index 9304ad5ca20..00000000000 --- a/config-model/src/test/examples/incorrectsummarytypes.sd +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -search incorrectsummarytypes { - - document incorrectsummarytypes { - - field somestring type string { - indexing: summary - } - - } - - document-summary incorrect { - - summary somestring type int { - } - - } - -} diff --git a/config-model/src/test/examples/name-check.sd b/config-model/src/test/examples/name-check.sd deleted file mode 100644 index bd7c0713e3a..00000000000 --- a/config-model/src/test/examples/name-check.sd +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# An entry-level configuration. -# You can get a reasonable configuration by only configuring -# a document -search simple { - - document name-check { - - field title type string { - indexing: summary | index - } - - # reserved name, should trigger error - field sddocname type string { - indexing: index - } - - } - -} diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java index cadb2cda324..71db60065aa 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java @@ -165,7 +165,7 @@ public class AttributeSettingsTestCase extends SearchDefinitionTestCase { " }\n" + " }\n" + " field m type long {\n" + - " indexing: 0 | to_long | attribute\n" + + " indexing: attribute\n" + " attribute: mutable\n" + " }\n" + " field f type long {\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java index f11799495a2..627394bb6ea 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java @@ -17,7 +17,18 @@ public class IncorrectSummaryTypesTestCase extends SearchDefinitionTestCase { @Test public void testImportingIncorrect() throws IOException, ParseException { try { - SearchBuilder.buildFromFile("src/test/examples/incorrectsummarytypes.sd"); + SearchBuilder.createFromString( + "search incorrectsummarytypes {\n" + + " document incorrectsummarytypes {\n" + + " field somestring type string {\n" + + " indexing: summary\n" + + " }\n" + + " }\n" + + " document-summary incorrect {\n" + + " summary somestring type int {\n" + + " }\n" + + " }\n" + + "}\n"); fail("processing should have failed"); } catch (RuntimeException e) { assertEquals("'summary somestring type string' in 'destinations(default )' is inconsistent with 'summary somestring type int' in 'destinations(incorrect )': All declarations of the same summary field must have the same type", e.getMessage()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java index c19244a4bf2..b539c65150d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java @@ -22,7 +22,18 @@ public class NameFieldCheckTestCase extends SearchDefinitionTestCase { @Test public void testNameField() throws IOException, ParseException { try { - SearchBuilder.buildFromFile("src/test/examples/name-check.sd"); + SearchBuilder.createFromString( + "search simple {\n" + + " document name-check {\n" + + " field title type string {\n" + + " indexing: summary | index\n" + + " }\n" + + " # reserved name, should trigger error\n" + + " field sddocname type string {\n" + + " indexing: index\n" + + " }\n" + + " }\n" + + "}"); fail("Should throw exception."); } catch (Exception expected) { // Success @@ -32,7 +43,16 @@ public class NameFieldCheckTestCase extends SearchDefinitionTestCase { @Test public void testDuplicateNamesInSearchDifferentType() { try { - SearchBuilder.buildFromFile("src/test/examples/duplicatenamesinsearchdifferenttype.sd"); + SearchBuilder.createFromString( + "search duplicatenamesinsearch {\n" + + " document {\n" + + " field grpphotoids64 type string { }\n" + + " }\n" + + " field grpphotoids64 type array<long> {\n" + + " indexing: input grpphotoids64 | split \" \" | for_each {\n" + + " base64decode } | attribute\n" + + " }\n" + + "}"); fail("Should throw exception."); } catch (Exception e) { assertEquals("For search 'duplicatenamesinsearch', field 'grpphotoids64': Incompatible types. Expected Array<long> for index field 'grpphotoids64', got string.", e.getMessage()); @@ -42,7 +62,17 @@ public class NameFieldCheckTestCase extends SearchDefinitionTestCase { @Test public void testDuplicateNamesInDoc() { try { - SearchBuilder.buildFromFile("src/test/examples/duplicatenamesindoc.sd"); + SearchBuilder.createFromString( + "search duplicatenamesindoc {\n" + + " document {\n" + + " field foo type int {\n" + + " indexing: attribute\n" + + " }\n" + + " field fOo type string {\n" + + " indexing: index\n" + + " }\n" + + " }\n" + + "}"); fail("Should throw exception."); } catch (Exception e) { assertTrue(e.getMessage().matches(".*Duplicate.*")); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingInputsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingInputsTestCase.java index 8fb40a73f0a..582f5b08517 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingInputsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingInputsTestCase.java @@ -18,27 +18,27 @@ public class IndexingInputsTestCase { assertBuildFails("src/test/examples/indexing_extra_field_input_extra_field.sd", "For search 'indexing_extra_field_input_extra_field', field 'bar': Indexing script refers " + "to field 'bar' which does not exist in document type " + - "'indexing_extra_field_input_extra_field'."); + "'indexing_extra_field_input_extra_field', and is not a mutable attribute."); } @Test public void requireThatExtraFieldInputImplicitThrows() throws IOException, ParseException { assertBuildFails("src/test/examples/indexing_extra_field_input_implicit.sd", "For search 'indexing_extra_field_input_implicit', field 'foo': Indexing script refers to " + - "field 'foo' which does not exist in document type 'indexing_extra_field_input_implicit'."); + "field 'foo' which does not exist in document type 'indexing_extra_field_input_implicit', and is not a mutable attribute."); } @Test public void requireThatExtraFieldInputNullThrows() throws IOException, ParseException { assertBuildFails("src/test/examples/indexing_extra_field_input_null.sd", "For search 'indexing_extra_field_input_null', field 'foo': Indexing script refers to field " + - "'foo' which does not exist in document type 'indexing_extra_field_input_null'."); + "'foo' which does not exist in document type 'indexing_extra_field_input_null', and is not a mutable attribute."); } @Test public void requireThatExtraFieldInputSelfThrows() throws IOException, ParseException { assertBuildFails("src/test/examples/indexing_extra_field_input_self.sd", "For search 'indexing_extra_field_input_self', field 'foo': Indexing script refers to field " + - "'foo' which does not exist in document type 'indexing_extra_field_input_self'."); + "'foo' which does not exist in document type 'indexing_extra_field_input_self', and is not a mutable attribute."); } } diff --git a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java index d729b092670..805fa52c105 100644 --- a/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java +++ b/container-accesslogging/src/main/java/com/yahoo/container/logging/LogFileHandler.java @@ -2,6 +2,7 @@ package com.yahoo.container.logging; import com.yahoo.container.core.AccessLogConfig; +import com.yahoo.io.NativeIO; import com.yahoo.log.LogFileDb; import java.io.File; @@ -263,15 +264,31 @@ public class LogFileHandler extends StreamHandler { numberOfRecords = 0; lastRotationTime = now; nextRotationTime = 0; //figure it out later (lazy evaluation) - if (compressOnRotation && (oldFileName != null)) { - triggerCompression(oldFileName); + if ((oldFileName != null)) { + File oldFile = new File(oldFileName); + if (oldFile.exists()) { + if (compressOnRotation) { + triggerCompression(oldFile); + } else { + NativeIO nativeIO = new NativeIO(); + nativeIO.dropFileFromCache(oldFile); + } + } } } - private void triggerCompression(String oldFileName) { + private void triggerCompression(File oldFile) { try { + String oldFileName = oldFile.getPath(); + String gzippedFileName = oldFileName + ".gz"; Runtime r = Runtime.getRuntime(); - Process p = r.exec(new String[] { "gzip", oldFileName }); + StringBuilder cmd = new StringBuilder("gzip"); + cmd.append(" < "). append(oldFileName).append(" > ").append(gzippedFileName); + Process p = r.exec(cmd.toString()); + NativeIO nativeIO = new NativeIO(); + nativeIO.dropFileFromCache(oldFile); // Drop from cache in case somebody else has a reference to it preventing from dying quickly. + oldFile.delete(); + nativeIO.dropFileFromCache(new File(gzippedFileName)); // Detonator pattern: Think of all the fun we can have if gzip isn't what we // think it is, if it doesn't return, etc, etc } catch (IOException e) { diff --git a/container-dev/pom.xml b/container-dev/pom.xml index c4c93563026..1b0f36e3adb 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -142,6 +142,10 @@ <groupId>commons-lang</groupId> <artifactId>commons-lang</artifactId> </exclusion> + <exclusion> + <groupId>net.java.dev.jna</groupId> + <artifactId>jna</artifactId> + </exclusion> </exclusions> </dependency> <dependency> diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java index a238facc983..713fd7e273c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java @@ -25,6 +25,7 @@ public enum JobType { productionEuWest1 ("production-eu-west-1" , ZoneId.from("prod" , "eu-west-1") , null ), productionAwsUsEast1a ("production-aws-us-east-1a" , ZoneId.from("prod" , "aws-us-east-1a") , null ), productionAwsUsWest1b ("production-aws-us-west-1b" , ZoneId.from("prod" , "aws-us-west-1b") , null ), + productionAwsUsEast1b ("production-aws-us-east-1b" , ZoneId.from("prod" , "aws-us-east-1b") , null ), productionCdAwsUsEast1a("production-cd-aws-us-east-1a", null , ZoneId.from("prod" , "cd-aws-us-east-1a")), productionCdUsCentral1 ("production-cd-us-central-1" , null , ZoneId.from("prod" , "cd-us-central-1") ), productionCdUsCentral2 ("production-cd-us-central-2" , null , ZoneId.from("prod" , "cd-us-central-2") ); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index a6c3f11470d..cd5178d80ac 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -199,6 +199,7 @@ public class Application { // Rotation status only contains VIP host names, one per zone in the system. The only way to map VIP hostname to // this deployment, and thereby determine rotation status, is to check if VIP hostname contains the // deployment's environment and region. + // TODO: change this map to be indexed by zones, then? return rotationStatus.entrySet().stream() .filter(kv -> kv.getKey().value().contains(deployment.zone().value())) .map(Map.Entry::getValue) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 2dd3b1dda23..b3158102182 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -44,6 +44,7 @@ import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.JobStatus.JobRun; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.concurrent.Once; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.rotation.Rotation; @@ -330,13 +331,9 @@ public class ApplicationController { validateRun(application.get(), zone, platformVersion, applicationVersion); } - validate(applicationPackage.deploymentSpec()); - // Update application with information from application package - if ( ! preferOldestVersion && ! application.get().deploymentJobs().builtInternally()) { - application = withUpdatedConfig(application, applicationPackage); - store(application); // store missing information even if we fail deployment below - } + if ( ! preferOldestVersion && ! application.get().deploymentJobs().builtInternally()) + application = storeWithUpdatedConfig(application, applicationPackage); // Assign global rotation application = withRotation(application, zone); @@ -359,8 +356,9 @@ public class ApplicationController { } /** Stores the deployment spec and validation overrides from the application package, and runs cleanup. */ - public LockedApplication withUpdatedConfig(LockedApplication application, ApplicationPackage applicationPackage) { - // Store information about application package + public LockedApplication storeWithUpdatedConfig(LockedApplication application, ApplicationPackage applicationPackage) { + validate(applicationPackage.deploymentSpec()); + application = application.with(applicationPackage.deploymentSpec()); application = application.with(applicationPackage.validationOverrides()); @@ -649,6 +647,7 @@ public class ApplicationController { /** Verify that each of the production zones listed in the deployment spec exist in this system. */ private void validate(DeploymentSpec deploymentSpec) { + new DeploymentSteps(deploymentSpec, controller::system).jobs(); deploymentSpec.zones().stream() .filter(zone -> zone.environment() == Environment.prod) .forEach(zone -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 2bb878366c5..71b93731a44 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -101,7 +101,7 @@ public class InternalStepRunner implements StepRunner { case endTests: return endTests(id, logger); case deactivateReal: return deactivateReal(id, logger); case deactivateTester: return deactivateTester(id, logger); - case report: return report(id); + case report: return report(id, logger); default: throw new AssertionError("Unknown step '" + step + "'!"); } } @@ -398,8 +398,13 @@ public class InternalStepRunner implements StepRunner { return Optional.of(running); } - private Optional<RunStatus> report(RunId id) { - controller.jobController().active(id).ifPresent(run -> controller.applications().deploymentTrigger().notifyOfCompletion(report(run))); + private Optional<RunStatus> report(RunId id, DualLogger logger) { + try { + controller.jobController().active(id).ifPresent(run -> controller.applications().deploymentTrigger().notifyOfCompletion(report(run))); + } + catch (IllegalStateException e) { + logger.log(INFO, "Job '" + id.type() + "'no longer supposed to run?:", e); + } return Optional.of(running); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index e9b0267a443..3cff6ac2430 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -203,32 +203,31 @@ public class JobController { * Accepts and stores a new application package and test jar pair under a generated application version key. */ public ApplicationVersion submit(ApplicationId id, SourceRevision revision, - byte[] applicationPackage, byte[] applicationTestPackage) { + byte[] packageBytes, byte[] testPackageBytes) { AtomicReference<ApplicationVersion> version = new AtomicReference<>(); controller.applications().lockOrThrow(id, application -> { + if ( ! application.get().deploymentJobs().builtInternally()) { + // Copy all current packages to the new application store + application.get().deployments().values().stream() + .map(Deployment::applicationVersion) + .distinct() + .forEach(appVersion -> { + byte[] content = controller.applications().artifacts().getApplicationPackage(application.get().id(), appVersion.id()); + controller.applications().applicationStore().putApplicationPackage(application.get().id(), appVersion.id(), content); + }); + } + long run = nextBuild(id); version.set(ApplicationVersion.from(revision, run)); controller.applications().applicationStore().putApplicationPackage(id, - version.get().id(), - applicationPackage); + version.get().id(), + packageBytes); controller.applications().applicationStore().putTesterPackage(testerOf(id), version.get().id(), - applicationTestPackage); - - if (!application.get().deploymentJobs().builtInternally()) { - // Copy all current packages to the new application store - application.get().deployments().values().stream() - .map(Deployment::applicationVersion) - .distinct() - .forEach(appVersion -> { - byte[] content = controller.applications().artifacts().getApplicationPackage(application.get().id(), appVersion.id()); - controller.applications().applicationStore().putApplicationPackage(application.get().id(), appVersion.id(), content); - }); - } + testPackageBytes); - controller.applications().store(controller.applications().withUpdatedConfig(application.withBuiltInternally(true), - new ApplicationPackage(applicationPackage))); + controller.applications().storeWithUpdatedConfig(application.withBuiltInternally(true), new ApplicationPackage(packageBytes)); notifyOfNewSubmission(id, revision, run); }); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 635e0c1fb26..a9353af1b20 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -147,6 +147,19 @@ public class ControllerTest { assertEquals(5, applications.get(app1.id()).get().deploymentJobs().jobStatus().size()); + // Production zone for which there is no JobType is not allowed. + applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("deep-space-9") + .build(); + try { + tester.controller().jobController().submit(app1.id(), BuildJob.defaultSourceRevision, applicationPackage.zippedContent(), new byte[0]); + fail("Expected exception due to illegal deployment spec."); + } + catch (IllegalArgumentException e) { + assertEquals("No job is known for zone prod.deep-space-9 in default.", e.getMessage()); + } + // prod zone removal is not allowed applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java index 10e3ce561d7..a2321e912e1 100644 --- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java +++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/DocumentScript.java @@ -9,7 +9,6 @@ import com.yahoo.document.datatypes.Array; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.MapFieldValue; import com.yahoo.document.datatypes.StringFieldValue; -import com.yahoo.document.datatypes.Struct; import com.yahoo.document.datatypes.StructuredFieldValue; import com.yahoo.document.datatypes.WeightedSet; import com.yahoo.document.fieldpathupdate.AssignFieldPathUpdate; diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java index e486e2647cf..8e934001381 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java @@ -116,11 +116,10 @@ public class JsonSerializationHelper { } public static void serializeStringField(JsonGenerator generator, FieldBase field, StringFieldValue value) { - // Hide empty strings - if (value.getString().length() == 0) { + // Hide fields which only contains an empty string + if (value.getString().length() == 0 && field != null) { return; } - serializeString(generator, field, value.getString()); } @@ -237,10 +236,6 @@ public class JsonSerializationHelper { } public static void serializeString(JsonGenerator generator, FieldBase field, String value) { - if (value.length() == 0) { - return; - } - fieldNameIfNotNull(generator, field); wrapIOException(() -> generator.writeString(value)); } diff --git a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java index 50d02d70c9b..0ab00b4e2bc 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java @@ -213,7 +213,7 @@ public class JsonWriterTestCase { @Test public void arrayTest() throws IOException { roundTripEquality("id:unittest:testarray::whee", "{ \"actualarray\": [" - + " \"nalle\"," + " \"tralle\"]}"); + + " \"nalle\"," + " \"\"," + " \"tralle\"]}"); } @Test @@ -342,6 +342,10 @@ public class JsonWriterTestCase { ObjectMapper m = new ObjectMapper(); Map<?, ?> exp = m.readValue(expected, Map.class); Map<?, ?> gen = m.readValue(generated, Map.class); + if (! exp.equals(gen)) { + System.err.println("expected: "+Utf8.toString(expected)); + System.err.println("generated: "+Utf8.toString(generated)); + } assertEquals(exp, gen); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java index 6983eb4fbd9..10a316f7751 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java @@ -301,63 +301,50 @@ public class DocumentProtocol implements Protocol { putRoutingPolicyFactory("SubsetService", new RoutingPolicyFactories.SubsetServicePolicyFactory()); // Prepare version specifications to use when adding routable factories. - VersionSpecification version50 = new VersionSpecification(5, 0); - VersionSpecification version51 = new VersionSpecification(5, 1); VersionSpecification version52 = new VersionSpecification(5, 115); VersionSpecification version6 = new VersionSpecification(6, 221); - List<VersionSpecification> from50 = Arrays.asList(version50, version51, version52, version6); - List<VersionSpecification> from51 = Arrays.asList(version51, version52, version6); List<VersionSpecification> from52 = Arrays.asList(version52, version6); List<VersionSpecification> from6 = Collections.singletonList(version6); // TODO decide minor version... - // 5.0 serialization (keep alphabetized please) - putRoutableFactory(MESSAGE_CREATEVISITOR, new RoutableFactories50.CreateVisitorMessageFactory(), from50); - putRoutableFactory(MESSAGE_DESTROYVISITOR, new RoutableFactories50.DestroyVisitorMessageFactory(), from50); - putRoutableFactory(MESSAGE_DOCUMENTLIST, new RoutableFactories50.DocumentListMessageFactory(), from50); - putRoutableFactory(MESSAGE_DOCUMENTSUMMARY, new RoutableFactories50.DocumentSummaryMessageFactory(), from50); - putRoutableFactory(MESSAGE_EMPTYBUCKETS, new RoutableFactories50.EmptyBucketsMessageFactory(), from50); - putRoutableFactory(MESSAGE_GETBUCKETLIST, new RoutableFactories50.GetBucketListMessageFactory(), from50); - putRoutableFactory(MESSAGE_GETBUCKETSTATE, new RoutableFactories50.GetBucketStateMessageFactory(), from50); - putRoutableFactory(MESSAGE_GETDOCUMENT, new RoutableFactories50.GetDocumentMessageFactory(), from50); - putRoutableFactory(MESSAGE_MAPVISITOR, new RoutableFactories50.MapVisitorMessageFactory(), from50); - putRoutableFactory(MESSAGE_PUTDOCUMENT, new RoutableFactories50.PutDocumentMessageFactory(), from50); - putRoutableFactory(MESSAGE_QUERYRESULT, new RoutableFactories50.QueryResultMessageFactory(), from50); - putRoutableFactory(MESSAGE_REMOVEDOCUMENT, new RoutableFactories50.RemoveDocumentMessageFactory(), from50); - putRoutableFactory(MESSAGE_REMOVELOCATION, new RoutableFactories50.RemoveLocationMessageFactory(), from50); - putRoutableFactory(MESSAGE_SEARCHRESULT, new RoutableFactories50.SearchResultMessageFactory(), from50); - putRoutableFactory(MESSAGE_STATBUCKET, new RoutableFactories50.StatBucketMessageFactory(), from50); - putRoutableFactory(MESSAGE_UPDATEDOCUMENT, new RoutableFactories50.UpdateDocumentMessageFactory(), from50); - putRoutableFactory(MESSAGE_VISITORINFO, new RoutableFactories50.VisitorInfoMessageFactory(), from50); - putRoutableFactory(REPLY_CREATEVISITOR, new RoutableFactories50.CreateVisitorReplyFactory(), from50); - putRoutableFactory(REPLY_DESTROYVISITOR, new RoutableFactories50.DestroyVisitorReplyFactory(), from50); - putRoutableFactory(REPLY_DOCUMENTLIST, new RoutableFactories50.DocumentListReplyFactory(), from50); - putRoutableFactory(REPLY_DOCUMENTSUMMARY, new RoutableFactories50.DocumentSummaryReplyFactory(), from50); - putRoutableFactory(REPLY_EMPTYBUCKETS, new RoutableFactories50.EmptyBucketsReplyFactory(), from50); - putRoutableFactory(REPLY_GETBUCKETLIST, new RoutableFactories50.GetBucketListReplyFactory(), from50); - putRoutableFactory(REPLY_GETBUCKETSTATE, new RoutableFactories50.GetBucketStateReplyFactory(), from50); - putRoutableFactory(REPLY_GETDOCUMENT, new RoutableFactories50.GetDocumentReplyFactory(), from50); - putRoutableFactory(REPLY_MAPVISITOR, new RoutableFactories50.MapVisitorReplyFactory(), from50); - putRoutableFactory(REPLY_PUTDOCUMENT, new RoutableFactories50.PutDocumentReplyFactory(), from50); - putRoutableFactory(REPLY_QUERYRESULT, new RoutableFactories50.QueryResultReplyFactory(), from50); - putRoutableFactory(REPLY_REMOVEDOCUMENT, new RoutableFactories50.RemoveDocumentReplyFactory(), from50); - putRoutableFactory(REPLY_REMOVELOCATION, new RoutableFactories50.RemoveLocationReplyFactory(), from50); - putRoutableFactory(REPLY_SEARCHRESULT, new RoutableFactories50.SearchResultReplyFactory(), from50); - putRoutableFactory(REPLY_STATBUCKET, new RoutableFactories50.StatBucketReplyFactory(), from50); - putRoutableFactory(REPLY_UPDATEDOCUMENT, new RoutableFactories50.UpdateDocumentReplyFactory(), from50); - putRoutableFactory(REPLY_UPDATEDOCUMENT, new RoutableFactories50.UpdateDocumentReplyFactory(), from50); - putRoutableFactory(REPLY_VISITORINFO, new RoutableFactories50.VisitorInfoReplyFactory(), from50); - putRoutableFactory(REPLY_WRONGDISTRIBUTION, new RoutableFactories50.WrongDistributionReplyFactory(), from50); - - // 5.1 serialization - putRoutableFactory(MESSAGE_CREATEVISITOR, new RoutableFactories51.CreateVisitorMessageFactory(), from51); - putRoutableFactory(MESSAGE_GETDOCUMENT, new RoutableFactories51.GetDocumentMessageFactory(), from51); - putRoutableFactory(REPLY_DOCUMENTIGNORED, new RoutableFactories51.DocumentIgnoredReplyFactory(), from51); - - // 5.2 serialization + // 5.2 serialization (keep alphabetized please) + putRoutableFactory(MESSAGE_CREATEVISITOR, new RoutableFactories52.CreateVisitorMessageFactory(), from52); + putRoutableFactory(MESSAGE_DESTROYVISITOR, new RoutableFactories52.DestroyVisitorMessageFactory(), from52); + putRoutableFactory(MESSAGE_DOCUMENTLIST, new RoutableFactories52.DocumentListMessageFactory(), from52); + putRoutableFactory(MESSAGE_DOCUMENTSUMMARY, new RoutableFactories52.DocumentSummaryMessageFactory(), from52); + putRoutableFactory(MESSAGE_EMPTYBUCKETS, new RoutableFactories52.EmptyBucketsMessageFactory(), from52); + putRoutableFactory(MESSAGE_GETBUCKETLIST, new RoutableFactories52.GetBucketListMessageFactory(), from52); + putRoutableFactory(MESSAGE_GETBUCKETSTATE, new RoutableFactories52.GetBucketStateMessageFactory(), from52); + putRoutableFactory(MESSAGE_GETDOCUMENT, new RoutableFactories52.GetDocumentMessageFactory(), from52); + putRoutableFactory(MESSAGE_MAPVISITOR, new RoutableFactories52.MapVisitorMessageFactory(), from52); putRoutableFactory(MESSAGE_PUTDOCUMENT, new RoutableFactories52.PutDocumentMessageFactory(), from52); - putRoutableFactory(MESSAGE_UPDATEDOCUMENT, new RoutableFactories52.UpdateDocumentMessageFactory(), from52); + putRoutableFactory(MESSAGE_QUERYRESULT, new RoutableFactories52.QueryResultMessageFactory(), from52); putRoutableFactory(MESSAGE_REMOVEDOCUMENT, new RoutableFactories52.RemoveDocumentMessageFactory(), from52); + putRoutableFactory(MESSAGE_REMOVELOCATION, new RoutableFactories52.RemoveLocationMessageFactory(), from52); + putRoutableFactory(MESSAGE_SEARCHRESULT, new RoutableFactories52.SearchResultMessageFactory(), from52); + putRoutableFactory(MESSAGE_STATBUCKET, new RoutableFactories52.StatBucketMessageFactory(), from52); + putRoutableFactory(MESSAGE_UPDATEDOCUMENT, new RoutableFactories52.UpdateDocumentMessageFactory(), from52); + putRoutableFactory(MESSAGE_VISITORINFO, new RoutableFactories52.VisitorInfoMessageFactory(), from52); + putRoutableFactory(REPLY_CREATEVISITOR, new RoutableFactories52.CreateVisitorReplyFactory(), from52); + putRoutableFactory(REPLY_DESTROYVISITOR, new RoutableFactories52.DestroyVisitorReplyFactory(), from52); + putRoutableFactory(REPLY_DOCUMENTIGNORED, new RoutableFactories52.DocumentIgnoredReplyFactory(), from52); + putRoutableFactory(REPLY_DOCUMENTLIST, new RoutableFactories52.DocumentListReplyFactory(), from52); + putRoutableFactory(REPLY_DOCUMENTSUMMARY, new RoutableFactories52.DocumentSummaryReplyFactory(), from52); + putRoutableFactory(REPLY_EMPTYBUCKETS, new RoutableFactories52.EmptyBucketsReplyFactory(), from52); + putRoutableFactory(REPLY_GETBUCKETLIST, new RoutableFactories52.GetBucketListReplyFactory(), from52); + putRoutableFactory(REPLY_GETBUCKETSTATE, new RoutableFactories52.GetBucketStateReplyFactory(), from52); + putRoutableFactory(REPLY_GETDOCUMENT, new RoutableFactories52.GetDocumentReplyFactory(), from52); + putRoutableFactory(REPLY_MAPVISITOR, new RoutableFactories52.MapVisitorReplyFactory(), from52); + putRoutableFactory(REPLY_PUTDOCUMENT, new RoutableFactories52.PutDocumentReplyFactory(), from52); + putRoutableFactory(REPLY_QUERYRESULT, new RoutableFactories52.QueryResultReplyFactory(), from52); + putRoutableFactory(REPLY_REMOVEDOCUMENT, new RoutableFactories52.RemoveDocumentReplyFactory(), from52); + putRoutableFactory(REPLY_REMOVELOCATION, new RoutableFactories52.RemoveLocationReplyFactory(), from52); + putRoutableFactory(REPLY_SEARCHRESULT, new RoutableFactories52.SearchResultReplyFactory(), from52); + putRoutableFactory(REPLY_STATBUCKET, new RoutableFactories52.StatBucketReplyFactory(), from52); + putRoutableFactory(REPLY_UPDATEDOCUMENT, new RoutableFactories52.UpdateDocumentReplyFactory(), from52); + putRoutableFactory(REPLY_UPDATEDOCUMENT, new RoutableFactories52.UpdateDocumentReplyFactory(), from52); + putRoutableFactory(REPLY_VISITORINFO, new RoutableFactories52.VisitorInfoReplyFactory(), from52); + putRoutableFactory(REPLY_WRONGDISTRIBUTION, new RoutableFactories52.WrongDistributionReplyFactory(), from52); // 6.x serialization putRoutableFactory(MESSAGE_CREATEVISITOR, new RoutableFactories60.CreateVisitorMessageFactory(), from6); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java index 5a68e5bda2a..051ec44dd07 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java @@ -13,6 +13,7 @@ import com.yahoo.messagebus.routing.RoutingContext; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; /** * This policy implements the necessary logic to communicate with an external Vespa application and resolve its list of @@ -30,6 +31,7 @@ public class ExternPolicy implements DocumentProtocolRoutingPolicy { private int offset = 0; private int generation = 0; private final List<Hop> recipients = new ArrayList<>(); + private final AtomicBoolean destroyed = new AtomicBoolean(false); /** * Constructs a new instance of this policy. The argument given is the connection spec to the slobrok to use for @@ -110,13 +112,6 @@ public class ExternPolicy implements DocumentProtocolRoutingPolicy { } @Override - @SuppressWarnings("deprecation") // finalize() is deprecated from Java 9 - public void finalize() throws Throwable { - super.finalize(); - mirror.shutdown(); - orb.transport().shutdown().join(); - } - public void select(RoutingContext ctx) { if (error != null) { ctx.setError(DocumentProtocol.ERROR_POLICY_FAILURE, error); @@ -135,13 +130,19 @@ public class ExternPolicy implements DocumentProtocolRoutingPolicy { } } + @Override public void merge(RoutingContext ctx) { DocumentProtocol.merge(ctx); } + @Override public void destroy() { + if (destroyed.getAndSet(true)) throw new RuntimeException("Already destroyed"); + mirror.shutdown(); + orb.transport().shutdown().join(); } + @Override public MetricSet getMetrics() { return null; } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories50.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories50.java deleted file mode 100755 index c7c1121ae68..00000000000 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories50.java +++ /dev/null @@ -1,917 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol; - -import com.yahoo.document.*; -import com.yahoo.document.serialization.DocumentDeserializer; -import com.yahoo.document.serialization.DocumentSerializer; -import com.yahoo.document.serialization.DocumentSerializerFactory; -import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; -import com.yahoo.log.LogLevel; -import com.yahoo.messagebus.Routable; -import com.yahoo.vdslib.DocumentList; -import com.yahoo.vdslib.DocumentSummary; -import com.yahoo.vdslib.SearchResult; -import com.yahoo.vdslib.VisitorStatistics; -import com.yahoo.vespa.objects.Deserializer; -import com.yahoo.vespa.objects.Serializer; - -import java.util.Map; -import java.util.logging.Logger; - - -/** - * This class encapsulates all the {@link RoutableFactory} classes needed to implement serialization for the document - * protocol. When adding new factories to this class, please KEEP THE THEM ORDERED alphabetically like they are now. - */ -public abstract class RoutableFactories50 { - - /** - * Implements the shared factory logic required for {@link DocumentMessage} objects, and it offers a more convenient - * interface for implementing {@link RoutableFactory}. - * - * @author Simon Thoresen Hult - */ - public static abstract class DocumentMessageFactory extends AbstractRoutableFactory { - - /** - * This method encodes the given message using the given serializer. You are guaranteed to only receive messages - * of the type that this factory was registered for. - * <p> - * This method is NOT exception safe. Return false to - * signal failure. - * - * @param msg The message to encode. - * @param serializer The serializer to use for encoding. - * @return True if the message was encoded. - */ - protected abstract boolean doEncode(DocumentMessage msg, DocumentSerializer serializer); - - /** - * This method decodes a message from the given deserializer. You are guaranteed to only receive byte buffers - * generated by a previous call to {@link #doEncode(DocumentMessage, DocumentSerializer)}. - * <p> - * This method is NOT exception safe. Return null to signal failure. - * - * @param deserializer The deserializer to use for decoding. - * @return The decoded message. - */ - protected abstract DocumentMessage doDecode(DocumentDeserializer deserializer); - - public boolean encode(Routable obj, DocumentSerializer out) { - if (!(obj instanceof DocumentMessage)) { - throw new AssertionError( - "Document message factory (" + getClass().getName() + ") registered for incompatible " + - "routable type " + obj.getType() + "(" + obj.getClass().getName() + ")."); - } - DocumentMessage msg = (DocumentMessage)obj; - out.putByte(null, (byte)(msg.getPriority().getValue())); - out.putInt(null, msg.getLoadType().getId()); - return doEncode(msg, out); - } - - public Routable decode(DocumentDeserializer in, LoadTypeSet loadTypes) { - byte pri = in.getByte(null); - int loadType = in.getInt(null); - DocumentMessage msg = doDecode(in); - if (msg != null) { - msg.setPriority(DocumentProtocol.getPriority(pri)); - msg.setLoadType(loadTypes.getIdMap().get(loadType)); - } - return msg; - } - } - - /** - * Implements the shared factory logic required for {@link DocumentReply} objects, and it offers a more convenient - * interface for implementing {@link RoutableFactory}. - * - * @author Simon Thoresen Hult - */ - public static abstract class DocumentReplyFactory extends AbstractRoutableFactory { - - /** - * This method encodes the given reply into the given byte buffer. You are guaranteed to only receive replies of - * the type that this factory was registered for. - * <p> - * This method is NOT exception safe. Return false to signal - * failure. - * - * @param reply The reply to encode. - * @param buf The byte buffer to write to. - * @return True if the message was encoded. - */ - protected abstract boolean doEncode(DocumentReply reply, DocumentSerializer buf); - - /** - * This method decodes a reply from the given byte buffer. You are guaranteed to only receive byte buffers - * generated by a previous call to {@link #doEncode(DocumentReply, com.yahoo.document.serialization.DocumentSerializer)}. - * - * <p> - * This method is NOT exception safe. Return null to signal failure. - * - * @param buf The byte buffer to read from. - * @return The decoded reply. - */ - protected abstract DocumentReply doDecode(DocumentDeserializer buf); - - public boolean encode(Routable obj, DocumentSerializer out) { - if (!(obj instanceof DocumentReply)) { - throw new AssertionError( - "Document reply factory (" + getClass().getName() + ") registered for incompatible " + - "routable type " + obj.getType() + "(" + obj.getClass().getName() + ")."); - } - DocumentReply reply = (DocumentReply)obj; - out.putByte(null, (byte)(reply.getPriority().getValue())); - return doEncode(reply, out); - } - - public Routable decode(DocumentDeserializer in, LoadTypeSet loadTypes) { - byte pri = in.getByte(null); - DocumentReply reply = doDecode(in); - if (reply != null) { - reply.setPriority(DocumentProtocol.getPriority(pri)); - } - return reply; - } - } - - public static class CreateVisitorMessageFactory extends DocumentMessageFactory { - - @Override - @SuppressWarnings("deprecation") - protected DocumentMessage doDecode(DocumentDeserializer buf) { - CreateVisitorMessage msg = new CreateVisitorMessage(); - msg.setLibraryName(decodeString(buf)); - msg.setInstanceId(decodeString(buf)); - msg.setControlDestination(decodeString(buf)); - msg.setDataDestination(decodeString(buf)); - msg.setDocumentSelection(decodeString(buf)); - msg.setMaxPendingReplyCount(buf.getInt(null)); - - int size = buf.getInt(null); - for (int i = 0; i < size; i++) { - long reversed = buf.getLong(null); - long rawid = ((reversed >>> 56) & 0x00000000000000FFl) | ((reversed >>> 40) & 0x000000000000FF00l) | - ((reversed >>> 24) & 0x0000000000FF0000l) | ((reversed >>> 8) & 0x00000000FF000000l) | - ((reversed << 8) & 0x000000FF00000000l) | ((reversed << 24) & 0x0000FF0000000000l) | - ((reversed << 40) & 0x00FF000000000000l) | ((reversed << 56) & 0xFF00000000000000l); - msg.getBuckets().add(new BucketId(rawid)); - } - - msg.setFromTimestamp(buf.getLong(null)); - msg.setToTimestamp(buf.getLong(null)); - msg.setVisitRemoves(buf.getByte(null) == (byte)1); - buf.getByte(null); // removed feature "visitHeadersOnly" - msg.setVisitInconsistentBuckets(buf.getByte(null) == (byte)1); - - size = buf.getInt(null); - for (int i = 0; i < size; i++) { - String key = decodeString(buf); - int sz = buf.getInt(null); - msg.getParameters().put(key, buf.getBytes(null, sz)); - } - - msg.setVisitorOrdering(buf.getInt(null)); - msg.setMaxBucketsPerVisitor(buf.getInt(null)); - msg.setVisitorDispatcherVersion(50); - return msg; - } - - @Override - @SuppressWarnings("deprecation") - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - CreateVisitorMessage msg = (CreateVisitorMessage)obj; - encodeString(msg.getLibraryName(), buf); - encodeString(msg.getInstanceId(), buf); - encodeString(msg.getControlDestination(), buf); - encodeString(msg.getDataDestination(), buf); - encodeString(msg.getDocumentSelection(), buf); - buf.putInt(null, msg.getMaxPendingReplyCount()); - - buf.putInt(null, msg.getBuckets().size()); - for (BucketId id : msg.getBuckets()) { - long rawid = id.getRawId(); - long reversed = ((rawid >>> 56) & 0x00000000000000FFl) | ((rawid >>> 40) & 0x000000000000FF00l) | - ((rawid >>> 24) & 0x0000000000FF0000l) | ((rawid >>> 8) & 0x00000000FF000000l) | - ((rawid << 8) & 0x000000FF00000000l) | ((rawid << 24) & 0x0000FF0000000000l) | - ((rawid << 40) & 0x00FF000000000000l) | ((rawid << 56) & 0xFF00000000000000l); - buf.putLong(null, reversed); - } - - buf.putLong(null, msg.getFromTimestamp()); - buf.putLong(null, msg.getToTimestamp()); - buf.putByte(null, msg.getVisitRemoves() ? (byte)1 : (byte)0); - buf.putByte(null, (byte)0); // removed feature "visitHeadersOnly" - buf.putByte(null, msg.getVisitInconsistentBuckets() ? (byte)1 : (byte)0); - - buf.putInt(null, msg.getParameters().size()); - for (Map.Entry<String, byte[]> pairs : msg.getParameters().entrySet()) { - encodeString(pairs.getKey(), buf); - byte[] b = pairs.getValue(); - buf.putInt(null, b.length); - buf.put(null, b); - } - - buf.putInt(null, msg.getVisitorOrdering()); - buf.putInt(null, msg.getMaxBucketsPerVisitor()); - return true; - } - } - - public static class CreateVisitorReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - CreateVisitorReply reply = new CreateVisitorReply(DocumentProtocol.REPLY_CREATEVISITOR); - reply.setLastBucket(new BucketId(buf.getLong(null))); - - VisitorStatistics vs = new VisitorStatistics(); - vs.setBucketsVisited(buf.getInt(null)); - vs.setDocumentsVisited(buf.getLong(null)); - vs.setBytesVisited(buf.getLong(null)); - vs.setDocumentsReturned(buf.getLong(null)); - vs.setBytesReturned(buf.getLong(null)); - vs.setSecondPassDocumentsReturned(buf.getLong(null)); - vs.setSecondPassBytesReturned(buf.getLong(null)); - reply.setVisitorStatistics(vs); - return reply; - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - CreateVisitorReply reply = (CreateVisitorReply)obj; - buf.putLong(null, reply.getLastBucket().getRawId()); - buf.putInt(null, reply.getVisitorStatistics().getBucketsVisited()); - buf.putLong(null, reply.getVisitorStatistics().getDocumentsVisited()); - buf.putLong(null, reply.getVisitorStatistics().getBytesVisited()); - buf.putLong(null, reply.getVisitorStatistics().getDocumentsReturned()); - buf.putLong(null, reply.getVisitorStatistics().getBytesReturned()); - buf.putLong(null, reply.getVisitorStatistics().getSecondPassDocumentsReturned()); - buf.putLong(null, reply.getVisitorStatistics().getSecondPassBytesReturned()); - return true; - } - } - - public static class DestroyVisitorMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - DestroyVisitorMessage msg = new DestroyVisitorMessage(); - msg.setInstanceId(decodeString(buf)); - return msg; - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - DestroyVisitorMessage msg = (DestroyVisitorMessage)obj; - encodeString(msg.getInstanceId(), buf); - return true; - } - } - - public static class DestroyVisitorReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - return new VisitorReply(DocumentProtocol.REPLY_DESTROYVISITOR); - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - return true; - } - } - - public static class DocumentListMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - DocumentListMessage msg = new DocumentListMessage(); - msg.setBucketId(new BucketId(buf.getLong(null))); - int len = buf.getInt(null); - for (int i = 0; i < len; i++) { - msg.getDocuments().add(new DocumentListEntry(buf)); - } - return msg; - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - DocumentListMessage msg = (DocumentListMessage)obj; - buf.putLong(null, msg.getBucketId().getRawId()); - buf.putInt(null, msg.getDocuments().size()); - - for (int i = 0; i < msg.getDocuments().size(); i++) { - msg.getDocuments().get(i).serialize(buf); - } - return true; - } - } - - public static class DocumentListReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - return new VisitorReply(DocumentProtocol.REPLY_DOCUMENTLIST); - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - return true; - } - } - - public static class DocumentSummaryMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - DocumentSummaryMessage msg = new DocumentSummaryMessage(); - msg.setDocumentSummary(new DocumentSummary(buf)); - return msg; - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - return false; // not supported - } - } - - public static class DocumentSummaryReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - return new VisitorReply(DocumentProtocol.REPLY_DOCUMENTSUMMARY); - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - return true; - } - } - - public static class EmptyBucketsMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - EmptyBucketsMessage msg = new EmptyBucketsMessage(); - int size = buf.getInt(null); - for (int i = 0; i < size; ++i) { - msg.getBucketIds().add(new BucketId(buf.getLong(null))); - } - return msg; - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - EmptyBucketsMessage msg = (EmptyBucketsMessage)obj; - buf.putInt(null, msg.getBucketIds().size()); - for (BucketId bid : msg.getBucketIds()) { - buf.putLong(null, bid.getRawId()); - } - return true; - } - } - - public static class EmptyBucketsReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - return new VisitorReply(DocumentProtocol.REPLY_EMPTYBUCKETS); - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - return true; - } - } - - public static class GetBucketListMessageFactory extends DocumentMessageFactory { - - protected String decodeBucketSpace(Deserializer deserializer) { - return FixedBucketSpaces.defaultSpace(); - } - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - GetBucketListMessage msg = new GetBucketListMessage(); - msg.setBucketId(new BucketId(buf.getLong(null))); - msg.setBucketSpace(decodeBucketSpace(buf)); - return msg; - } - - protected boolean encodeBucketSpace(String bucketSpace, DocumentSerializer buf) { - return FixedBucketSpaces.defaultSpace().equals(bucketSpace); - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - GetBucketListMessage msg = (GetBucketListMessage)obj; - buf.putLong(null, msg.getBucketId().getRawId()); - return encodeBucketSpace(msg.getBucketSpace(), buf); - } - } - - public static class GetBucketListReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - GetBucketListReply reply = new GetBucketListReply(); - int len = buf.getInt(null); - for (int i = 0; i < len; i++) { - GetBucketListReply.BucketInfo info = new GetBucketListReply.BucketInfo(); - info.bucket = new BucketId(buf.getLong(null)); - info.bucketInformation = decodeString(buf); - reply.getBuckets().add(info); - } - return reply; - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - GetBucketListReply reply = (GetBucketListReply)obj; - buf.putInt(null, reply.getBuckets().size()); - for (GetBucketListReply.BucketInfo info : reply.getBuckets()) { - buf.putLong(null, info.bucket.getRawId()); - encodeString(info.bucketInformation, buf); - } - return true; - } - } - - public static class GetBucketStateMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - GetBucketStateMessage msg = new GetBucketStateMessage(); - msg.setBucketId(new BucketId(buf.getLong(null))); - return msg; - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - GetBucketStateMessage msg = (GetBucketStateMessage)obj; - buf.putLong(null, msg.getBucketId().getRawId()); - return true; - } - } - - public static class GetBucketStateReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - GetBucketStateReply reply = new GetBucketStateReply(); - int size = buf.getInt(null); - for (int i = 0; i < size; i++) { - reply.getBucketState().add(new DocumentState(buf)); - } - return reply; - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - GetBucketStateReply reply = (GetBucketStateReply)obj; - buf.putInt(null, reply.getBucketState().size()); - for (DocumentState stat : reply.getBucketState()) { - stat.serialize(buf); - } - return true; - } - } - - public static class GetDocumentMessageFactory extends DocumentMessageFactory { - - @Override - @SuppressWarnings("deprecation") - protected DocumentMessage doDecode(DocumentDeserializer buf) { - GetDocumentMessage msg = new GetDocumentMessage(); - msg.setDocumentId(new DocumentId(buf)); - buf.getInt(null); // removed feature "flags"; ignore - return msg; - } - - @Override - @SuppressWarnings("deprecation") - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - GetDocumentMessage msg = (GetDocumentMessage)obj; - msg.getDocumentId().serialize(buf); - buf.putInt(null, 0); // removed feature "flags" - return true; - } - } - - public static class GetDocumentReplyFactory extends DocumentReplyFactory { - - private final LazyDecoder decoder = new LazyDecoder() { - - public void decode(Routable obj, DocumentDeserializer buf) { - GetDocumentReply reply = (GetDocumentReply)obj; - - Document doc = null; - byte flag = buf.getByte(null); - if (flag != 0) { - doc = Document.createDocument(buf); - reply.setDocument(doc); - } - long lastModified = buf.getLong(null); - reply.setLastModified(lastModified); - if (doc != null) { - doc.setLastModified(lastModified); - } - } - }; - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - GetDocumentReply reply = new GetDocumentReply(decoder, buf); - - return reply; - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - GetDocumentReply reply = (GetDocumentReply)obj; - if (reply.getSerializedBuffer() != null) { - buf.put(null, reply.getSerializedBuffer()); - } else { - Document document = reply.getDocument(); - buf.putByte(null, (byte)(document == null ? 0 : 1)); - if (document != null) { - document.serialize(buf); - } - buf.putLong(null, reply.getLastModified()); - } - return true; - } - } - - public static class MapVisitorMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - MapVisitorMessage msg = new MapVisitorMessage(); - int size = buf.getInt(null); - for (int i = 0; i < size; i++) { - String key = decodeString(buf); - String value = decodeString(buf); - msg.getData().put(key, value); - } - return msg; - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - MapVisitorMessage msg = (MapVisitorMessage)obj; - buf.putInt(null, msg.getData().size()); - for (Map.Entry<String, String> pairs : msg.getData().entrySet()) { - encodeString(pairs.getKey(), buf); - encodeString(pairs.getValue(), buf); - } - return true; - } - } - - public static class MapVisitorReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - return new VisitorReply(DocumentProtocol.REPLY_MAPVISITOR); - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - return true; - } - } - - public static class PutDocumentMessageFactory extends DocumentMessageFactory { - protected void decodeInto(PutDocumentMessage msg, DocumentDeserializer buf) { - msg.setDocumentPut(new DocumentPut(Document.createDocument(buf))); - msg.setTimestamp(buf.getLong(null)); - } - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buffer) { - final LazyDecoder decoder = (obj, buf) -> { - decodeInto((PutDocumentMessage) obj, buf); - }; - - return new PutDocumentMessage(decoder, buffer); - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - PutDocumentMessage msg = (PutDocumentMessage)obj; - if (msg.getSerializedBuffer() != null) { - buf.put(null, msg.getSerializedBuffer()); - } else { - msg.getDocumentPut().getDocument().serialize(buf); - buf.putLong(null, msg.getTimestamp()); - } - return true; - } - } - - public static class PutDocumentReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - WriteDocumentReply rep = new WriteDocumentReply(DocumentProtocol.REPLY_PUTDOCUMENT); - rep.setHighestModificationTimestamp(buf.getLong(null)); - return rep; - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - WriteDocumentReply rep = (WriteDocumentReply)obj; - buf.putLong(null, rep.getHighestModificationTimestamp()); - return true; - } - } - - public static class RemoveDocumentMessageFactory extends DocumentMessageFactory { - protected void decodeInto(RemoveDocumentMessage msg, DocumentDeserializer buf) { - msg.setDocumentId(new DocumentId(buf)); - } - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - RemoveDocumentMessage msg = new RemoveDocumentMessage(); - decodeInto(msg, buf); - return msg; - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - RemoveDocumentMessage msg = (RemoveDocumentMessage)obj; - msg.getDocumentId().serialize(buf); - return true; - } - } - - public static class RemoveDocumentReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - RemoveDocumentReply reply = new RemoveDocumentReply(); - byte flag = buf.getByte(null); - reply.setWasFound(flag != 0); - reply.setHighestModificationTimestamp(buf.getLong(null)); - return reply; - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - RemoveDocumentReply reply = (RemoveDocumentReply)obj; - buf.putByte(null, (byte)(reply.wasFound() ? 1 : 0)); - buf.putLong(null, reply.getHighestModificationTimestamp()); - return true; - } - } - - public static class RemoveLocationMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - return new RemoveLocationMessage(decodeString(buf)); - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - RemoveLocationMessage msg = (RemoveLocationMessage)obj; - encodeString(msg.getDocumentSelection(), buf); - return true; - } - } - - public static class RemoveLocationReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - return new DocumentReply(DocumentProtocol.REPLY_REMOVELOCATION); - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - return true; - } - } - - public static class SearchResultMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - SearchResultMessage msg = new SearchResultMessage(); - msg.setSearchResult(new SearchResult(buf)); - return msg; - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - return false; // not supported - } - } - - public static class QueryResultMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - QueryResultMessage msg = new QueryResultMessage(); - msg.setSearchResult(new SearchResult(buf)); - msg.setSummary(new DocumentSummary(buf)); - return msg; - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - return false; // not supported - } - } - - public static class SearchResultReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - return new VisitorReply(DocumentProtocol.REPLY_SEARCHRESULT); - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - return true; - } - } - - public static class QueryResultReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - return new VisitorReply(DocumentProtocol.REPLY_QUERYRESULT); - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - return true; - } - } - - public static class StatBucketMessageFactory extends DocumentMessageFactory { - - protected String decodeBucketSpace(Deserializer deserializer) { - return FixedBucketSpaces.defaultSpace(); - } - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - StatBucketMessage msg = new StatBucketMessage(); - msg.setBucketId(new BucketId(buf.getLong(null))); - msg.setDocumentSelection(decodeString(buf)); - msg.setBucketSpace(decodeBucketSpace(buf)); - return msg; - } - - protected boolean encodeBucketSpace(String bucketSpace, DocumentSerializer buf) { - return FixedBucketSpaces.defaultSpace().equals(bucketSpace); - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - StatBucketMessage msg = (StatBucketMessage)obj; - buf.putLong(null, msg.getBucketId().getRawId()); - encodeString(msg.getDocumentSelection(), buf); - return encodeBucketSpace(msg.getBucketSpace(), buf); - } - } - - public static class StatBucketReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - StatBucketReply reply = new StatBucketReply(); - reply.setResults(decodeString(buf)); - return reply; - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - StatBucketReply reply = (StatBucketReply)obj; - encodeString(reply.getResults(), buf); - return true; - } - } - - public static class UpdateDocumentMessageFactory extends DocumentMessageFactory { - protected void decodeInto(UpdateDocumentMessage msg, DocumentDeserializer buf) { - msg.setDocumentUpdate(new DocumentUpdate(buf)); - msg.setOldTimestamp(buf.getLong(null)); - msg.setNewTimestamp(buf.getLong(null)); - } - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buffer) { - final LazyDecoder decoder = (obj, buf) -> { - decodeInto((UpdateDocumentMessage) obj, buf); - }; - - return new UpdateDocumentMessage(decoder, buffer); - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - UpdateDocumentMessage msg = (UpdateDocumentMessage)obj; - if (msg.getSerializedBuffer() != null) { - buf.put(null, msg.getSerializedBuffer()); - } else { - msg.getDocumentUpdate().serialize(buf); - buf.putLong(null, msg.getOldTimestamp()); - buf.putLong(null, msg.getNewTimestamp()); - } - return true; - } - } - - public static class UpdateDocumentReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - UpdateDocumentReply rep = new UpdateDocumentReply(); - byte flag = buf.getByte(null); - rep.setWasFound(flag != 0); - rep.setHighestModificationTimestamp(buf.getLong(null)); - return rep; - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - UpdateDocumentReply rep = (UpdateDocumentReply)obj; - buf.putByte(null, (byte)(rep.wasFound() ? 1 : 0)); - buf.putLong(null, rep.getHighestModificationTimestamp()); - return true; - } - } - - public static class VisitorInfoMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - VisitorInfoMessage msg = new VisitorInfoMessage(); - int size = buf.getInt(null); - for (int i = 0; i < size; i++) { - long reversed = buf.getLong(null); - long rawid = ((reversed >>> 56) & 0x00000000000000FFl) | ((reversed >>> 40) & 0x000000000000FF00l) | - ((reversed >>> 24) & 0x0000000000FF0000l) | ((reversed >>> 8) & 0x00000000FF000000l) | - ((reversed << 8) & 0x000000FF00000000l) | ((reversed << 24) & 0x0000FF0000000000l) | - ((reversed << 40) & 0x00FF000000000000l) | ((reversed << 56) & 0xFF00000000000000l); - msg.getFinishedBuckets().add(new BucketId(rawid)); - } - - msg.setErrorMessage(decodeString(buf)); - return msg; - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - VisitorInfoMessage msg = (VisitorInfoMessage)obj; - buf.putInt(null, msg.getFinishedBuckets().size()); - for (BucketId id : msg.getFinishedBuckets()) { - long rawid = id.getRawId(); - long reversed = ((rawid >>> 56) & 0x00000000000000FFl) | ((rawid >>> 40) & 0x000000000000FF00l) | - ((rawid >>> 24) & 0x0000000000FF0000l) | ((rawid >>> 8) & 0x00000000FF000000l) | - ((rawid << 8) & 0x000000FF00000000l) | ((rawid << 24) & 0x0000FF0000000000l) | - ((rawid << 40) & 0x00FF000000000000l) | ((rawid << 56) & 0xFF00000000000000l); - buf.putLong(null, reversed); - } - encodeString(msg.getErrorMessage(), buf); - return true; - } - } - - public static class VisitorInfoReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - return new VisitorReply(DocumentProtocol.REPLY_VISITORINFO); - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - return true; - } - } - - public static class WrongDistributionReplyFactory extends DocumentReplyFactory { - - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - WrongDistributionReply reply = new WrongDistributionReply(); - reply.setSystemState(decodeString(buf)); - return reply; - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - WrongDistributionReply reply = (WrongDistributionReply)obj; - encodeString(reply.getSystemState(), buf); - return true; - } - } - -} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories51.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories51.java deleted file mode 100755 index 0ef5835008a..00000000000 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories51.java +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol; - -import com.yahoo.document.BucketId; -import com.yahoo.document.DocumentId; -import com.yahoo.document.FixedBucketSpaces; -import com.yahoo.document.serialization.DocumentDeserializer; -import com.yahoo.document.serialization.DocumentSerializer; -import com.yahoo.vespa.objects.Deserializer; - -import java.util.Map; - -/** - * This class encapsulates all the {@link RoutableFactory} classes needed to implement serialization for the document - * protocol. When adding new factories to this class, please KEEP THE THEM ORDERED alphabetically like they are now. - * - */ -public abstract class RoutableFactories51 extends RoutableFactories50 { - - public static class CreateVisitorMessageFactory extends DocumentMessageFactory { - - protected String decodeBucketSpace(Deserializer deserializer) { - return FixedBucketSpaces.defaultSpace(); - } - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - CreateVisitorMessage msg = new CreateVisitorMessage(); - msg.setLibraryName(decodeString(buf)); - msg.setInstanceId(decodeString(buf)); - msg.setControlDestination(decodeString(buf)); - msg.setDataDestination(decodeString(buf)); - msg.setDocumentSelection(decodeString(buf)); - msg.setMaxPendingReplyCount(buf.getInt(null)); - - int size = buf.getInt(null); - for (int i = 0; i < size; i++) { - long reversed = buf.getLong(null); - long rawid = ((reversed >>> 56) & 0x00000000000000FFl) | ((reversed >>> 40) & 0x000000000000FF00l) | - ((reversed >>> 24) & 0x0000000000FF0000l) | ((reversed >>> 8) & 0x00000000FF000000l) | - ((reversed << 8) & 0x000000FF00000000l) | ((reversed << 24) & 0x0000FF0000000000l) | - ((reversed << 40) & 0x00FF000000000000l) | ((reversed << 56) & 0xFF00000000000000l); - msg.getBuckets().add(new BucketId(rawid)); - } - - msg.setFromTimestamp(buf.getLong(null)); - msg.setToTimestamp(buf.getLong(null)); - msg.setVisitRemoves(buf.getByte(null) == (byte)1); - msg.setFieldSet(decodeString(buf)); - msg.setVisitInconsistentBuckets(buf.getByte(null) == (byte)1); - - size = buf.getInt(null); - for (int i = 0; i < size; i++) { - String key = decodeString(buf); - int sz = buf.getInt(null); - msg.getParameters().put(key, buf.getBytes(null, sz)); - } - - msg.setVisitorOrdering(buf.getInt(null)); - msg.setMaxBucketsPerVisitor(buf.getInt(null)); - msg.setVisitorDispatcherVersion(50); - msg.setBucketSpace(decodeBucketSpace(buf)); - return msg; - } - - protected boolean encodeBucketSpace(String bucketSpace, DocumentSerializer buf) { - return FixedBucketSpaces.defaultSpace().equals(bucketSpace); - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - CreateVisitorMessage msg = (CreateVisitorMessage)obj; - encodeString(msg.getLibraryName(), buf); - encodeString(msg.getInstanceId(), buf); - encodeString(msg.getControlDestination(), buf); - encodeString(msg.getDataDestination(), buf); - encodeString(msg.getDocumentSelection(), buf); - buf.putInt(null, msg.getMaxPendingReplyCount()); - - buf.putInt(null, msg.getBuckets().size()); - for (BucketId id : msg.getBuckets()) { - long rawid = id.getRawId(); - long reversed = ((rawid >>> 56) & 0x00000000000000FFl) | ((rawid >>> 40) & 0x000000000000FF00l) | - ((rawid >>> 24) & 0x0000000000FF0000l) | ((rawid >>> 8) & 0x00000000FF000000l) | - ((rawid << 8) & 0x000000FF00000000l) | ((rawid << 24) & 0x0000FF0000000000l) | - ((rawid << 40) & 0x00FF000000000000l) | ((rawid << 56) & 0xFF00000000000000l); - buf.putLong(null, reversed); - } - - buf.putLong(null, msg.getFromTimestamp()); - buf.putLong(null, msg.getToTimestamp()); - buf.putByte(null, msg.getVisitRemoves() ? (byte)1 : (byte)0); - encodeString(msg.getFieldSet(), buf); - buf.putByte(null, msg.getVisitInconsistentBuckets() ? (byte)1 : (byte)0); - - buf.putInt(null, msg.getParameters().size()); - for (Map.Entry<String, byte[]> pairs : msg.getParameters().entrySet()) { - encodeString(pairs.getKey(), buf); - byte[] b = pairs.getValue(); - buf.putInt(null, b.length); - buf.put(null, b); - } - - buf.putInt(null, msg.getVisitorOrdering()); - buf.putInt(null, msg.getMaxBucketsPerVisitor()); - return encodeBucketSpace(msg.getBucketSpace(), buf); - } - } - - public static class GetDocumentMessageFactory extends DocumentMessageFactory { - - @Override - protected DocumentMessage doDecode(DocumentDeserializer buf) { - return new GetDocumentMessage(new DocumentId(buf), decodeString(buf)); - } - - @Override - protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - GetDocumentMessage msg = (GetDocumentMessage)obj; - msg.getDocumentId().serialize(buf); - encodeString(msg.getFieldSet(), buf); - return true; - } - } - - public static class DocumentIgnoredReplyFactory extends DocumentReplyFactory { - @Override - protected DocumentReply doDecode(DocumentDeserializer buf) { - return new DocumentIgnoredReply(); - } - - @Override - protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { - return true; - } - } -} diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories52.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories52.java index 2e18c681c50..89a0aa22bb3 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories52.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories52.java @@ -1,82 +1,948 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi.messagebus.protocol; -import com.google.common.annotations.Beta; +import com.yahoo.document.BucketId; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.FixedBucketSpaces; import com.yahoo.document.TestAndSetCondition; import com.yahoo.document.serialization.DocumentDeserializer; import com.yahoo.document.serialization.DocumentSerializer; +import com.yahoo.document.serialization.DocumentSerializerFactory; +import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.log.LogLevel; +import com.yahoo.messagebus.Routable; +import com.yahoo.vdslib.DocumentList; +import com.yahoo.vdslib.DocumentSummary; +import com.yahoo.vdslib.SearchResult; +import com.yahoo.vdslib.VisitorStatistics; +import com.yahoo.vespa.objects.Deserializer; +import com.yahoo.vespa.objects.Serializer; + +import java.util.Map; +import java.util.logging.Logger; import static com.yahoo.documentapi.messagebus.protocol.AbstractRoutableFactory.decodeString; import static com.yahoo.documentapi.messagebus.protocol.AbstractRoutableFactory.encodeString; + /** * @author Vegard Sjonfjell + * This class encapsulates all the {@link RoutableFactory} classes needed to implement serialization for the document + * protocol. When adding new factories to this class, please KEEP THE THEM ORDERED alphabetically like they are now. */ +public abstract class RoutableFactories52 { + + /** + * Implements the shared factory logic required for {@link DocumentMessage} objects, and it offers a more convenient + * interface for implementing {@link RoutableFactory}. + * + * @author Simon Thoresen Hult + */ + public static abstract class DocumentMessageFactory extends AbstractRoutableFactory { + + /** + * This method encodes the given message using the given serializer. You are guaranteed to only receive messages + * of the type that this factory was registered for. + * <p> + * This method is NOT exception safe. Return false to + * signal failure. + * + * @param msg The message to encode. + * @param serializer The serializer to use for encoding. + * @return True if the message was encoded. + */ + protected abstract boolean doEncode(DocumentMessage msg, DocumentSerializer serializer); + + /** + * This method decodes a message from the given deserializer. You are guaranteed to only receive byte buffers + * generated by a previous call to {@link #doEncode(DocumentMessage, DocumentSerializer)}. + * <p> + * This method is NOT exception safe. Return null to signal failure. + * + * @param deserializer The deserializer to use for decoding. + * @return The decoded message. + */ + protected abstract DocumentMessage doDecode(DocumentDeserializer deserializer); + + public boolean encode(Routable obj, DocumentSerializer out) { + if (!(obj instanceof DocumentMessage)) { + throw new AssertionError( + "Document message factory (" + getClass().getName() + ") registered for incompatible " + + "routable type " + obj.getType() + "(" + obj.getClass().getName() + ")."); + } + DocumentMessage msg = (DocumentMessage)obj; + out.putByte(null, (byte)(msg.getPriority().getValue())); + out.putInt(null, msg.getLoadType().getId()); + return doEncode(msg, out); + } + + public Routable decode(DocumentDeserializer in, LoadTypeSet loadTypes) { + byte pri = in.getByte(null); + int loadType = in.getInt(null); + DocumentMessage msg = doDecode(in); + if (msg != null) { + msg.setPriority(DocumentProtocol.getPriority(pri)); + msg.setLoadType(loadTypes.getIdMap().get(loadType)); + } + return msg; + } + } + + /** + * Implements the shared factory logic required for {@link DocumentReply} objects, and it offers a more convenient + * interface for implementing {@link RoutableFactory}. + * + * @author Simon Thoresen Hult + */ + public static abstract class DocumentReplyFactory extends AbstractRoutableFactory { + + /** + * This method encodes the given reply into the given byte buffer. You are guaranteed to only receive replies of + * the type that this factory was registered for. + * <p> + * This method is NOT exception safe. Return false to signal + * failure. + * + * @param reply The reply to encode. + * @param buf The byte buffer to write to. + * @return True if the message was encoded. + */ + protected abstract boolean doEncode(DocumentReply reply, DocumentSerializer buf); + + /** + * This method decodes a reply from the given byte buffer. You are guaranteed to only receive byte buffers + * generated by a previous call to {@link #doEncode(DocumentReply, com.yahoo.document.serialization.DocumentSerializer)}. + * + * <p> + * This method is NOT exception safe. Return null to signal failure. + * + * @param buf The byte buffer to read from. + * @return The decoded reply. + */ + protected abstract DocumentReply doDecode(DocumentDeserializer buf); + + public boolean encode(Routable obj, DocumentSerializer out) { + if (!(obj instanceof DocumentReply)) { + throw new AssertionError( + "Document reply factory (" + getClass().getName() + ") registered for incompatible " + + "routable type " + obj.getType() + "(" + obj.getClass().getName() + ")."); + } + DocumentReply reply = (DocumentReply)obj; + out.putByte(null, (byte)(reply.getPriority().getValue())); + return doEncode(reply, out); + } + + public Routable decode(DocumentDeserializer in, LoadTypeSet loadTypes) { + byte pri = in.getByte(null); + DocumentReply reply = doDecode(in); + if (reply != null) { + reply.setPriority(DocumentProtocol.getPriority(pri)); + } + return reply; + } + } + + public static class CreateVisitorMessageFactory extends DocumentMessageFactory { + + protected String decodeBucketSpace(Deserializer deserializer) { + return FixedBucketSpaces.defaultSpace(); + } -@Beta -public abstract class RoutableFactories52 extends RoutableFactories51 { - protected static class PutDocumentMessageFactory extends RoutableFactories51.PutDocumentMessageFactory { @Override - protected void decodeInto(PutDocumentMessage msg, DocumentDeserializer buf) { - super.decodeInto(msg, buf); - decodeTasCondition(msg, buf); + protected DocumentMessage doDecode(DocumentDeserializer buf) { + CreateVisitorMessage msg = new CreateVisitorMessage(); + msg.setLibraryName(decodeString(buf)); + msg.setInstanceId(decodeString(buf)); + msg.setControlDestination(decodeString(buf)); + msg.setDataDestination(decodeString(buf)); + msg.setDocumentSelection(decodeString(buf)); + msg.setMaxPendingReplyCount(buf.getInt(null)); + + int size = buf.getInt(null); + for (int i = 0; i < size; i++) { + long reversed = buf.getLong(null); + long rawid = ((reversed >>> 56) & 0x00000000000000FFl) | ((reversed >>> 40) & 0x000000000000FF00l) | + ((reversed >>> 24) & 0x0000000000FF0000l) | ((reversed >>> 8) & 0x00000000FF000000l) | + ((reversed << 8) & 0x000000FF00000000l) | ((reversed << 24) & 0x0000FF0000000000l) | + ((reversed << 40) & 0x00FF000000000000l) | ((reversed << 56) & 0xFF00000000000000l); + msg.getBuckets().add(new BucketId(rawid)); + } + + msg.setFromTimestamp(buf.getLong(null)); + msg.setToTimestamp(buf.getLong(null)); + msg.setVisitRemoves(buf.getByte(null) == (byte)1); + msg.setFieldSet(decodeString(buf)); + msg.setVisitInconsistentBuckets(buf.getByte(null) == (byte)1); + + size = buf.getInt(null); + for (int i = 0; i < size; i++) { + String key = decodeString(buf); + int sz = buf.getInt(null); + msg.getParameters().put(key, buf.getBytes(null, sz)); + } + + msg.setVisitorOrdering(buf.getInt(null)); + msg.setMaxBucketsPerVisitor(buf.getInt(null)); + msg.setVisitorDispatcherVersion(50); + msg.setBucketSpace(decodeBucketSpace(buf)); + return msg; + } + + protected boolean encodeBucketSpace(String bucketSpace, DocumentSerializer buf) { + return FixedBucketSpaces.defaultSpace().equals(bucketSpace); } @Override protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - if (!super.doEncode(obj, buf)) { - return false; + CreateVisitorMessage msg = (CreateVisitorMessage)obj; + encodeString(msg.getLibraryName(), buf); + encodeString(msg.getInstanceId(), buf); + encodeString(msg.getControlDestination(), buf); + encodeString(msg.getDataDestination(), buf); + encodeString(msg.getDocumentSelection(), buf); + buf.putInt(null, msg.getMaxPendingReplyCount()); + + buf.putInt(null, msg.getBuckets().size()); + for (BucketId id : msg.getBuckets()) { + long rawid = id.getRawId(); + long reversed = ((rawid >>> 56) & 0x00000000000000FFl) | ((rawid >>> 40) & 0x000000000000FF00l) | + ((rawid >>> 24) & 0x0000000000FF0000l) | ((rawid >>> 8) & 0x00000000FF000000l) | + ((rawid << 8) & 0x000000FF00000000l) | ((rawid << 24) & 0x0000FF0000000000l) | + ((rawid << 40) & 0x00FF000000000000l) | ((rawid << 56) & 0xFF00000000000000l); + buf.putLong(null, reversed); } - // If the serialized buffer exists, the test and set condition has already been encoded - if (((PutDocumentMessage)obj).getSerializedBuffer() == null) { - encodeTasCondition(buf, (TestAndSetMessage) obj); + buf.putLong(null, msg.getFromTimestamp()); + buf.putLong(null, msg.getToTimestamp()); + buf.putByte(null, msg.getVisitRemoves() ? (byte)1 : (byte)0); + encodeString(msg.getFieldSet(), buf); + buf.putByte(null, msg.getVisitInconsistentBuckets() ? (byte)1 : (byte)0); + + buf.putInt(null, msg.getParameters().size()); + for (Map.Entry<String, byte[]> pairs : msg.getParameters().entrySet()) { + encodeString(pairs.getKey(), buf); + byte[] b = pairs.getValue(); + buf.putInt(null, b.length); + buf.put(null, b); + } + + buf.putInt(null, msg.getVisitorOrdering()); + buf.putInt(null, msg.getMaxBucketsPerVisitor()); + return encodeBucketSpace(msg.getBucketSpace(), buf); + } + } + + public static class CreateVisitorReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + CreateVisitorReply reply = new CreateVisitorReply(DocumentProtocol.REPLY_CREATEVISITOR); + reply.setLastBucket(new BucketId(buf.getLong(null))); + + VisitorStatistics vs = new VisitorStatistics(); + vs.setBucketsVisited(buf.getInt(null)); + vs.setDocumentsVisited(buf.getLong(null)); + vs.setBytesVisited(buf.getLong(null)); + vs.setDocumentsReturned(buf.getLong(null)); + vs.setBytesReturned(buf.getLong(null)); + vs.setSecondPassDocumentsReturned(buf.getLong(null)); + vs.setSecondPassBytesReturned(buf.getLong(null)); + reply.setVisitorStatistics(vs); + return reply; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + CreateVisitorReply reply = (CreateVisitorReply)obj; + buf.putLong(null, reply.getLastBucket().getRawId()); + buf.putInt(null, reply.getVisitorStatistics().getBucketsVisited()); + buf.putLong(null, reply.getVisitorStatistics().getDocumentsVisited()); + buf.putLong(null, reply.getVisitorStatistics().getBytesVisited()); + buf.putLong(null, reply.getVisitorStatistics().getDocumentsReturned()); + buf.putLong(null, reply.getVisitorStatistics().getBytesReturned()); + buf.putLong(null, reply.getVisitorStatistics().getSecondPassDocumentsReturned()); + buf.putLong(null, reply.getVisitorStatistics().getSecondPassBytesReturned()); + return true; + } + } + + public static class DestroyVisitorMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + DestroyVisitorMessage msg = new DestroyVisitorMessage(); + msg.setInstanceId(decodeString(buf)); + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + DestroyVisitorMessage msg = (DestroyVisitorMessage)obj; + encodeString(msg.getInstanceId(), buf); + return true; + } + } + + public static class DestroyVisitorReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new VisitorReply(DocumentProtocol.REPLY_DESTROYVISITOR); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class DocumentIgnoredReplyFactory extends DocumentReplyFactory { + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new DocumentIgnoredReply(); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class DocumentListMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + DocumentListMessage msg = new DocumentListMessage(); + msg.setBucketId(new BucketId(buf.getLong(null))); + int len = buf.getInt(null); + for (int i = 0; i < len; i++) { + msg.getDocuments().add(new DocumentListEntry(buf)); } + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + DocumentListMessage msg = (DocumentListMessage)obj; + buf.putLong(null, msg.getBucketId().getRawId()); + buf.putInt(null, msg.getDocuments().size()); + for (int i = 0; i < msg.getDocuments().size(); i++) { + msg.getDocuments().get(i).serialize(buf); + } return true; - } + } } - protected static class RemoveDocumentMessageFactory extends RoutableFactories51.RemoveDocumentMessageFactory { + public static class DocumentListReplyFactory extends DocumentReplyFactory { + @Override - protected void decodeInto(RemoveDocumentMessage msg, DocumentDeserializer buf) { - super.decodeInto(msg, buf); + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new VisitorReply(DocumentProtocol.REPLY_DOCUMENTLIST); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class DocumentSummaryMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + DocumentSummaryMessage msg = new DocumentSummaryMessage(); + msg.setDocumentSummary(new DocumentSummary(buf)); + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + return false; // not supported + } + } + + public static class DocumentSummaryReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new VisitorReply(DocumentProtocol.REPLY_DOCUMENTSUMMARY); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class EmptyBucketsMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + EmptyBucketsMessage msg = new EmptyBucketsMessage(); + int size = buf.getInt(null); + for (int i = 0; i < size; ++i) { + msg.getBucketIds().add(new BucketId(buf.getLong(null))); + } + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + EmptyBucketsMessage msg = (EmptyBucketsMessage)obj; + buf.putInt(null, msg.getBucketIds().size()); + for (BucketId bid : msg.getBucketIds()) { + buf.putLong(null, bid.getRawId()); + } + return true; + } + } + + public static class EmptyBucketsReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new VisitorReply(DocumentProtocol.REPLY_EMPTYBUCKETS); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class GetBucketListMessageFactory extends DocumentMessageFactory { + + protected String decodeBucketSpace(Deserializer deserializer) { + return FixedBucketSpaces.defaultSpace(); + } + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + GetBucketListMessage msg = new GetBucketListMessage(); + msg.setBucketId(new BucketId(buf.getLong(null))); + msg.setBucketSpace(decodeBucketSpace(buf)); + return msg; + } + + protected boolean encodeBucketSpace(String bucketSpace, DocumentSerializer buf) { + return FixedBucketSpaces.defaultSpace().equals(bucketSpace); + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + GetBucketListMessage msg = (GetBucketListMessage)obj; + buf.putLong(null, msg.getBucketId().getRawId()); + return encodeBucketSpace(msg.getBucketSpace(), buf); + } + } + + public static class GetBucketListReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + GetBucketListReply reply = new GetBucketListReply(); + int len = buf.getInt(null); + for (int i = 0; i < len; i++) { + GetBucketListReply.BucketInfo info = new GetBucketListReply.BucketInfo(); + info.bucket = new BucketId(buf.getLong(null)); + info.bucketInformation = decodeString(buf); + reply.getBuckets().add(info); + } + return reply; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + GetBucketListReply reply = (GetBucketListReply)obj; + buf.putInt(null, reply.getBuckets().size()); + for (GetBucketListReply.BucketInfo info : reply.getBuckets()) { + buf.putLong(null, info.bucket.getRawId()); + encodeString(info.bucketInformation, buf); + } + return true; + } + } + + public static class GetBucketStateMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + GetBucketStateMessage msg = new GetBucketStateMessage(); + msg.setBucketId(new BucketId(buf.getLong(null))); + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + GetBucketStateMessage msg = (GetBucketStateMessage)obj; + buf.putLong(null, msg.getBucketId().getRawId()); + return true; + } + } + + public static class GetBucketStateReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + GetBucketStateReply reply = new GetBucketStateReply(); + int size = buf.getInt(null); + for (int i = 0; i < size; i++) { + reply.getBucketState().add(new DocumentState(buf)); + } + return reply; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + GetBucketStateReply reply = (GetBucketStateReply)obj; + buf.putInt(null, reply.getBucketState().size()); + for (DocumentState stat : reply.getBucketState()) { + stat.serialize(buf); + } + return true; + } + } + + public static class GetDocumentMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + return new GetDocumentMessage(new DocumentId(buf), decodeString(buf)); + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + GetDocumentMessage msg = (GetDocumentMessage)obj; + msg.getDocumentId().serialize(buf); + encodeString(msg.getFieldSet(), buf); + return true; + } + } + + public static class GetDocumentReplyFactory extends DocumentReplyFactory { + + private final LazyDecoder decoder = new LazyDecoder() { + + public void decode(Routable obj, DocumentDeserializer buf) { + GetDocumentReply reply = (GetDocumentReply)obj; + + Document doc = null; + byte flag = buf.getByte(null); + if (flag != 0) { + doc = Document.createDocument(buf); + reply.setDocument(doc); + } + long lastModified = buf.getLong(null); + reply.setLastModified(lastModified); + if (doc != null) { + doc.setLastModified(lastModified); + } + } + }; + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + GetDocumentReply reply = new GetDocumentReply(decoder, buf); + + return reply; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + GetDocumentReply reply = (GetDocumentReply)obj; + if (reply.getSerializedBuffer() != null) { + buf.put(null, reply.getSerializedBuffer()); + } else { + Document document = reply.getDocument(); + buf.putByte(null, (byte)(document == null ? 0 : 1)); + if (document != null) { + document.serialize(buf); + } + buf.putLong(null, reply.getLastModified()); + } + return true; + } + } + + public static class MapVisitorMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + MapVisitorMessage msg = new MapVisitorMessage(); + int size = buf.getInt(null); + for (int i = 0; i < size; i++) { + String key = decodeString(buf); + String value = decodeString(buf); + msg.getData().put(key, value); + } + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + MapVisitorMessage msg = (MapVisitorMessage)obj; + buf.putInt(null, msg.getData().size()); + for (Map.Entry<String, String> pairs : msg.getData().entrySet()) { + encodeString(pairs.getKey(), buf); + encodeString(pairs.getValue(), buf); + } + return true; + } + } + + public static class MapVisitorReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new VisitorReply(DocumentProtocol.REPLY_MAPVISITOR); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class PutDocumentMessageFactory extends DocumentMessageFactory { + protected void decodeInto(PutDocumentMessage msg, DocumentDeserializer buf) { + msg.setDocumentPut(new DocumentPut(Document.createDocument(buf))); + msg.setTimestamp(buf.getLong(null)); decodeTasCondition(msg, buf); } @Override + protected DocumentMessage doDecode(DocumentDeserializer buffer) { + final LazyDecoder decoder = (obj, buf) -> { + decodeInto((PutDocumentMessage) obj, buf); + }; + + return new PutDocumentMessage(decoder, buffer); + } + + @Override protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - if (!super.doEncode(obj, buf)) { - return false; + PutDocumentMessage msg = (PutDocumentMessage)obj; + if (msg.getSerializedBuffer() != null) { + buf.put(null, msg.getSerializedBuffer()); + } else { + msg.getDocumentPut().getDocument().serialize(buf); + buf.putLong(null, msg.getTimestamp()); + encodeTasCondition(buf, (TestAndSetMessage) obj); } + return true; + } + } + public static class PutDocumentReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + WriteDocumentReply rep = new WriteDocumentReply(DocumentProtocol.REPLY_PUTDOCUMENT); + rep.setHighestModificationTimestamp(buf.getLong(null)); + return rep; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + WriteDocumentReply rep = (WriteDocumentReply)obj; + buf.putLong(null, rep.getHighestModificationTimestamp()); + return true; + } + } + + public static class RemoveDocumentMessageFactory extends DocumentMessageFactory { + protected void decodeInto(RemoveDocumentMessage msg, DocumentDeserializer buf) { + msg.setDocumentId(new DocumentId(buf)); + decodeTasCondition(msg, buf); + } + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + RemoveDocumentMessage msg = new RemoveDocumentMessage(); + decodeInto(msg, buf); + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + RemoveDocumentMessage msg = (RemoveDocumentMessage)obj; + msg.getDocumentId().serialize(buf); encodeTasCondition(buf, (TestAndSetMessage) obj); return true; } } - protected static class UpdateDocumentMessageFactory extends RoutableFactories51.UpdateDocumentMessageFactory { + public static class RemoveDocumentReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + RemoveDocumentReply reply = new RemoveDocumentReply(); + byte flag = buf.getByte(null); + reply.setWasFound(flag != 0); + reply.setHighestModificationTimestamp(buf.getLong(null)); + return reply; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + RemoveDocumentReply reply = (RemoveDocumentReply)obj; + buf.putByte(null, (byte)(reply.wasFound() ? 1 : 0)); + buf.putLong(null, reply.getHighestModificationTimestamp()); + return true; + } + } + + public static class RemoveLocationMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + return new RemoveLocationMessage(decodeString(buf)); + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + RemoveLocationMessage msg = (RemoveLocationMessage)obj; + encodeString(msg.getDocumentSelection(), buf); + return true; + } + } + + public static class RemoveLocationReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new DocumentReply(DocumentProtocol.REPLY_REMOVELOCATION); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class SearchResultMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + SearchResultMessage msg = new SearchResultMessage(); + msg.setSearchResult(new SearchResult(buf)); + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + return false; // not supported + } + } + + public static class QueryResultMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + QueryResultMessage msg = new QueryResultMessage(); + msg.setSearchResult(new SearchResult(buf)); + msg.setSummary(new DocumentSummary(buf)); + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + return false; // not supported + } + } + + public static class SearchResultReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new VisitorReply(DocumentProtocol.REPLY_SEARCHRESULT); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class QueryResultReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new VisitorReply(DocumentProtocol.REPLY_QUERYRESULT); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class StatBucketMessageFactory extends DocumentMessageFactory { + + protected String decodeBucketSpace(Deserializer deserializer) { + return FixedBucketSpaces.defaultSpace(); + } + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + StatBucketMessage msg = new StatBucketMessage(); + msg.setBucketId(new BucketId(buf.getLong(null))); + msg.setDocumentSelection(decodeString(buf)); + msg.setBucketSpace(decodeBucketSpace(buf)); + return msg; + } + + protected boolean encodeBucketSpace(String bucketSpace, DocumentSerializer buf) { + return FixedBucketSpaces.defaultSpace().equals(bucketSpace); + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + StatBucketMessage msg = (StatBucketMessage)obj; + buf.putLong(null, msg.getBucketId().getRawId()); + encodeString(msg.getDocumentSelection(), buf); + return encodeBucketSpace(msg.getBucketSpace(), buf); + } + } + + public static class StatBucketReplyFactory extends DocumentReplyFactory { + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + StatBucketReply reply = new StatBucketReply(); + reply.setResults(decodeString(buf)); + return reply; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + StatBucketReply reply = (StatBucketReply)obj; + encodeString(reply.getResults(), buf); + return true; + } + } + + public static class UpdateDocumentMessageFactory extends DocumentMessageFactory { protected void decodeInto(UpdateDocumentMessage msg, DocumentDeserializer buf) { - super.decodeInto(msg, buf); + msg.setDocumentUpdate(new DocumentUpdate(buf)); + msg.setOldTimestamp(buf.getLong(null)); + msg.setNewTimestamp(buf.getLong(null)); decodeTasCondition(msg, buf); } @Override + protected DocumentMessage doDecode(DocumentDeserializer buffer) { + final LazyDecoder decoder = (obj, buf) -> { + decodeInto((UpdateDocumentMessage) obj, buf); + }; + + return new UpdateDocumentMessage(decoder, buffer); + } + + @Override protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { - if (!super.doEncode(obj, buf)) { - return false; + UpdateDocumentMessage msg = (UpdateDocumentMessage)obj; + if (msg.getSerializedBuffer() != null) { + buf.put(null, msg.getSerializedBuffer()); + } else { + msg.getDocumentUpdate().serialize(buf); + buf.putLong(null, msg.getOldTimestamp()); + buf.putLong(null, msg.getNewTimestamp()); + encodeTasCondition(buf, (TestAndSetMessage) obj); } + return true; + } + } - // If the serialized buffer exists, the test and set condition has already been encoded - if (((UpdateDocumentMessage)obj).getSerializedBuffer() == null) { - encodeTasCondition(buf, (TestAndSetMessage) obj); + public static class UpdateDocumentReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + UpdateDocumentReply rep = new UpdateDocumentReply(); + byte flag = buf.getByte(null); + rep.setWasFound(flag != 0); + rep.setHighestModificationTimestamp(buf.getLong(null)); + return rep; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + UpdateDocumentReply rep = (UpdateDocumentReply)obj; + buf.putByte(null, (byte)(rep.wasFound() ? 1 : 0)); + buf.putLong(null, rep.getHighestModificationTimestamp()); + return true; + } + } + + public static class VisitorInfoMessageFactory extends DocumentMessageFactory { + + @Override + protected DocumentMessage doDecode(DocumentDeserializer buf) { + VisitorInfoMessage msg = new VisitorInfoMessage(); + int size = buf.getInt(null); + for (int i = 0; i < size; i++) { + long reversed = buf.getLong(null); + long rawid = ((reversed >>> 56) & 0x00000000000000FFl) | ((reversed >>> 40) & 0x000000000000FF00l) | + ((reversed >>> 24) & 0x0000000000FF0000l) | ((reversed >>> 8) & 0x00000000FF000000l) | + ((reversed << 8) & 0x000000FF00000000l) | ((reversed << 24) & 0x0000FF0000000000l) | + ((reversed << 40) & 0x00FF000000000000l) | ((reversed << 56) & 0xFF00000000000000l); + msg.getFinishedBuckets().add(new BucketId(rawid)); } + msg.setErrorMessage(decodeString(buf)); + return msg; + } + + @Override + protected boolean doEncode(DocumentMessage obj, DocumentSerializer buf) { + VisitorInfoMessage msg = (VisitorInfoMessage)obj; + buf.putInt(null, msg.getFinishedBuckets().size()); + for (BucketId id : msg.getFinishedBuckets()) { + long rawid = id.getRawId(); + long reversed = ((rawid >>> 56) & 0x00000000000000FFl) | ((rawid >>> 40) & 0x000000000000FF00l) | + ((rawid >>> 24) & 0x0000000000FF0000l) | ((rawid >>> 8) & 0x00000000FF000000l) | + ((rawid << 8) & 0x000000FF00000000l) | ((rawid << 24) & 0x0000FF0000000000l) | + ((rawid << 40) & 0x00FF000000000000l) | ((rawid << 56) & 0xFF00000000000000l); + buf.putLong(null, reversed); + } + encodeString(msg.getErrorMessage(), buf); return true; } } + public static class VisitorInfoReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + return new VisitorReply(DocumentProtocol.REPLY_VISITORINFO); + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + return true; + } + } + + public static class WrongDistributionReplyFactory extends DocumentReplyFactory { + + @Override + protected DocumentReply doDecode(DocumentDeserializer buf) { + WrongDistributionReply reply = new WrongDistributionReply(); + reply.setSystemState(decodeString(buf)); + return reply; + } + + @Override + protected boolean doEncode(DocumentReply obj, DocumentSerializer buf) { + WrongDistributionReply reply = (WrongDistributionReply)obj; + encodeString(reply.getSystemState(), buf); + return true; + } + } static void decodeTasCondition(TestAndSetMessage msg, DocumentDeserializer buf) { msg.setCondition(new TestAndSetCondition(decodeString(buf))); } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocolTest.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocolTest.java deleted file mode 100644 index 50778e00519..00000000000 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocolTest.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol; - -import com.yahoo.component.Version; -import com.yahoo.document.DocumentId; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.DocumentTypeManagerConfigurer; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author Simon Thoresen Hult - */ -public class DocumentProtocolTest { - - private final DocumentTypeManager manager = new DocumentTypeManager(); - - @Before - public void setUp() { - DocumentTypeManagerConfigurer.configure(manager, "file:./test/cfg/testdoc.cfg"); - } - - @SuppressWarnings("deprecation") - @Test - public void requireThat50SerializationPrecedes5xSerialization() { - DocumentProtocol protocol = new DocumentProtocol(manager); - GetDocumentMessage prev = new GetDocumentMessage(new DocumentId("doc:scheme:"), "foo"); - byte[] buf = protocol.encode(new Version(5, 0), prev); - - GetDocumentMessage next = (GetDocumentMessage)protocol.decode(new Version(5, 0), buf); - assertEquals(GetDocumentMessage.DEFAULT_FIELD_SET, next.getFieldSet()); - } - -} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages50TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages50TestCase.java deleted file mode 100644 index 5eca8f49967..00000000000 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages50TestCase.java +++ /dev/null @@ -1,904 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol.test; - -import com.yahoo.component.Version; -import com.yahoo.document.*; -import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate; -import com.yahoo.document.idstring.IdString; -import com.yahoo.document.select.OrderingSpecification; -import com.yahoo.documentapi.messagebus.protocol.*; -import com.yahoo.messagebus.Routable; -import com.yahoo.text.Utf8; -import com.yahoo.vdslib.DocumentList; -import com.yahoo.vdslib.Entry; -import com.yahoo.vdslib.SearchResult; - -import java.io.FileInputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.*; - -/** - * @author Simon Thoresen Hult - */ -public class Messages50TestCase extends MessagesTestBase { - - @Override - protected void registerTests(Map<Integer, RunnableTest> out) { - // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support - // version 5.0. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now. - out.put(DocumentProtocol.MESSAGE_CREATEVISITOR, new testCreateVisitorMessage()); - out.put(DocumentProtocol.MESSAGE_DESTROYVISITOR, new testDestroyVisitorMessage()); - out.put(DocumentProtocol.MESSAGE_DOCUMENTLIST, new testDocumentListMessage()); - out.put(DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, new testDocumentSummaryMessage()); - out.put(DocumentProtocol.MESSAGE_EMPTYBUCKETS, new testEmptyBucketsMessage()); - out.put(DocumentProtocol.MESSAGE_GETBUCKETLIST, new testGetBucketListMessage()); - out.put(DocumentProtocol.MESSAGE_GETBUCKETSTATE, new testGetBucketStateMessage()); - out.put(DocumentProtocol.MESSAGE_GETDOCUMENT, new testGetDocumentMessage()); - out.put(DocumentProtocol.MESSAGE_MAPVISITOR, new testMapVisitorMessage()); - out.put(DocumentProtocol.MESSAGE_PUTDOCUMENT, new testPutDocumentMessage()); - out.put(DocumentProtocol.MESSAGE_QUERYRESULT, new testQueryResultMessage()); - out.put(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, new testRemoveDocumentMessage()); - out.put(DocumentProtocol.MESSAGE_REMOVELOCATION, new testRemoveLocationMessage()); - out.put(DocumentProtocol.MESSAGE_SEARCHRESULT, new testSearchResultMessage()); - out.put(DocumentProtocol.MESSAGE_STATBUCKET, new testStatBucketMessage()); - out.put(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, new testUpdateDocumentMessage()); - out.put(DocumentProtocol.MESSAGE_VISITORINFO, new testVisitorInfoMessage()); - out.put(DocumentProtocol.REPLY_CREATEVISITOR, new testCreateVisitorReply()); - out.put(DocumentProtocol.REPLY_DESTROYVISITOR, new testDestroyVisitorReply()); - out.put(DocumentProtocol.REPLY_DOCUMENTLIST, new testDocumentListReply()); - out.put(DocumentProtocol.REPLY_DOCUMENTSUMMARY, new testDocumentSummaryReply()); - out.put(DocumentProtocol.REPLY_EMPTYBUCKETS, new testEmptyBucketsReply()); - out.put(DocumentProtocol.REPLY_GETBUCKETLIST, new testGetBucketListReply()); - out.put(DocumentProtocol.REPLY_GETBUCKETSTATE, new testGetBucketStateReply()); - out.put(DocumentProtocol.REPLY_GETDOCUMENT, new testGetDocumentReply()); - out.put(DocumentProtocol.REPLY_MAPVISITOR, new testMapVisitorReply()); - out.put(DocumentProtocol.REPLY_PUTDOCUMENT, new testPutDocumentReply()); - out.put(DocumentProtocol.REPLY_QUERYRESULT, new testQueryResultReply()); - out.put(DocumentProtocol.REPLY_REMOVEDOCUMENT, new testRemoveDocumentReply()); - out.put(DocumentProtocol.REPLY_REMOVELOCATION, new testRemoveLocationReply()); - out.put(DocumentProtocol.REPLY_SEARCHRESULT, new testSearchResultReply()); - out.put(DocumentProtocol.REPLY_STATBUCKET, new testStatBucketReply()); - out.put(DocumentProtocol.REPLY_UPDATEDOCUMENT, new testUpdateDocumentReply()); - out.put(DocumentProtocol.REPLY_VISITORINFO, new testVisitorInfoReply()); - out.put(DocumentProtocol.REPLY_WRONGDISTRIBUTION, new testWrongDistributionReply()); - } - - @Override - protected Version version() { - return new Version(5, 0); - } - - @Override - protected boolean shouldTestCoverage() { - return false; - } - - //////////////////////////////////////////////////////////////////////////////// - // - // Tests - // - //////////////////////////////////////////////////////////////////////////////// - - private static int BASE_MESSAGE_LENGTH = 5; - - public class testRemoveLocationMessage implements RunnableTest { - - @Override - public void run() { - { - RemoveLocationMessage msg = new RemoveLocationMessage("id.group == \"mygroup\""); - assertEquals(BASE_MESSAGE_LENGTH + 29, serialize("RemoveLocationMessage", msg)); - - for (Language lang : LANGUAGES) { - msg = (RemoveLocationMessage)deserialize("RemoveLocationMessage", DocumentProtocol.MESSAGE_REMOVELOCATION, lang); - assertEquals("id.group == \"mygroup\"", msg.getDocumentSelection()); - } - } - } - } - - public class testGetBucketListMessage implements RunnableTest { - - @Override - public void run() { - GetBucketListMessage msg = new GetBucketListMessage(new BucketId(16, 123)); - msg.setLoadType(loadTypes.getNameMap().get("foo")); - assertEquals(BASE_MESSAGE_LENGTH + 12, serialize("GetBucketListMessage", msg)); - - for (Language lang : LANGUAGES) { - msg = (GetBucketListMessage)deserialize("GetBucketListMessage", DocumentProtocol.MESSAGE_GETBUCKETLIST, lang); - assertEquals(new BucketId(16, 123), msg.getBucketId()); - assertEquals("foo", msg.getLoadType().getName()); - } - } - } - - - public class testStatBucketMessage implements RunnableTest { - - @Override - public void run() { - StatBucketMessage msg = new StatBucketMessage(new BucketId(16, 123), "id.user=123"); - msg.setLoadType(null); - assertEquals(BASE_MESSAGE_LENGTH + 27, serialize("StatBucketMessage", msg)); - - for (Language lang : LANGUAGES) { - msg = (StatBucketMessage)deserialize("StatBucketMessage", DocumentProtocol.MESSAGE_STATBUCKET, lang); - assertEquals(new BucketId(16, 123), msg.getBucketId()); - assertEquals("id.user=123", msg.getDocumentSelection()); - assertEquals("default", msg.getLoadType().getName()); - } - } - } - - public class testGetBucketStateMessage implements RunnableTest { - - @Override - public void run() { - GetBucketStateMessage msg = new GetBucketStateMessage(new BucketId(16, 666)); - assertEquals(BASE_MESSAGE_LENGTH + 12, serialize("GetBucketStateMessage", msg)); - - for (Language lang : LANGUAGES) { - msg = (GetBucketStateMessage)deserialize("GetBucketStateMessage", DocumentProtocol.MESSAGE_GETBUCKETSTATE, lang); - assertEquals(16, msg.getBucketId().getUsedBits()); - assertEquals(4611686018427388570l, msg.getBucketId().getId()); - } - } - } - - public class testCreateVisitorMessage implements RunnableTest { - - @Override - @SuppressWarnings("deprecation") - public void run() { - CreateVisitorMessage msg = new CreateVisitorMessage("SomeLibrary", "myvisitor", "newyork", "london"); - msg.setDocumentSelection("true and false or true"); - msg.getParameters().put("myvar", Utf8.toBytes("somevalue")); - msg.getParameters().put("anothervar", Utf8.toBytes("34")); - msg.getBuckets().add(new BucketId(16, 1234)); - msg.setVisitRemoves(true); - msg.setVisitorOrdering(OrderingSpecification.DESCENDING); - msg.setMaxBucketsPerVisitor(2); - assertEquals(BASE_MESSAGE_LENGTH + 168, serialize("CreateVisitorMessage", msg)); - - for (Language lang : LANGUAGES) { - msg = (CreateVisitorMessage)deserialize("CreateVisitorMessage", DocumentProtocol.MESSAGE_CREATEVISITOR, lang); - assertEquals("SomeLibrary", msg.getLibraryName()); - assertEquals("myvisitor", msg.getInstanceId()); - assertEquals("newyork", msg.getControlDestination()); - assertEquals("london", msg.getDataDestination()); - assertEquals("true and false or true", msg.getDocumentSelection()); - assertEquals(8, msg.getMaxPendingReplyCount()); - assertEquals(true, msg.getVisitRemoves()); - assertEquals(false, msg.getVisitInconsistentBuckets()); - assertEquals(1, msg.getBuckets().size()); - assertEquals(new BucketId(16, 1234), msg.getBuckets().iterator().next()); - assertEquals("somevalue", Utf8.toString(msg.getParameters().get("myvar"))); - assertEquals("34", Utf8.toString(msg.getParameters().get("anothervar"))); - assertEquals(OrderingSpecification.DESCENDING, msg.getVisitorOrdering()); - assertEquals(2, msg.getMaxBucketsPerVisitor()); - } - - msg.getBuckets().clear(); - - assertEquals("CreateVisitorMessage(" + - "No buckets, " + - "selection 'true and false or true', " + - "bucket space 'default', " + - "library SomeLibrary, including removes, " + - "get fields: [all]" + - ")", - msg.toString()); - - msg.getBuckets().add(new BucketId(16, 1234)); - - assertEquals("CreateVisitorMessage(" + - "Bucket BucketId(0x40000000000004d2), " + - "selection 'true and false or true', " + - "bucket space 'default', " + - "library SomeLibrary, including removes, " + - "get fields: [all]" + - ")", - msg.toString()); - - msg.getBuckets().add(new BucketId(16, 1235)); - msg.getBuckets().add(new BucketId(16, 1236)); - msg.getBuckets().add(new BucketId(16, 1237)); - msg.getBuckets().add(new BucketId(16, 1238)); - msg.setFromTimestamp(10001); - msg.setToTimestamp(20002); - msg.setVisitInconsistentBuckets(true); - assertEquals("CreateVisitorMessage(" + - "5 buckets: BucketId(0x40000000000004d2) BucketId(0x40000000000004d3) BucketId(0x40000000000004d4) ..., " + - "time 10001-20002, " + - "selection 'true and false or true', " + - "bucket space 'default', " + - "library SomeLibrary, including removes, " + - "get fields: [all], " + - "visit inconsistent buckets" + - ")", - msg.toString()); - } - } - - public class testCreateVisitorReply implements RunnableTest { - - @Override - public void run() { - CreateVisitorReply reply = new CreateVisitorReply(DocumentProtocol.REPLY_CREATEVISITOR); - reply.setLastBucket(new BucketId(16, 123)); - reply.getVisitorStatistics().setBucketsVisited(3); - reply.getVisitorStatistics().setDocumentsVisited(1000); - reply.getVisitorStatistics().setBytesVisited(1024000); - reply.getVisitorStatistics().setDocumentsReturned(123); - reply.getVisitorStatistics().setBytesReturned(512000); - reply.getVisitorStatistics().setSecondPassDocumentsReturned(456); - reply.getVisitorStatistics().setSecondPassBytesReturned(789100); - - assertEquals(65, serialize("CreateVisitorReply", reply)); - - for (Language lang : LANGUAGES) { - reply = (CreateVisitorReply)deserialize("CreateVisitorReply", DocumentProtocol.REPLY_CREATEVISITOR, lang); - assertNotNull(reply); - assertEquals(new BucketId(16, 123), reply.getLastBucket()); - assertEquals(3, reply.getVisitorStatistics().getBucketsVisited()); - assertEquals(1000, reply.getVisitorStatistics().getDocumentsVisited()); - assertEquals(1024000, reply.getVisitorStatistics().getBytesVisited()); - assertEquals(123, reply.getVisitorStatistics().getDocumentsReturned()); - assertEquals(512000, reply.getVisitorStatistics().getBytesReturned()); - assertEquals(456, reply.getVisitorStatistics().getSecondPassDocumentsReturned()); - assertEquals(789100, reply.getVisitorStatistics().getSecondPassBytesReturned()); - } - } - } - - public class testDestroyVisitorReply implements RunnableTest { - - @Override - public void run() { - testVisitorReply("DestroyVisitorReply", DocumentProtocol.REPLY_DESTROYVISITOR); - } - } - - public class testDocumentListReply implements RunnableTest { - - @Override - public void run() { - testVisitorReply("DocumentListReply", DocumentProtocol.REPLY_DOCUMENTLIST); - } - } - - public class testDocumentSummaryReply implements RunnableTest { - - @Override - public void run() { - testVisitorReply("DocumentSummaryReply", DocumentProtocol.REPLY_DOCUMENTSUMMARY); - } - } - - public class testEmptyBucketsReply implements RunnableTest { - - @Override - public void run() { - testVisitorReply("EmptyBucketsReply", DocumentProtocol.REPLY_EMPTYBUCKETS); - } - } - - public class testDestroyVisitorMessage implements RunnableTest { - - @Override - public void run() { - DestroyVisitorMessage msg = new DestroyVisitorMessage("myvisitor"); - assertEquals(BASE_MESSAGE_LENGTH + 17, serialize("DestroyVisitorMessage", msg)); - - for (Language lang : LANGUAGES) { - msg = (DestroyVisitorMessage)deserialize("DestroyVisitorMessage", DocumentProtocol.MESSAGE_DESTROYVISITOR, lang); - assertEquals("myvisitor", msg.getInstanceId()); - } - } - } - - public class testDocumentListMessage implements RunnableTest { - - @Override - public void run() { - DocumentListMessage msg = (DocumentListMessage)deserialize("DocumentListMessage", DocumentProtocol.MESSAGE_DOCUMENTLIST, Language.CPP); - assertEquals("userdoc:scheme:1234:", msg.getDocuments().get(0).getDocument().getId().toString()); - assertEquals(1234, msg.getDocuments().get(0).getTimestamp()); - assertFalse(msg.getDocuments().get(0).isRemoveEntry()); - - assertEquals(BASE_MESSAGE_LENGTH + 63, serialize("DocumentListMessage", msg)); - msg = (DocumentListMessage)deserialize("DocumentListMessage", DocumentProtocol.MESSAGE_DOCUMENTLIST, Language.JAVA); - assertEquals("userdoc:scheme:1234:", msg.getDocuments().get(0).getDocument().getId().toString()); - assertEquals(1234, msg.getDocuments().get(0).getTimestamp()); - assertFalse(msg.getDocuments().get(0).isRemoveEntry()); - - } - } - - public class testEmptyBucketsMessage implements RunnableTest { - - @Override - public void run() { - List<BucketId> bids = new ArrayList<>(); - for (int i = 0; i < 13; ++i) { - bids.add(new BucketId(16, i)); - } - - EmptyBucketsMessage ebm = new EmptyBucketsMessage(bids); - assertEquals(BASE_MESSAGE_LENGTH + 112, serialize("EmptyBucketsMessage", ebm)); - for (Language lang : LANGUAGES) { - ebm = (EmptyBucketsMessage)deserialize("EmptyBucketsMessage", DocumentProtocol.MESSAGE_EMPTYBUCKETS, lang); - for (int i = 0; i < 13; ++i) { - assertEquals(new BucketId(16, i), ebm.getBucketIds().get(i)); - } - } - } - } - - public class testDocumentSummaryMessage implements RunnableTest { - - @Override - public void run() { - try { - FileInputStream stream = new FileInputStream(getPath("5-cpp-DocumentSummaryMessage-1.dat")); - byte[] data = new byte[stream.available()]; - assertEquals(data.length, stream.read(data)); - - Routable routable = decode(data); - assertTrue(routable instanceof DocumentSummaryMessage); - - DocumentSummaryMessage msg = (DocumentSummaryMessage)routable; - assertEquals(0, msg.getResult().getSummaryCount()); - - stream = new FileInputStream(getPath("5-cpp-DocumentSummaryMessage-2.dat")); - data = new byte[stream.available()]; - assertEquals(data.length, stream.read(data)); - - routable = decode(data); - assertTrue(routable instanceof DocumentSummaryMessage); - - msg = (DocumentSummaryMessage)routable; - assertEquals(2, msg.getResult().getSummaryCount()); - com.yahoo.vdslib.DocumentSummary.Summary s = msg.getResult().getSummary(0); - assertEquals("doc1", s.getDocId()); - byte[] b = s.getSummary(); - assertEquals(8, b.length); - byte[] c = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '1' }; - for (int i = 0; i < b.length; i++) { - assertEquals(c[i], b[i]); - } - - s = msg.getResult().getSummary(1); - assertEquals("aoc17", s.getDocId()); - b = s.getSummary(); - assertEquals(9, b.length); - byte[] d = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5' }; - for (int i = 0; i < b.length; i++) { - assertEquals(d[i], b[i]); - } - - stream = new FileInputStream(getPath("5-cpp-DocumentSummaryMessage-3.dat")); - data = new byte[stream.available()]; - assertEquals(data.length, stream.read(data)); - - routable = decode(data); - assertTrue(routable instanceof DocumentSummaryMessage); - - msg = (DocumentSummaryMessage)routable; - assertEquals(2, msg.getResult().getSummaryCount()); - - s = msg.getResult().getSummary(0); - assertEquals("aoc17", s.getDocId()); - b = s.getSummary(); - assertEquals(9, b.length); - byte[] e = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5' }; - for (int i = 0; i < b.length; i++) { - assertEquals(e[i], b[i]); - } - - s = msg.getResult().getSummary(1); - assertEquals("doc1", s.getDocId()); - b = s.getSummary(); - assertEquals(8, b.length); - byte[] f = { 's', 'u', 'm', 'm', 'a', 'r', 'y', '1' }; - for (int i = 0; i < b.length; i++) { - assertEquals(f[i], b[i]); - } - } catch (IOException e) { - fail(e.toString()); - } - } - } - - - public class testGetDocumentMessage implements RunnableTest { - - @Override - @SuppressWarnings("deprecation") - public void run() { - GetDocumentMessage msg = new GetDocumentMessage(new DocumentId("doc:scheme:")); - assertEquals(BASE_MESSAGE_LENGTH + 20, serialize("GetDocumentMessage", msg)); - - for (Language lang : LANGUAGES) { - msg = (GetDocumentMessage)deserialize("GetDocumentMessage", DocumentProtocol.MESSAGE_GETDOCUMENT, lang); - assertEquals("doc:scheme:", msg.getDocumentId().toString()); - } - } - } - - - public class testRemoveDocumentMessage implements RunnableTest { - - @Override - public void run() { - RemoveDocumentMessage msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:")); - assertEquals(BASE_MESSAGE_LENGTH + 16, serialize("RemoveDocumentMessage", msg)); - - for (Language lang : LANGUAGES) { - msg = (RemoveDocumentMessage)deserialize("RemoveDocumentMessage", DocumentProtocol.MESSAGE_REMOVEDOCUMENT, lang); - assertEquals("doc:scheme:", msg.getDocumentId().toString()); - } - } - } - - public class testMapVisitorMessage implements RunnableTest { - - @Override - public void run() { - MapVisitorMessage msg = (MapVisitorMessage)deserialize("MapVisitorMessage", DocumentProtocol.MESSAGE_MAPVISITOR, Language.CPP); - assertEquals("3", msg.getData().get("foo")); - assertEquals("5", msg.getData().get("bar")); - - assertEquals(BASE_MESSAGE_LENGTH + 32, serialize("MapVisitorMessage", msg)); - - msg = (MapVisitorMessage)deserialize("MapVisitorMessage", DocumentProtocol.MESSAGE_MAPVISITOR, Language.JAVA); - assertEquals("3", msg.getData().get("foo")); - assertEquals("5", msg.getData().get("bar")); - } - } - - - public class testVisitorInfoMessage implements RunnableTest { - - @Override - public void run() { - VisitorInfoMessage msg = new VisitorInfoMessage(); - msg.getFinishedBuckets().add(new BucketId(16, 1)); - msg.getFinishedBuckets().add(new BucketId(16, 2)); - msg.getFinishedBuckets().add(new BucketId(16, 4)); - msg.setErrorMessage("error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6"); - assertEquals(BASE_MESSAGE_LENGTH + 67, serialize("VisitorInfoMessage", msg)); - - for (Language lang : LANGUAGES) { - msg = (VisitorInfoMessage)deserialize("VisitorInfoMessage", DocumentProtocol.MESSAGE_VISITORINFO, lang); - assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 1))); - assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 2))); - assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 4))); - assertEquals("error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6", msg.getErrorMessage()); - } - } - } - - public class testSearchResultMessage implements RunnableTest { - - @Override - public void run() throws Exception { - FileInputStream stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-1.dat")); - byte[] data = new byte[stream.available()]; - assertEquals(data.length, stream.read(data)); - - Routable routable = decode(data); - assertTrue(routable instanceof SearchResultMessage); - - SearchResultMessage msg = (SearchResultMessage)routable; - assertEquals(0, msg.getResult().getHitCount()); - - stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-2.dat")); - data = new byte[stream.available()]; - assertEquals(data.length, stream.read(data)); - - routable = decode(data); - assertTrue(routable instanceof SearchResultMessage); - - msg = (SearchResultMessage)routable; - assertEquals(2, msg.getResult().getHitCount()); - com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0); - assertEquals(89.0, h.getRank(), 1E-6); - assertEquals("doc1", h.getDocId()); - h = msg.getResult().getHit(1); - assertEquals(109.0, h.getRank(), 1E-6); - assertEquals("doc17", h.getDocId()); - - stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-3.dat")); - data = new byte[stream.available()]; - assertEquals(data.length, stream.read(data)); - - routable = decode(data); - assertTrue(routable instanceof SearchResultMessage); - - msg = (SearchResultMessage)routable; - assertEquals(2, msg.getResult().getHitCount()); - h = msg.getResult().getHit(0); - assertEquals(109.0, h.getRank(), 1E-6); - assertEquals("doc17", h.getDocId()); - h = msg.getResult().getHit(1); - assertEquals(89.0, h.getRank(), 1E-6); - assertEquals("doc1", h.getDocId()); - - stream = new FileInputStream(getPath("5-cpp-SearchResultMessage-4.dat")); - data = new byte[stream.available()]; - assertEquals(data.length, stream.read(data)); - - routable = decode(data); - assertTrue(routable instanceof SearchResultMessage); - - msg = (SearchResultMessage)routable; - assertEquals(3, msg.getResult().getHitCount()); - h = msg.getResult().getHit(0); - assertTrue(h instanceof SearchResult.HitWithSortBlob); - assertEquals(89.0, h.getRank(), 1E-6); - assertEquals("doc1", h.getDocId()); - byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); - assertEquals(9, b.length); - byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' }; - for (int i = 0; i < b.length; i++) { - assertEquals(e[i], b[i]); - } - h = msg.getResult().getHit(1); - assertTrue(h instanceof SearchResult.HitWithSortBlob); - assertEquals(109.0, h.getRank(), 1E-6); - assertEquals("doc17", h.getDocId()); - b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); - assertEquals(9, b.length); - byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' }; - for (int i = 0; i < b.length; i++) { - assertEquals(d[i], b[i]); - } - h = msg.getResult().getHit(2); - assertTrue(h instanceof SearchResult.HitWithSortBlob); - assertEquals(90.0, h.getRank(), 1E-6); - assertEquals("doc18", h.getDocId()); - b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); - assertEquals(9, b.length); - byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' }; - for (int i = 0; i < b.length; i++) { - assertEquals(c[i], b[i]); - } - } - } - - public class testPutDocumentMessage implements RunnableTest { - - @Override - public void run() { - PutDocumentMessage msg = new PutDocumentMessage(new DocumentPut(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:"))); - msg.setTimestamp(666); - assertEquals(BASE_MESSAGE_LENGTH + 41, serialize("PutDocumentMessage", msg)); - - for (Language lang : LANGUAGES) { - msg = (PutDocumentMessage)deserialize("PutDocumentMessage", DocumentProtocol.MESSAGE_PUTDOCUMENT, lang); - assertEquals("testdoc", msg.getDocumentPut().getDocument().getDataType().getName()); - assertEquals("doc:scheme:", msg.getDocumentPut().getDocument().getId().toString()); - assertEquals(666, msg.getTimestamp()); - } - } - } - - public class testPutDocumentReply implements RunnableTest { - - @Override - public void run() { - WriteDocumentReply reply = new WriteDocumentReply(DocumentProtocol.REPLY_PUTDOCUMENT); - reply.setHighestModificationTimestamp(30); - - assertEquals(13, serialize("PutDocumentReply", reply)); - - for (Language lang : LANGUAGES) { - WriteDocumentReply obj = (WriteDocumentReply)deserialize("PutDocumentReply", DocumentProtocol.REPLY_PUTDOCUMENT, lang); - assertNotNull(obj); - assertEquals(30, obj.getHighestModificationTimestamp()); - } - } - } - - public class testUpdateDocumentMessage implements RunnableTest { - - @Override - public void run() { - DocumentType docType = protocol.getDocumentTypeManager().getDocumentType("testdoc"); - DocumentUpdate update = new DocumentUpdate(docType, new DocumentId("doc:scheme:")); - update.addFieldPathUpdate(new RemoveFieldPathUpdate(docType, "intfield", "testdoc.intfield > 0")); - UpdateDocumentMessage msg = new UpdateDocumentMessage(update); - msg.setNewTimestamp(777); - msg.setOldTimestamp(666); - - assertEquals(BASE_MESSAGE_LENGTH + 89, serialize("UpdateDocumentMessage", msg)); - - for (Language lang : LANGUAGES) { - msg = (UpdateDocumentMessage)deserialize("UpdateDocumentMessage", DocumentProtocol.MESSAGE_UPDATEDOCUMENT, lang); - assertEquals(update, msg.getDocumentUpdate()); - assertEquals(777, msg.getNewTimestamp()); - assertEquals(666, msg.getOldTimestamp()); - } - } - } - - public class testUpdateDocumentReply implements RunnableTest { - - @Override - public void run() { - UpdateDocumentReply reply = new UpdateDocumentReply(); - reply.setHighestModificationTimestamp(30); - reply.setWasFound(false); - - assertEquals(14, serialize("UpdateDocumentReply", reply)); - - for (Language lang : LANGUAGES) { - UpdateDocumentReply obj = (UpdateDocumentReply)deserialize("UpdateDocumentReply", DocumentProtocol.REPLY_UPDATEDOCUMENT, lang); - assertNotNull(obj); - assertEquals(30, reply.getHighestModificationTimestamp()); - assertEquals(false, obj.wasFound()); - } - } - } - - public class testVisitorInfoReply implements RunnableTest { - - @Override - public void run() { - testVisitorReply("VisitorInfoReply", DocumentProtocol.REPLY_VISITORINFO); - } - } - - public class testWrongDistributionReply implements RunnableTest { - - @Override - public void run() { - WrongDistributionReply reply = new WrongDistributionReply("distributor:3 storage:2"); - assertEquals(32, serialize("WrongDistributionReply", reply)); - - for (Language lang : LANGUAGES) { - reply = (WrongDistributionReply)deserialize("WrongDistributionReply", DocumentProtocol.REPLY_WRONGDISTRIBUTION, lang); - assertEquals("distributor:3 storage:2", reply.getSystemState()); - } - } - } - - public class testRemoveDocumentReply implements RunnableTest { - - @Override - public void run() { - RemoveDocumentReply reply = new RemoveDocumentReply(); - reply.setHighestModificationTimestamp(30); - reply.setWasFound(false); - - assertEquals(14, serialize("RemoveDocumentReply", reply)); - - for (Language lang : LANGUAGES) { - RemoveDocumentReply obj = (RemoveDocumentReply)deserialize("RemoveDocumentReply", DocumentProtocol.REPLY_REMOVEDOCUMENT, lang); - assertNotNull(obj); - assertEquals(30, obj.getHighestModificationTimestamp()); - assertEquals(false, obj.wasFound()); - } - } - } - - public class testRemoveLocationReply implements RunnableTest { - - @Override - public void run() { - testDocumentReply("RemoveLocationReply", DocumentProtocol.REPLY_REMOVELOCATION); - } - } - - public class testSearchResultReply implements RunnableTest { - - @Override - public void run() { - testVisitorReply("SearchResultReply", DocumentProtocol.REPLY_SEARCHRESULT); - } - } - - public class testStatBucketReply implements RunnableTest { - - @Override - public void run() { - StatBucketReply msg = new StatBucketReply(); - msg.setResults("These are the votes of the Norwegian jury"); - - assertEquals(50, serialize("StatBucketReply", msg)); - - for (Language lang : LANGUAGES) { - msg = (StatBucketReply)deserialize("StatBucketReply", DocumentProtocol.REPLY_STATBUCKET, lang); - assertEquals("These are the votes of the Norwegian jury", msg.getResults()); - } - } - } - - public class testQueryResultReply implements RunnableTest { - - @Override - public void run() { - testVisitorReply("QueryResultReply", DocumentProtocol.REPLY_QUERYRESULT); - } - } - - public class testQueryResultMessage implements RunnableTest { - - @Override - public void run() throws Exception { - FileInputStream stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-1.dat")); - byte[] data = new byte[stream.available()]; - assertEquals(data.length, stream.read(data)); - - Routable routable = decode(data); - assertTrue(routable instanceof QueryResultMessage); - - QueryResultMessage msg = (QueryResultMessage)routable; - assertEquals(0, msg.getResult().getHitCount()); - - stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-2.dat")); - data = new byte[stream.available()]; - assertEquals(data.length, stream.read(data)); - - routable = decode(data); - assertTrue(routable instanceof QueryResultMessage); - - msg = (QueryResultMessage)routable; - assertEquals(2, msg.getResult().getHitCount()); - com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0); - assertEquals(89.0, h.getRank(), 1E-6); - assertEquals("doc1", h.getDocId()); - h = msg.getResult().getHit(1); - assertEquals(109.0, h.getRank(), 1E-6); - assertEquals("doc17", h.getDocId()); - - stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-3.dat")); - data = new byte[stream.available()]; - assertEquals(data.length, stream.read(data)); - - routable = decode(data); - assertTrue(routable instanceof QueryResultMessage); - - msg = (QueryResultMessage)routable; - assertEquals(2, msg.getResult().getHitCount()); - h = msg.getResult().getHit(0); - assertEquals(109.0, h.getRank(), 1E-6); - assertEquals("doc17", h.getDocId()); - h = msg.getResult().getHit(1); - assertEquals(89.0, h.getRank(), 1E-6); - assertEquals("doc1", h.getDocId()); - - stream = new FileInputStream(getPath("5-cpp-QueryResultMessage-4.dat")); - data = new byte[stream.available()]; - assertEquals(data.length, stream.read(data)); - - routable = decode(data); - assertTrue(routable instanceof QueryResultMessage); - - msg = (QueryResultMessage)routable; - assertEquals(3, msg.getResult().getHitCount()); - h = msg.getResult().getHit(0); - assertTrue(h instanceof SearchResult.HitWithSortBlob); - assertEquals(89.0, h.getRank(), 1E-6); - assertEquals("doc1", h.getDocId()); - byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); - assertEquals(9, b.length); - byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' }; - for (int i = 0; i < b.length; i++) { - assertEquals(e[i], b[i]); - } - h = msg.getResult().getHit(1); - assertTrue(h instanceof SearchResult.HitWithSortBlob); - assertEquals(109.0, h.getRank(), 1E-6); - assertEquals("doc17", h.getDocId()); - b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); - assertEquals(9, b.length); - byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' }; - for (int i = 0; i < b.length; i++) { - assertEquals(d[i], b[i]); - } - h = msg.getResult().getHit(2); - assertTrue(h instanceof SearchResult.HitWithSortBlob); - assertEquals(90.0, h.getRank(), 1E-6); - assertEquals("doc18", h.getDocId()); - b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); - assertEquals(9, b.length); - byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' }; - for (int i = 0; i < b.length; i++) { - assertEquals(c[i], b[i]); - } - } - } - - public class testGetBucketListReply implements RunnableTest { - - public void run() { - GetBucketListReply reply = new GetBucketListReply(); - reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(16, 123), "foo")); - reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(17, 1123), "bar")); - reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(18, 11123), "zoink")); - - assertEquals(56, serialize("GetBucketListReply", reply)); - - for (Language lang : LANGUAGES) { - reply = (GetBucketListReply)deserialize("GetBucketListReply", DocumentProtocol.REPLY_GETBUCKETLIST, lang); - assertEquals(reply.getBuckets().get(0), new GetBucketListReply.BucketInfo(new BucketId(16, 123), "foo")); - assertEquals(reply.getBuckets().get(1), new GetBucketListReply.BucketInfo(new BucketId(17, 1123), "bar")); - assertEquals(reply.getBuckets().get(2), new GetBucketListReply.BucketInfo(new BucketId(18, 11123), "zoink")); - } - } - } - - public class testGetBucketStateReply implements RunnableTest { - - public void run() { - GlobalId foo = new GlobalId(IdString.createIdString("doc:scheme:foo")); - GlobalId bar = new GlobalId(IdString.createIdString("doc:scheme:bar")); - - GetBucketStateReply reply = new GetBucketStateReply(); - List<DocumentState> state = new ArrayList<>(2); - state.add(new DocumentState(foo, 777, false)); - state.add(new DocumentState(bar, 888, true)); - reply.setBucketState(state); - assertEquals(53, serialize("GetBucketStateReply", reply)); - - for (Language lang : LANGUAGES) { - reply = (GetBucketStateReply)deserialize("GetBucketStateReply", DocumentProtocol.REPLY_GETBUCKETSTATE, lang); - assertEquals(777, reply.getBucketState().get(0).getTimestamp()); - assertEquals(foo, reply.getBucketState().get(0).getGid()); - assertEquals(false, reply.getBucketState().get(0).isRemoveEntry()); - assertEquals(888, reply.getBucketState().get(1).getTimestamp()); - assertEquals(bar, reply.getBucketState().get(1).getGid()); - assertEquals(true, reply.getBucketState().get(1).isRemoveEntry()); - } - } - } - - public class testGetDocumentReply implements RunnableTest { - - public void run() { - GetDocumentReply reply = new GetDocumentReply(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:")); - assertEquals(43, serialize("GetDocumentReply", reply)); - - for (Language lang : LANGUAGES) { - reply = (GetDocumentReply)deserialize("GetDocumentReply", DocumentProtocol.REPLY_GETDOCUMENT, lang); - assertEquals("testdoc", reply.getDocument().getDataType().getName()); - assertEquals("doc:scheme:", reply.getDocument().getId().toString()); - } - } - } - - public class testMapVisitorReply implements RunnableTest { - - public void run() { - testVisitorReply("MapVisitorReply", DocumentProtocol.REPLY_MAPVISITOR); - } - } - - protected void testDocumentReply(String filename, int type) { - DocumentReply reply = new DocumentReply(type); - assertEquals(5, serialize(filename, reply)); - - for (Language lang : LANGUAGES) { - reply = (DocumentReply)deserialize(filename, type, lang); - assertNotNull(reply); - } - } - - protected void testVisitorReply(String filename, int type) { - VisitorReply reply = new VisitorReply(type); - assertEquals(5, serialize(filename, reply)); - - for (Language lang : LANGUAGES) { - reply = (VisitorReply)deserialize(filename, type, lang); - assertNotNull(reply); - } - } - -} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages51TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages51TestCase.java deleted file mode 100644 index 862244236f8..00000000000 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages51TestCase.java +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.documentapi.messagebus.protocol.test; - -import com.yahoo.component.Version; -import com.yahoo.document.BucketId; -import com.yahoo.document.DocumentId; -import com.yahoo.document.select.OrderingSpecification; -import com.yahoo.documentapi.messagebus.protocol.CreateVisitorMessage; -import com.yahoo.documentapi.messagebus.protocol.DocumentIgnoredReply; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; -import com.yahoo.text.Utf8; - -import java.util.Map; - -import static org.junit.Assert.assertEquals; - -/** - * @author Simon Thoresen Hult - */ -public class Messages51TestCase extends Messages50TestCase { - - //////////////////////////////////////////////////////////////////////////////// - // - // Setup - // - /////////////////////////////////////////////////////////////////////////////// - - @Override - protected void registerTests(Map<Integer, RunnableTest> out) { - super.registerTests(out); - - // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support - // version 5.0. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now. - out.put(DocumentProtocol.MESSAGE_CREATEVISITOR, new testCreateVisitorMessage()); - out.put(DocumentProtocol.MESSAGE_GETDOCUMENT, new testGetDocumentMessage()); - out.put(DocumentProtocol.REPLY_DOCUMENTIGNORED, new testDocumentIgnoredReply()); - } - - @Override - protected Version version() { - return new Version(5, 1); - } - - @Override - protected boolean shouldTestCoverage() { - return true; - } - - //////////////////////////////////////////////////////////////////////////////// - // - // Tests - // - //////////////////////////////////////////////////////////////////////////////// - - protected static int BASE_MESSAGE_LENGTH = 5; - - public class testCreateVisitorMessage implements RunnableTest { - - @Override - public void run() { - CreateVisitorMessage msg = new CreateVisitorMessage("SomeLibrary", "myvisitor", "newyork", "london"); - msg.setDocumentSelection("true and false or true"); - msg.getParameters().put("myvar", Utf8.toBytes("somevalue")); - msg.getParameters().put("anothervar", Utf8.toBytes("34")); - msg.getBuckets().add(new BucketId(16, 1234)); - msg.setVisitRemoves(true); - msg.setFieldSet("foo bar"); - msg.setVisitorOrdering(OrderingSpecification.DESCENDING); - msg.setMaxBucketsPerVisitor(2); - assertEquals(BASE_MESSAGE_LENGTH + 178, serialize("CreateVisitorMessage", msg)); - - for (Language lang : LANGUAGES) { - msg = (CreateVisitorMessage)deserialize("CreateVisitorMessage", DocumentProtocol.MESSAGE_CREATEVISITOR, lang); - assertEquals("SomeLibrary", msg.getLibraryName()); - assertEquals("myvisitor", msg.getInstanceId()); - assertEquals("newyork", msg.getControlDestination()); - assertEquals("london", msg.getDataDestination()); - assertEquals("true and false or true", msg.getDocumentSelection()); - assertEquals(8, msg.getMaxPendingReplyCount()); - assertEquals(true, msg.getVisitRemoves()); - assertEquals("foo bar", msg.getFieldSet()); - assertEquals(false, msg.getVisitInconsistentBuckets()); - assertEquals(1, msg.getBuckets().size()); - assertEquals(new BucketId(16, 1234), msg.getBuckets().iterator().next()); - assertEquals("somevalue", Utf8.toString(msg.getParameters().get("myvar"))); - assertEquals("34", Utf8.toString(msg.getParameters().get("anothervar"))); - assertEquals(OrderingSpecification.DESCENDING, msg.getVisitorOrdering()); - assertEquals(2, msg.getMaxBucketsPerVisitor()); - } - } - } - - public class testGetDocumentMessage implements RunnableTest { - - @Override - public void run() { - GetDocumentMessage msg = new GetDocumentMessage(new DocumentId("doc:scheme:"), "foo bar"); - assertEquals(BASE_MESSAGE_LENGTH + 27, serialize("GetDocumentMessage", msg)); - - for (Language lang : LANGUAGES) { - msg = (GetDocumentMessage)deserialize("GetDocumentMessage", DocumentProtocol.MESSAGE_GETDOCUMENT, lang); - assertEquals("doc:scheme:", msg.getDocumentId().toString()); - assertEquals("foo bar", msg.getFieldSet()); - } - } - } - - public class testDocumentIgnoredReply implements RunnableTest { - - @Override - public void run() { - DocumentIgnoredReply reply = new DocumentIgnoredReply(); - assertEquals(BASE_MESSAGE_LENGTH, serialize("DocumentIgnoredReply", reply)); - - for (Language lang : LANGUAGES) { - reply = (DocumentIgnoredReply)deserialize("DocumentIgnoredReply", DocumentProtocol.REPLY_DOCUMENTIGNORED, lang); - } - } - } -} diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages52TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages52TestCase.java index 1bda3ea3610..448c8fcb520 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages52TestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages52TestCase.java @@ -2,19 +2,108 @@ package com.yahoo.documentapi.messagebus.protocol.test; import com.yahoo.component.Version; -import com.yahoo.document.*; +import com.yahoo.document.BucketId; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentType; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.GlobalId; +import com.yahoo.document.TestAndSetCondition; import com.yahoo.document.fieldpathupdate.RemoveFieldPathUpdate; -import com.yahoo.documentapi.messagebus.protocol.*; +import com.yahoo.document.idstring.IdString; +import com.yahoo.document.select.OrderingSpecification; +import com.yahoo.documentapi.messagebus.protocol.CreateVisitorMessage; +import com.yahoo.documentapi.messagebus.protocol.CreateVisitorReply; +import com.yahoo.documentapi.messagebus.protocol.DestroyVisitorMessage; +import com.yahoo.documentapi.messagebus.protocol.DocumentIgnoredReply; +import com.yahoo.documentapi.messagebus.protocol.DocumentListMessage; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.documentapi.messagebus.protocol.DocumentReply; +import com.yahoo.documentapi.messagebus.protocol.DocumentState; +import com.yahoo.documentapi.messagebus.protocol.DocumentSummaryMessage; +import com.yahoo.documentapi.messagebus.protocol.EmptyBucketsMessage; +import com.yahoo.documentapi.messagebus.protocol.GetBucketListMessage; +import com.yahoo.documentapi.messagebus.protocol.GetBucketListReply; +import com.yahoo.documentapi.messagebus.protocol.GetBucketStateMessage; +import com.yahoo.documentapi.messagebus.protocol.GetBucketStateReply; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; +import com.yahoo.documentapi.messagebus.protocol.MapVisitorMessage; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.QueryResultMessage; +import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentReply; +import com.yahoo.documentapi.messagebus.protocol.RemoveLocationMessage; +import com.yahoo.documentapi.messagebus.protocol.SearchResultMessage; +import com.yahoo.documentapi.messagebus.protocol.StatBucketMessage; +import com.yahoo.documentapi.messagebus.protocol.StatBucketReply; +import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentReply; +import com.yahoo.documentapi.messagebus.protocol.VisitorInfoMessage; +import com.yahoo.documentapi.messagebus.protocol.VisitorReply; +import com.yahoo.documentapi.messagebus.protocol.WriteDocumentReply; +import com.yahoo.documentapi.messagebus.protocol.WrongDistributionReply; +import com.yahoo.messagebus.Routable; +import com.yahoo.text.Utf8; +import com.yahoo.vdslib.SearchResult; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; /** + * @author Simon Thoresen Hult * @author Vegard Sjonfjell */ -public class Messages52TestCase extends Messages51TestCase { +public class Messages52TestCase extends MessagesTestBase { + @Override + protected void registerTests(Map<Integer, RunnableTest> out) { + // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support + // version 5.2. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now. + out.put(DocumentProtocol.MESSAGE_CREATEVISITOR, new testCreateVisitorMessage()); + out.put(DocumentProtocol.MESSAGE_DESTROYVISITOR, new testDestroyVisitorMessage()); + out.put(DocumentProtocol.MESSAGE_DOCUMENTLIST, new testDocumentListMessage()); + out.put(DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, new testDocumentSummaryMessage()); + out.put(DocumentProtocol.MESSAGE_EMPTYBUCKETS, new testEmptyBucketsMessage()); + out.put(DocumentProtocol.MESSAGE_GETBUCKETLIST, new testGetBucketListMessage()); + out.put(DocumentProtocol.MESSAGE_GETBUCKETSTATE, new testGetBucketStateMessage()); + out.put(DocumentProtocol.MESSAGE_GETDOCUMENT, new testGetDocumentMessage()); + out.put(DocumentProtocol.MESSAGE_MAPVISITOR, new testMapVisitorMessage()); + out.put(DocumentProtocol.MESSAGE_PUTDOCUMENT, new testPutDocumentMessage()); + out.put(DocumentProtocol.MESSAGE_QUERYRESULT, new testQueryResultMessage()); + out.put(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, new testRemoveDocumentMessage()); + out.put(DocumentProtocol.MESSAGE_REMOVELOCATION, new testRemoveLocationMessage()); + out.put(DocumentProtocol.MESSAGE_SEARCHRESULT, new testSearchResultMessage()); + out.put(DocumentProtocol.MESSAGE_STATBUCKET, new testStatBucketMessage()); + out.put(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, new testUpdateDocumentMessage()); + out.put(DocumentProtocol.MESSAGE_VISITORINFO, new testVisitorInfoMessage()); + out.put(DocumentProtocol.REPLY_CREATEVISITOR, new testCreateVisitorReply()); + out.put(DocumentProtocol.REPLY_DESTROYVISITOR, new testDestroyVisitorReply()); + out.put(DocumentProtocol.REPLY_DOCUMENTIGNORED, new testDocumentIgnoredReply()); + out.put(DocumentProtocol.REPLY_DOCUMENTLIST, new testDocumentListReply()); + out.put(DocumentProtocol.REPLY_DOCUMENTSUMMARY, new testDocumentSummaryReply()); + out.put(DocumentProtocol.REPLY_EMPTYBUCKETS, new testEmptyBucketsReply()); + out.put(DocumentProtocol.REPLY_GETBUCKETLIST, new testGetBucketListReply()); + out.put(DocumentProtocol.REPLY_GETBUCKETSTATE, new testGetBucketStateReply()); + out.put(DocumentProtocol.REPLY_GETDOCUMENT, new testGetDocumentReply()); + out.put(DocumentProtocol.REPLY_MAPVISITOR, new testMapVisitorReply()); + out.put(DocumentProtocol.REPLY_PUTDOCUMENT, new testPutDocumentReply()); + out.put(DocumentProtocol.REPLY_QUERYRESULT, new testQueryResultReply()); + out.put(DocumentProtocol.REPLY_REMOVEDOCUMENT, new testRemoveDocumentReply()); + out.put(DocumentProtocol.REPLY_REMOVELOCATION, new testRemoveLocationReply()); + out.put(DocumentProtocol.REPLY_SEARCHRESULT, new testSearchResultReply()); + out.put(DocumentProtocol.REPLY_STATBUCKET, new testStatBucketReply()); + out.put(DocumentProtocol.REPLY_UPDATEDOCUMENT, new testUpdateDocumentReply()); + out.put(DocumentProtocol.REPLY_VISITORINFO, new testVisitorInfoReply()); + out.put(DocumentProtocol.REPLY_WRONGDISTRIBUTION, new testWrongDistributionReply()); + } @Override protected Version version() { @@ -26,21 +115,488 @@ public class Messages52TestCase extends Messages51TestCase { return true; } - @Override - protected void registerTests(Map<Integer, RunnableTest> out) { - super.registerTests(out); + //////////////////////////////////////////////////////////////////////////////// + // + // Tests + // + //////////////////////////////////////////////////////////////////////////////// - // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support - // version 5.2. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now. + protected static int BASE_MESSAGE_LENGTH = 5; - out.put(DocumentProtocol.MESSAGE_PUTDOCUMENT, new testPutDocumentMessage()); - out.put(DocumentProtocol.MESSAGE_UPDATEDOCUMENT, new testUpdateDocumentMessage()); - out.put(DocumentProtocol.MESSAGE_REMOVEDOCUMENT, new testRemoveDocumentMessage()); + public class testRemoveLocationMessage implements RunnableTest { + + @Override + public void run() { + { + RemoveLocationMessage msg = new RemoveLocationMessage("id.group == \"mygroup\""); + assertEquals(BASE_MESSAGE_LENGTH + 29, serialize("RemoveLocationMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (RemoveLocationMessage)deserialize("RemoveLocationMessage", DocumentProtocol.MESSAGE_REMOVELOCATION, lang); + assertEquals("id.group == \"mygroup\"", msg.getDocumentSelection()); + } + } + } + } + + public class testGetBucketListMessage implements RunnableTest { + + @Override + public void run() { + GetBucketListMessage msg = new GetBucketListMessage(new BucketId(16, 123)); + msg.setLoadType(loadTypes.getNameMap().get("foo")); + assertEquals(BASE_MESSAGE_LENGTH + 12, serialize("GetBucketListMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (GetBucketListMessage)deserialize("GetBucketListMessage", DocumentProtocol.MESSAGE_GETBUCKETLIST, lang); + assertEquals(new BucketId(16, 123), msg.getBucketId()); + assertEquals("foo", msg.getLoadType().getName()); + } + } + } + + + public class testStatBucketMessage implements RunnableTest { + + @Override + public void run() { + StatBucketMessage msg = new StatBucketMessage(new BucketId(16, 123), "id.user=123"); + msg.setLoadType(null); + assertEquals(BASE_MESSAGE_LENGTH + 27, serialize("StatBucketMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (StatBucketMessage)deserialize("StatBucketMessage", DocumentProtocol.MESSAGE_STATBUCKET, lang); + assertEquals(new BucketId(16, 123), msg.getBucketId()); + assertEquals("id.user=123", msg.getDocumentSelection()); + assertEquals("default", msg.getLoadType().getName()); + } + } + } + + public class testGetBucketStateMessage implements RunnableTest { + + @Override + public void run() { + GetBucketStateMessage msg = new GetBucketStateMessage(new BucketId(16, 666)); + assertEquals(BASE_MESSAGE_LENGTH + 12, serialize("GetBucketStateMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (GetBucketStateMessage)deserialize("GetBucketStateMessage", DocumentProtocol.MESSAGE_GETBUCKETSTATE, lang); + assertEquals(16, msg.getBucketId().getUsedBits()); + assertEquals(4611686018427388570l, msg.getBucketId().getId()); + } + } + } + + public class testCreateVisitorMessage implements RunnableTest { + + @Override + public void run() { + CreateVisitorMessage msg = new CreateVisitorMessage("SomeLibrary", "myvisitor", "newyork", "london"); + msg.setDocumentSelection("true and false or true"); + msg.getParameters().put("myvar", Utf8.toBytes("somevalue")); + msg.getParameters().put("anothervar", Utf8.toBytes("34")); + msg.getBuckets().add(new BucketId(16, 1234)); + msg.setVisitRemoves(true); + msg.setFieldSet("foo bar"); + msg.setVisitorOrdering(OrderingSpecification.DESCENDING); + msg.setMaxBucketsPerVisitor(2); + assertEquals(BASE_MESSAGE_LENGTH + 178, serialize("CreateVisitorMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (CreateVisitorMessage)deserialize("CreateVisitorMessage", DocumentProtocol.MESSAGE_CREATEVISITOR, lang); + assertEquals("SomeLibrary", msg.getLibraryName()); + assertEquals("myvisitor", msg.getInstanceId()); + assertEquals("newyork", msg.getControlDestination()); + assertEquals("london", msg.getDataDestination()); + assertEquals("true and false or true", msg.getDocumentSelection()); + assertEquals(8, msg.getMaxPendingReplyCount()); + assertEquals(true, msg.getVisitRemoves()); + assertEquals("foo bar", msg.getFieldSet()); + assertEquals(false, msg.getVisitInconsistentBuckets()); + assertEquals(1, msg.getBuckets().size()); + assertEquals(new BucketId(16, 1234), msg.getBuckets().iterator().next()); + assertEquals("somevalue", Utf8.toString(msg.getParameters().get("myvar"))); + assertEquals("34", Utf8.toString(msg.getParameters().get("anothervar"))); + assertEquals(OrderingSpecification.DESCENDING, msg.getVisitorOrdering()); + assertEquals(2, msg.getMaxBucketsPerVisitor()); + } + + msg.getBuckets().clear(); + + assertEquals("CreateVisitorMessage(" + + "No buckets, " + + "selection 'true and false or true', " + + "bucket space 'default', " + + "library SomeLibrary, including removes, " + + "get fields: foo bar" + + ")", + msg.toString()); + + msg.getBuckets().add(new BucketId(16, 1234)); + + assertEquals("CreateVisitorMessage(" + + "Bucket BucketId(0x40000000000004d2), " + + "selection 'true and false or true', " + + "bucket space 'default', " + + "library SomeLibrary, including removes, " + + "get fields: foo bar" + + ")", + msg.toString()); + + msg.getBuckets().add(new BucketId(16, 1235)); + msg.getBuckets().add(new BucketId(16, 1236)); + msg.getBuckets().add(new BucketId(16, 1237)); + msg.getBuckets().add(new BucketId(16, 1238)); + msg.setFromTimestamp(10001); + msg.setToTimestamp(20002); + msg.setVisitInconsistentBuckets(true); + assertEquals("CreateVisitorMessage(" + + "5 buckets: BucketId(0x40000000000004d2) BucketId(0x40000000000004d3) BucketId(0x40000000000004d4) ..., " + + "time 10001-20002, " + + "selection 'true and false or true', " + + "bucket space 'default', " + + "library SomeLibrary, including removes, " + + "get fields: foo bar, " + + "visit inconsistent buckets" + + ")", + msg.toString()); + } + } + + public class testCreateVisitorReply implements RunnableTest { + + @Override + public void run() { + CreateVisitorReply reply = new CreateVisitorReply(DocumentProtocol.REPLY_CREATEVISITOR); + reply.setLastBucket(new BucketId(16, 123)); + reply.getVisitorStatistics().setBucketsVisited(3); + reply.getVisitorStatistics().setDocumentsVisited(1000); + reply.getVisitorStatistics().setBytesVisited(1024000); + reply.getVisitorStatistics().setDocumentsReturned(123); + reply.getVisitorStatistics().setBytesReturned(512000); + reply.getVisitorStatistics().setSecondPassDocumentsReturned(456); + reply.getVisitorStatistics().setSecondPassBytesReturned(789100); + + assertEquals(65, serialize("CreateVisitorReply", reply)); + + for (Language lang : LANGUAGES) { + reply = (CreateVisitorReply)deserialize("CreateVisitorReply", DocumentProtocol.REPLY_CREATEVISITOR, lang); + assertNotNull(reply); + assertEquals(new BucketId(16, 123), reply.getLastBucket()); + assertEquals(3, reply.getVisitorStatistics().getBucketsVisited()); + assertEquals(1000, reply.getVisitorStatistics().getDocumentsVisited()); + assertEquals(1024000, reply.getVisitorStatistics().getBytesVisited()); + assertEquals(123, reply.getVisitorStatistics().getDocumentsReturned()); + assertEquals(512000, reply.getVisitorStatistics().getBytesReturned()); + assertEquals(456, reply.getVisitorStatistics().getSecondPassDocumentsReturned()); + assertEquals(789100, reply.getVisitorStatistics().getSecondPassBytesReturned()); + } + } + } + + public class testDestroyVisitorReply implements RunnableTest { + + @Override + public void run() { + testVisitorReply("DestroyVisitorReply", DocumentProtocol.REPLY_DESTROYVISITOR); + } + } + + public class testDocumentIgnoredReply implements RunnableTest { + + @Override + public void run() { + DocumentIgnoredReply reply = new DocumentIgnoredReply(); + assertEquals(BASE_MESSAGE_LENGTH, serialize("DocumentIgnoredReply", reply)); + + for (Language lang : LANGUAGES) { + reply = (DocumentIgnoredReply)deserialize("DocumentIgnoredReply", DocumentProtocol.REPLY_DOCUMENTIGNORED, lang); + } + } + } + + public class testDocumentListReply implements RunnableTest { + + @Override + public void run() { + testVisitorReply("DocumentListReply", DocumentProtocol.REPLY_DOCUMENTLIST); + } + } + + public class testDocumentSummaryReply implements RunnableTest { + + @Override + public void run() { + testVisitorReply("DocumentSummaryReply", DocumentProtocol.REPLY_DOCUMENTSUMMARY); + } + } + + public class testEmptyBucketsReply implements RunnableTest { + + @Override + public void run() { + testVisitorReply("EmptyBucketsReply", DocumentProtocol.REPLY_EMPTYBUCKETS); + } + } + + public class testDestroyVisitorMessage implements RunnableTest { + + @Override + public void run() { + DestroyVisitorMessage msg = new DestroyVisitorMessage("myvisitor"); + assertEquals(BASE_MESSAGE_LENGTH + 17, serialize("DestroyVisitorMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (DestroyVisitorMessage)deserialize("DestroyVisitorMessage", DocumentProtocol.MESSAGE_DESTROYVISITOR, lang); + assertEquals("myvisitor", msg.getInstanceId()); + } + } + } + + public class testDocumentListMessage implements RunnableTest { + + @Override + public void run() { + DocumentListMessage msg = (DocumentListMessage)deserialize("DocumentListMessage", DocumentProtocol.MESSAGE_DOCUMENTLIST, Language.CPP); + assertEquals("userdoc:scheme:1234:", msg.getDocuments().get(0).getDocument().getId().toString()); + assertEquals(1234, msg.getDocuments().get(0).getTimestamp()); + assertFalse(msg.getDocuments().get(0).isRemoveEntry()); + + assertEquals(BASE_MESSAGE_LENGTH + 63, serialize("DocumentListMessage", msg)); + msg = (DocumentListMessage)deserialize("DocumentListMessage", DocumentProtocol.MESSAGE_DOCUMENTLIST, Language.JAVA); + assertEquals("userdoc:scheme:1234:", msg.getDocuments().get(0).getDocument().getId().toString()); + assertEquals(1234, msg.getDocuments().get(0).getTimestamp()); + assertFalse(msg.getDocuments().get(0).isRemoveEntry()); + + } + } + + public class testEmptyBucketsMessage implements RunnableTest { + + @Override + public void run() { + List<BucketId> bids = new ArrayList<>(); + for (int i = 0; i < 13; ++i) { + bids.add(new BucketId(16, i)); + } + + EmptyBucketsMessage ebm = new EmptyBucketsMessage(bids); + assertEquals(BASE_MESSAGE_LENGTH + 112, serialize("EmptyBucketsMessage", ebm)); + for (Language lang : LANGUAGES) { + ebm = (EmptyBucketsMessage)deserialize("EmptyBucketsMessage", DocumentProtocol.MESSAGE_EMPTYBUCKETS, lang); + for (int i = 0; i < 13; ++i) { + assertEquals(new BucketId(16, i), ebm.getBucketIds().get(i)); + } + } + } + } + + public class testDocumentSummaryMessage implements RunnableTest { + + @Override + public void run() { + Routable routable = deserialize("DocumentSummaryMessage-1", DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, Language.CPP); + assertTrue(routable instanceof DocumentSummaryMessage); + + DocumentSummaryMessage msg = (DocumentSummaryMessage) routable; + assertEquals(0, msg.getResult().getSummaryCount()); + + routable = deserialize("DocumentSummaryMessage-2", DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, Language.CPP); + assertTrue(routable instanceof DocumentSummaryMessage); + + msg = (DocumentSummaryMessage) routable; + assertEquals(2, msg.getResult().getSummaryCount()); + com.yahoo.vdslib.DocumentSummary.Summary s = msg.getResult().getSummary(0); + assertEquals("doc1", s.getDocId()); + byte[] b = s.getSummary(); + assertEquals(8, b.length); + byte[] c = {'s', 'u', 'm', 'm', 'a', 'r', 'y', '1'}; + for (int i = 0; i < b.length; i++) { + assertEquals(c[i], b[i]); + } + + s = msg.getResult().getSummary(1); + assertEquals("aoc17", s.getDocId()); + b = s.getSummary(); + assertEquals(9, b.length); + byte[] d = {'s', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5'}; + for (int i = 0; i < b.length; i++) { + assertEquals(d[i], b[i]); + } + routable = deserialize("DocumentSummaryMessage-3", DocumentProtocol.MESSAGE_DOCUMENTSUMMARY, Language.CPP); + assertTrue(routable instanceof DocumentSummaryMessage); + + msg = (DocumentSummaryMessage) routable; + assertEquals(2, msg.getResult().getSummaryCount()); + + s = msg.getResult().getSummary(0); + assertEquals("aoc17", s.getDocId()); + b = s.getSummary(); + assertEquals(9, b.length); + byte[] e = {'s', 'u', 'm', 'm', 'a', 'r', 'y', '4', '5'}; + for (int i = 0; i < b.length; i++) { + assertEquals(e[i], b[i]); + } + + s = msg.getResult().getSummary(1); + assertEquals("doc1", s.getDocId()); + b = s.getSummary(); + assertEquals(8, b.length); + byte[] f = {'s', 'u', 'm', 'm', 'a', 'r', 'y', '1'}; + for (int i = 0; i < b.length; i++) { + assertEquals(f[i], b[i]); + } + } + } + + + public class testGetDocumentMessage implements RunnableTest { + + @Override + public void run() { + GetDocumentMessage msg = new GetDocumentMessage(new DocumentId("doc:scheme:"), "foo bar"); + assertEquals(BASE_MESSAGE_LENGTH + 27, serialize("GetDocumentMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (GetDocumentMessage)deserialize("GetDocumentMessage", DocumentProtocol.MESSAGE_GETDOCUMENT, lang); + assertEquals("doc:scheme:", msg.getDocumentId().toString()); + assertEquals("foo bar", msg.getFieldSet()); + } + } + } + + + public class testRemoveDocumentMessage implements RunnableTest { + + @Override + public void run() { + final RemoveDocumentMessage msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:")); + msg.setCondition(new TestAndSetCondition(CONDITION_STRING)); + + assertEquals(BASE_MESSAGE_LENGTH + 16 + serializedLength(msg.getCondition().getSelection()), serialize("RemoveDocumentMessage", msg)); + + for (Language lang : LANGUAGES) { + final RemoveDocumentMessage deserializedMsg = (RemoveDocumentMessage)deserialize("RemoveDocumentMessage", DocumentProtocol.MESSAGE_REMOVEDOCUMENT, lang); + assertEquals(deserializedMsg.getDocumentId().toString(), msg.getDocumentId().toString()); + } + } + } + + public class testMapVisitorMessage implements RunnableTest { + + @Override + public void run() { + MapVisitorMessage msg = (MapVisitorMessage)deserialize("MapVisitorMessage", DocumentProtocol.MESSAGE_MAPVISITOR, Language.CPP); + assertEquals("3", msg.getData().get("foo")); + assertEquals("5", msg.getData().get("bar")); + + assertEquals(BASE_MESSAGE_LENGTH + 32, serialize("MapVisitorMessage", msg)); + + msg = (MapVisitorMessage)deserialize("MapVisitorMessage", DocumentProtocol.MESSAGE_MAPVISITOR, Language.JAVA); + assertEquals("3", msg.getData().get("foo")); + assertEquals("5", msg.getData().get("bar")); + } + } + + + public class testVisitorInfoMessage implements RunnableTest { + + @Override + public void run() { + VisitorInfoMessage msg = new VisitorInfoMessage(); + msg.getFinishedBuckets().add(new BucketId(16, 1)); + msg.getFinishedBuckets().add(new BucketId(16, 2)); + msg.getFinishedBuckets().add(new BucketId(16, 4)); + msg.setErrorMessage("error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6"); + assertEquals(BASE_MESSAGE_LENGTH + 67, serialize("VisitorInfoMessage", msg)); + + for (Language lang : LANGUAGES) { + msg = (VisitorInfoMessage)deserialize("VisitorInfoMessage", DocumentProtocol.MESSAGE_VISITORINFO, lang); + assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 1))); + assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 2))); + assertTrue(msg.getFinishedBuckets().contains(new BucketId(16, 4))); + assertEquals("error message: \u00e6\u00c6\u00f8\u00d8\u00e5\u00c5\u00f6\u00d6", msg.getErrorMessage()); + } + } + } + + public class testSearchResultMessage implements RunnableTest { + + @Override + public void run() throws Exception { + Routable routable = deserialize("SearchResultMessage-1", DocumentProtocol.MESSAGE_SEARCHRESULT, Language.CPP); + assertTrue(routable instanceof SearchResultMessage); + + SearchResultMessage msg = (SearchResultMessage)routable; + assertEquals(0, msg.getResult().getHitCount()); + + routable = deserialize("SearchResultMessage-2", DocumentProtocol.MESSAGE_SEARCHRESULT, Language.CPP); + assertTrue(routable instanceof SearchResultMessage); + + msg = (SearchResultMessage)routable; + assertEquals(2, msg.getResult().getHitCount()); + com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0); + assertEquals(89.0, h.getRank(), 1E-6); + assertEquals("doc1", h.getDocId()); + h = msg.getResult().getHit(1); + assertEquals(109.0, h.getRank(), 1E-6); + assertEquals("doc17", h.getDocId()); + + routable = deserialize("SearchResultMessage-3", DocumentProtocol.MESSAGE_SEARCHRESULT, Language.CPP); + assertTrue(routable instanceof SearchResultMessage); + + msg = (SearchResultMessage)routable; + assertEquals(2, msg.getResult().getHitCount()); + h = msg.getResult().getHit(0); + assertEquals(109.0, h.getRank(), 1E-6); + assertEquals("doc17", h.getDocId()); + h = msg.getResult().getHit(1); + assertEquals(89.0, h.getRank(), 1E-6); + assertEquals("doc1", h.getDocId()); + + routable = deserialize("SearchResultMessage-4", DocumentProtocol.MESSAGE_SEARCHRESULT, Language.CPP); + assertTrue(routable instanceof SearchResultMessage); + + msg = (SearchResultMessage)routable; + assertEquals(3, msg.getResult().getHitCount()); + h = msg.getResult().getHit(0); + assertTrue(h instanceof SearchResult.HitWithSortBlob); + assertEquals(89.0, h.getRank(), 1E-6); + assertEquals("doc1", h.getDocId()); + byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); + assertEquals(9, b.length); + byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' }; + for (int i = 0; i < b.length; i++) { + assertEquals(e[i], b[i]); + } + h = msg.getResult().getHit(1); + assertTrue(h instanceof SearchResult.HitWithSortBlob); + assertEquals(109.0, h.getRank(), 1E-6); + assertEquals("doc17", h.getDocId()); + b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); + assertEquals(9, b.length); + byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' }; + for (int i = 0; i < b.length; i++) { + assertEquals(d[i], b[i]); + } + h = msg.getResult().getHit(2); + assertTrue(h instanceof SearchResult.HitWithSortBlob); + assertEquals(90.0, h.getRank(), 1E-6); + assertEquals("doc18", h.getDocId()); + b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); + assertEquals(9, b.length); + byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' }; + for (int i = 0; i < b.length; i++) { + assertEquals(c[i], b[i]); + } + } } private static String CONDITION_STRING = "There's just one condition"; public class testPutDocumentMessage implements RunnableTest { + @Override public void run() { PutDocumentMessage msg = new PutDocumentMessage(new DocumentPut(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:"))); @@ -60,7 +616,25 @@ public class Messages52TestCase extends Messages51TestCase { } } + public class testPutDocumentReply implements RunnableTest { + + @Override + public void run() { + WriteDocumentReply reply = new WriteDocumentReply(DocumentProtocol.REPLY_PUTDOCUMENT); + reply.setHighestModificationTimestamp(30); + + assertEquals(13, serialize("PutDocumentReply", reply)); + + for (Language lang : LANGUAGES) { + WriteDocumentReply obj = (WriteDocumentReply)deserialize("PutDocumentReply", DocumentProtocol.REPLY_PUTDOCUMENT, lang); + assertNotNull(obj); + assertEquals(30, obj.getHighestModificationTimestamp()); + } + } + } + public class testUpdateDocumentMessage implements RunnableTest { + @Override public void run() { DocumentType docType = protocol.getDocumentTypeManager().getDocumentType("testdoc"); @@ -84,21 +658,263 @@ public class Messages52TestCase extends Messages51TestCase { } } - public class testRemoveDocumentMessage implements RunnableTest { + public class testUpdateDocumentReply implements RunnableTest { + @Override public void run() { - final RemoveDocumentMessage msg = new RemoveDocumentMessage(new DocumentId("doc:scheme:")); - msg.setCondition(new TestAndSetCondition(CONDITION_STRING)); + UpdateDocumentReply reply = new UpdateDocumentReply(); + reply.setHighestModificationTimestamp(30); + reply.setWasFound(false); - assertEquals(BASE_MESSAGE_LENGTH + 16 + serializedLength(msg.getCondition().getSelection()), serialize("RemoveDocumentMessage", msg)); + assertEquals(14, serialize("UpdateDocumentReply", reply)); for (Language lang : LANGUAGES) { - final RemoveDocumentMessage deserializedMsg = (RemoveDocumentMessage)deserialize("RemoveDocumentMessage", DocumentProtocol.MESSAGE_REMOVEDOCUMENT, lang); - assertEquals(deserializedMsg.getDocumentId().toString(), msg.getDocumentId().toString()); + UpdateDocumentReply obj = (UpdateDocumentReply)deserialize("UpdateDocumentReply", DocumentProtocol.REPLY_UPDATEDOCUMENT, lang); + assertNotNull(obj); + assertEquals(30, reply.getHighestModificationTimestamp()); + assertEquals(false, obj.wasFound()); + } + } + } + + public class testVisitorInfoReply implements RunnableTest { + + @Override + public void run() { + testVisitorReply("VisitorInfoReply", DocumentProtocol.REPLY_VISITORINFO); + } + } + + public class testWrongDistributionReply implements RunnableTest { + + @Override + public void run() { + WrongDistributionReply reply = new WrongDistributionReply("distributor:3 storage:2"); + assertEquals(32, serialize("WrongDistributionReply", reply)); + + for (Language lang : LANGUAGES) { + reply = (WrongDistributionReply)deserialize("WrongDistributionReply", DocumentProtocol.REPLY_WRONGDISTRIBUTION, lang); + assertEquals("distributor:3 storage:2", reply.getSystemState()); + } + } + } + + public class testRemoveDocumentReply implements RunnableTest { + + @Override + public void run() { + RemoveDocumentReply reply = new RemoveDocumentReply(); + reply.setHighestModificationTimestamp(30); + reply.setWasFound(false); + + assertEquals(14, serialize("RemoveDocumentReply", reply)); + + for (Language lang : LANGUAGES) { + RemoveDocumentReply obj = (RemoveDocumentReply)deserialize("RemoveDocumentReply", DocumentProtocol.REPLY_REMOVEDOCUMENT, lang); + assertNotNull(obj); + assertEquals(30, obj.getHighestModificationTimestamp()); + assertEquals(false, obj.wasFound()); + } + } + } + + public class testRemoveLocationReply implements RunnableTest { + + @Override + public void run() { + testDocumentReply("RemoveLocationReply", DocumentProtocol.REPLY_REMOVELOCATION); + } + } + + public class testSearchResultReply implements RunnableTest { + + @Override + public void run() { + testVisitorReply("SearchResultReply", DocumentProtocol.REPLY_SEARCHRESULT); + } + } + + public class testStatBucketReply implements RunnableTest { + + @Override + public void run() { + StatBucketReply msg = new StatBucketReply(); + msg.setResults("These are the votes of the Norwegian jury"); + + assertEquals(50, serialize("StatBucketReply", msg)); + + for (Language lang : LANGUAGES) { + msg = (StatBucketReply)deserialize("StatBucketReply", DocumentProtocol.REPLY_STATBUCKET, lang); + assertEquals("These are the votes of the Norwegian jury", msg.getResults()); + } + } + } + + public class testQueryResultReply implements RunnableTest { + + @Override + public void run() { + testVisitorReply("QueryResultReply", DocumentProtocol.REPLY_QUERYRESULT); + } + } + + public class testQueryResultMessage implements RunnableTest { + + @Override + public void run() throws Exception { + Routable routable = deserialize("QueryResultMessage-1", DocumentProtocol.MESSAGE_QUERYRESULT, Language.CPP); + assertTrue(routable instanceof QueryResultMessage); + + QueryResultMessage msg = (QueryResultMessage)routable; + assertEquals(0, msg.getResult().getHitCount()); + + routable = deserialize("QueryResultMessage-2", DocumentProtocol.MESSAGE_QUERYRESULT, Language.CPP); + assertTrue(routable instanceof QueryResultMessage); + + msg = (QueryResultMessage)routable; + assertEquals(2, msg.getResult().getHitCount()); + com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0); + assertEquals(89.0, h.getRank(), 1E-6); + assertEquals("doc1", h.getDocId()); + h = msg.getResult().getHit(1); + assertEquals(109.0, h.getRank(), 1E-6); + assertEquals("doc17", h.getDocId()); + + routable = deserialize("QueryResultMessage-3", DocumentProtocol.MESSAGE_QUERYRESULT, Language.CPP); + assertTrue(routable instanceof QueryResultMessage); + + msg = (QueryResultMessage)routable; + assertEquals(2, msg.getResult().getHitCount()); + h = msg.getResult().getHit(0); + assertEquals(109.0, h.getRank(), 1E-6); + assertEquals("doc17", h.getDocId()); + h = msg.getResult().getHit(1); + assertEquals(89.0, h.getRank(), 1E-6); + assertEquals("doc1", h.getDocId()); + + routable = deserialize("QueryResultMessage-4", DocumentProtocol.MESSAGE_QUERYRESULT, Language.CPP); + assertTrue(routable instanceof QueryResultMessage); + + msg = (QueryResultMessage)routable; + assertEquals(3, msg.getResult().getHitCount()); + h = msg.getResult().getHit(0); + assertTrue(h instanceof SearchResult.HitWithSortBlob); + assertEquals(89.0, h.getRank(), 1E-6); + assertEquals("doc1", h.getDocId()); + byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); + assertEquals(9, b.length); + byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' }; + for (int i = 0; i < b.length; i++) { + assertEquals(e[i], b[i]); + } + h = msg.getResult().getHit(1); + assertTrue(h instanceof SearchResult.HitWithSortBlob); + assertEquals(109.0, h.getRank(), 1E-6); + assertEquals("doc17", h.getDocId()); + b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); + assertEquals(9, b.length); + byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' }; + for (int i = 0; i < b.length; i++) { + assertEquals(d[i], b[i]); + } + h = msg.getResult().getHit(2); + assertTrue(h instanceof SearchResult.HitWithSortBlob); + assertEquals(90.0, h.getRank(), 1E-6); + assertEquals("doc18", h.getDocId()); + b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); + assertEquals(9, b.length); + byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' }; + for (int i = 0; i < b.length; i++) { + assertEquals(c[i], b[i]); + } + } + } + + public class testGetBucketListReply implements RunnableTest { + + public void run() { + GetBucketListReply reply = new GetBucketListReply(); + reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(16, 123), "foo")); + reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(17, 1123), "bar")); + reply.getBuckets().add(new GetBucketListReply.BucketInfo(new BucketId(18, 11123), "zoink")); + + assertEquals(56, serialize("GetBucketListReply", reply)); + + for (Language lang : LANGUAGES) { + reply = (GetBucketListReply)deserialize("GetBucketListReply", DocumentProtocol.REPLY_GETBUCKETLIST, lang); + assertEquals(reply.getBuckets().get(0), new GetBucketListReply.BucketInfo(new BucketId(16, 123), "foo")); + assertEquals(reply.getBuckets().get(1), new GetBucketListReply.BucketInfo(new BucketId(17, 1123), "bar")); + assertEquals(reply.getBuckets().get(2), new GetBucketListReply.BucketInfo(new BucketId(18, 11123), "zoink")); + } + } + } + + public class testGetBucketStateReply implements RunnableTest { + + public void run() { + GlobalId foo = new GlobalId(IdString.createIdString("doc:scheme:foo")); + GlobalId bar = new GlobalId(IdString.createIdString("doc:scheme:bar")); + + GetBucketStateReply reply = new GetBucketStateReply(); + List<DocumentState> state = new ArrayList<>(2); + state.add(new DocumentState(foo, 777, false)); + state.add(new DocumentState(bar, 888, true)); + reply.setBucketState(state); + assertEquals(53, serialize("GetBucketStateReply", reply)); + + for (Language lang : LANGUAGES) { + reply = (GetBucketStateReply)deserialize("GetBucketStateReply", DocumentProtocol.REPLY_GETBUCKETSTATE, lang); + assertEquals(777, reply.getBucketState().get(0).getTimestamp()); + assertEquals(foo, reply.getBucketState().get(0).getGid()); + assertEquals(false, reply.getBucketState().get(0).isRemoveEntry()); + assertEquals(888, reply.getBucketState().get(1).getTimestamp()); + assertEquals(bar, reply.getBucketState().get(1).getGid()); + assertEquals(true, reply.getBucketState().get(1).isRemoveEntry()); } } } + public class testGetDocumentReply implements RunnableTest { + + public void run() { + GetDocumentReply reply = new GetDocumentReply(new Document(protocol.getDocumentTypeManager().getDocumentType("testdoc"), "doc:scheme:")); + assertEquals(43, serialize("GetDocumentReply", reply)); + + for (Language lang : LANGUAGES) { + reply = (GetDocumentReply)deserialize("GetDocumentReply", DocumentProtocol.REPLY_GETDOCUMENT, lang); + assertEquals("testdoc", reply.getDocument().getDataType().getName()); + assertEquals("doc:scheme:", reply.getDocument().getId().toString()); + } + } + } + + public class testMapVisitorReply implements RunnableTest { + + public void run() { + testVisitorReply("MapVisitorReply", DocumentProtocol.REPLY_MAPVISITOR); + } + } + + protected void testDocumentReply(String filename, int type) { + DocumentReply reply = new DocumentReply(type); + assertEquals(5, serialize(filename, reply)); + + for (Language lang : LANGUAGES) { + reply = (DocumentReply)deserialize(filename, type, lang); + assertNotNull(reply); + } + } + + protected void testVisitorReply(String filename, int type) { + VisitorReply reply = new VisitorReply(type); + assertEquals(5, serialize(filename, reply)); + + for (Language lang : LANGUAGES) { + reply = (VisitorReply)deserialize(filename, type, lang); + assertNotNull(reply); + } + } + static int serializedLength(String str) { return 4 + str.length(); } diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java index 89698e44705..59cbf5b65ae 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/RoutableFactoryTestCase.java @@ -8,7 +8,7 @@ import com.yahoo.document.serialization.DocumentSerializer; import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.messagebus.protocol.DocumentReply; -import com.yahoo.documentapi.messagebus.protocol.RoutableFactories50; +import com.yahoo.documentapi.messagebus.protocol.RoutableFactories52; import com.yahoo.jrt.ListenFailedException; import com.yahoo.jrt.slobrok.server.Slobrok; import com.yahoo.messagebus.DestinationSession; @@ -130,7 +130,7 @@ public class RoutableFactoryTestCase { assertFalse(reply.hasErrors()); } - private static class MyMessageFactory extends RoutableFactories50.DocumentMessageFactory { + private static class MyMessageFactory extends RoutableFactories52.DocumentMessageFactory { @Override protected DocumentMessage doDecode(DocumentDeserializer buf) { @@ -143,7 +143,7 @@ public class RoutableFactoryTestCase { } } - private static class MyReplyFactory extends RoutableFactories50.DocumentReplyFactory { + private static class MyReplyFactory extends RoutableFactories52.DocumentReplyFactory { @Override protected DocumentReply doDecode(DocumentDeserializer buf) { diff --git a/documentapi/src/test/java/com/yahoo/documentapi/test/AbstractDocumentApiTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/test/AbstractDocumentApiTestCase.java index ed862b75828..6160c9deca6 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/test/AbstractDocumentApiTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/test/AbstractDocumentApiTestCase.java @@ -33,7 +33,7 @@ public abstract class AbstractDocumentApiTestCase { @Test public void requireThatSyncSessionWorks() { - SyncSession session = access().createSyncSession(new SyncParameters()); + SyncSession session = access().createSyncSession(new SyncParameters.Builder().build()); DocumentType type = access().getDocumentTypeManager().getDocumentType("music"); Document doc1 = new Document(type, new DocumentId("doc:music:1")); diff --git a/documentapi/src/tests/messagebus/messagebus_test.cpp b/documentapi/src/tests/messagebus/messagebus_test.cpp index dff3f3f1bac..22a46951ecc 100644 --- a/documentapi/src/tests/messagebus/messagebus_test.cpp +++ b/documentapi/src/tests/messagebus/messagebus_test.cpp @@ -4,6 +4,7 @@ #include <vespa/document/datatype/documenttype.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/update/documentupdate.h> #include <vespa/documentapi/documentapi.h> #include <vespa/documentapi/loadtypes/loadtypeset.h> #include <vespa/vdslib/state/clusterstate.h> @@ -69,10 +70,10 @@ void Test::testMessage() { LoadTypeSet set; DocumentProtocol protocol(set, _repo); - Blob blob = protocol.encode(vespalib::Version(5,0), upd1); + Blob blob = protocol.encode(vespalib::Version(5,115), upd1); EXPECT_TRUE(blob.size() > 0); - Routable::UP dec1 = protocol.decode(vespalib::Version(5,0), blob); + Routable::UP dec1 = protocol.decode(vespalib::Version(5,115), blob); EXPECT_TRUE(dec1.get() != NULL); EXPECT_TRUE(dec1->isReply() == false); EXPECT_TRUE(dec1->getType() == DocumentProtocol::MESSAGE_UPDATEDOCUMENT); diff --git a/documentapi/src/tests/messages/CMakeLists.txt b/documentapi/src/tests/messages/CMakeLists.txt index b762eaca657..19e1901003d 100644 --- a/documentapi/src/tests/messages/CMakeLists.txt +++ b/documentapi/src/tests/messages/CMakeLists.txt @@ -1,23 +1,4 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(documentapi_messages50_test_app TEST - SOURCES - testbase.cpp - messages50test.cpp - messages50app.cpp - DEPENDS - documentapi -) -vespa_add_test(NAME documentapi_messages50_test_app COMMAND documentapi_messages50_test_app) -vespa_add_executable(documentapi_messages51_test_app TEST - SOURCES - testbase.cpp - messages50test.cpp - messages51test.cpp - messages51app.cpp - DEPENDS - documentapi -) -vespa_add_test(NAME documentapi_messages51_test_app COMMAND documentapi_messages51_test_app) vespa_add_executable(documentapi_messages52_test_app TEST SOURCES testbase.cpp diff --git a/documentapi/src/tests/messages/messages50app.cpp b/documentapi/src/tests/messages/messages50app.cpp deleted file mode 100644 index aa42302beba..00000000000 --- a/documentapi/src/tests/messages/messages50app.cpp +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "messages50test.h" - -TEST_APPHOOK(Messages50Test); diff --git a/documentapi/src/tests/messages/messages50test.cpp b/documentapi/src/tests/messages/messages50test.cpp index 6705277d5c3..cee0af5ad8e 100644 --- a/documentapi/src/tests/messages/messages50test.cpp +++ b/documentapi/src/tests/messages/messages50test.cpp @@ -5,6 +5,7 @@ #include <vespa/document/datatype/documenttype.h> #include <vespa/document/select/parser.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/update/documentupdate.h> #include <vespa/document/update/fieldpathupdates.h> #include <vespa/documentapi/documentapi.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> @@ -22,7 +23,6 @@ Messages50Test::Messages50Test() { // This list MUST mirror the list of routable factories from the DocumentProtocol constructor that support // version 5.0. When adding tests to this list, please KEEP THEM ORDERED alphabetically like they are now. - putTest(DocumentProtocol::MESSAGE_BATCHDOCUMENTUPDATE, TEST_METHOD(Messages50Test::testBatchDocumentUpdateMessage)); putTest(DocumentProtocol::MESSAGE_CREATEVISITOR, TEST_METHOD(Messages50Test::testCreateVisitorMessage)); putTest(DocumentProtocol::MESSAGE_DESTROYVISITOR, TEST_METHOD(Messages50Test::testDestroyVisitorMessage)); putTest(DocumentProtocol::MESSAGE_DOCUMENTLIST, TEST_METHOD(Messages50Test::testDocumentListMessage)); @@ -41,7 +41,6 @@ Messages50Test::Messages50Test() putTest(DocumentProtocol::MESSAGE_UPDATEDOCUMENT, TEST_METHOD(Messages50Test::testUpdateDocumentMessage)); putTest(DocumentProtocol::MESSAGE_VISITORINFO, TEST_METHOD(Messages50Test::testVisitorInfoMessage)); - putTest(DocumentProtocol::REPLY_BATCHDOCUMENTUPDATE, TEST_METHOD(Messages50Test::testBatchDocumentUpdateReply)); putTest(DocumentProtocol::REPLY_CREATEVISITOR, TEST_METHOD(Messages50Test::testCreateVisitorReply)); putTest(DocumentProtocol::REPLY_DESTROYVISITOR, TEST_METHOD(Messages50Test::testDestroyVisitorReply)); putTest(DocumentProtocol::REPLY_DOCUMENTLIST, TEST_METHOD(Messages50Test::testDocumentListReply)); @@ -259,38 +258,27 @@ Messages50Test::testDocumentSummaryMessage() EXPECT_EQUAL(srm.hasSequenceId(), false); EXPECT_EQUAL(srm.getSummaryCount(), size_t(0)); - mbus::Blob data = encode(srm); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(12), serialize("DocumentSummaryMessage-1", srm)); - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(12), data.size()); - - writeFile(getPath("5-cpp-DocumentSummaryMessage-1.dat"), data); - // print(data); - - mbus::Routable::UP routable = decode(data); + mbus::Routable::UP routable = deserialize("DocumentSummaryMessage-1", DocumentProtocol::MESSAGE_DOCUMENTSUMMARY, LANG_CPP); if (!EXPECT_TRUE(routable.get() != NULL)) { return false; } - EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_DOCUMENTSUMMARY); DocumentSummaryMessage * dm = static_cast<DocumentSummaryMessage *>(routable.get()); EXPECT_EQUAL(dm->getSummaryCount(), size_t(0)); srm.addSummary("doc1", "summary1", 8); srm.addSummary("aoc17", "summary45", 9); - data = encode(srm); - //print(data); - const void *summary(NULL); const char *docId(NULL); size_t sz(0); - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 52u, data.size()); - writeFile(getPath("5-cpp-DocumentSummaryMessage-2.dat"), data); - routable = decode(data); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 52u, serialize("DocumentSummaryMessage-2", srm)); + routable = deserialize("DocumentSummaryMessage-2", DocumentProtocol::MESSAGE_DOCUMENTSUMMARY, LANG_CPP); if (!EXPECT_TRUE(routable.get() != NULL)) { return false; } - EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_DOCUMENTSUMMARY); dm = static_cast<DocumentSummaryMessage *>(routable.get()); EXPECT_EQUAL(dm->getSummaryCount(), size_t(2)); dm->getSummary(0, docId, summary, sz); @@ -304,14 +292,11 @@ Messages50Test::testDocumentSummaryMessage() srm.sort(); - data = encode(srm); - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 52u, data.size()); - writeFile(getPath("5-cpp-DocumentSummaryMessage-3.dat"), data); - routable = decode(data); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 52u, serialize("DocumentSummaryMessage-3", srm)); + routable = deserialize("DocumentSummaryMessage-3", DocumentProtocol::MESSAGE_DOCUMENTSUMMARY, LANG_CPP); if (!EXPECT_TRUE(routable.get() != NULL)) { return false; } - EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_DOCUMENTSUMMARY); dm = static_cast<DocumentSummaryMessage *>(routable.get()); EXPECT_EQUAL(dm->getSummaryCount(), size_t(2)); dm->getSummary(0, docId, summary, sz); @@ -524,38 +509,25 @@ Messages50Test::testSearchResultMessage() EXPECT_EQUAL(srm.vdslib::SearchResult::getSerializedSize(), 20u); EXPECT_EQUAL(srm.getSerializedSize(), 20u); - mbus::Blob data = encode(srm); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(24), serialize("SearchResultMessage-1", srm)); - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(24), data.size()); - - writeFile(getPath("5-cpp-SearchResultMessage-1.dat"), data); - // print(data); - - mbus::Routable::UP routable = decode(data); + mbus::Routable::UP routable = deserialize("SearchResultMessage-1", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); if (!EXPECT_TRUE(routable.get() != NULL)) { return false; } - EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_SEARCHRESULT); SearchResultMessage * dm = static_cast<SearchResultMessage *>(routable.get()); EXPECT_EQUAL(dm->getSequenceId(), size_t(0)); EXPECT_EQUAL(dm->getHitCount(), size_t(0)); srm.addHit(0, "doc1", 89); srm.addHit(1, "doc17", 109); - //srm.setSequenceId(567); - - data = encode(srm); - //EXPECT_EQUAL(srm.getSequenceId(), size_t(567)); - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 55u, data.size()); - writeFile(getPath("5-cpp-SearchResultMessage-2.dat"), data); - routable = decode(data); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 55u, serialize("SearchResultMessage-2", srm)); + routable = deserialize("SearchResultMessage-2", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); if (!EXPECT_TRUE(routable.get() != NULL)) { return false; } - EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_SEARCHRESULT); dm = static_cast<SearchResultMessage *>(routable.get()); -// EXPECT_EQUAL(dm->getSequenceId(), size_t(567)); EXPECT_EQUAL(dm->getHitCount(), size_t(2)); const char *docId; SearchResultMessage::RankType rank; @@ -568,16 +540,12 @@ Messages50Test::testSearchResultMessage() srm.sort(); - data = encode(srm); - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 55u, data.size()); - writeFile(getPath("5-cpp-SearchResultMessage-3.dat"), data); - routable = decode(data); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 55u, serialize("SearchResultMessage-3", srm)); + routable = deserialize("SearchResultMessage-3", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); if (!EXPECT_TRUE(routable.get() != NULL)) { return false; } - EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_SEARCHRESULT); dm = static_cast<SearchResultMessage *>(routable.get()); -// EXPECT_EQUAL(dm->getSequenceId(), size_t(567)); EXPECT_EQUAL(dm->getHitCount(), size_t(2)); dm->getHit(0, docId, rank); EXPECT_EQUAL(rank, SearchResultMessage::RankType(109)); @@ -590,18 +558,13 @@ Messages50Test::testSearchResultMessage() srm2.addHit(0, "doc1", 89, "sortdata2", 9); srm2.addHit(1, "doc17", 109, "sortdata1", 9); srm2.addHit(2, "doc18", 90, "sortdata3", 9); - //srm2.setSequenceId(567); - data = encode(srm2); - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 108u, data.size()); - writeFile(getPath("5-cpp-SearchResultMessage-4.dat"), data); - routable = decode(data); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 108u, serialize("SearchResultMessage-4", srm2)); + routable = deserialize("SearchResultMessage-4", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); if (!EXPECT_TRUE(routable.get() != NULL)) { return false; } - EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_SEARCHRESULT); dm = static_cast<SearchResultMessage *>(routable.get()); - //EXPECT_EQUAL(dm->getSequenceId(), size_t(567)); EXPECT_EQUAL(dm->getHitCount(), size_t(3)); dm->getHit(0, docId, rank); EXPECT_EQUAL(rank, SearchResultMessage::RankType(89)); @@ -635,16 +598,12 @@ Messages50Test::testSearchResultMessage() EXPECT_EQUAL(rank, SearchResultMessage::RankType(90)); EXPECT_EQUAL(strcmp("doc18", docId), 0); - data = encode(srm2); - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 108u, data.size()); - writeFile(getPath("5-cpp-SearchResultMessage-5.dat"), data); - routable = decode(data); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 108u, serialize("SearchResultMessage-5", srm2)); + routable = deserialize("SearchResultMessage-5", DocumentProtocol::MESSAGE_SEARCHRESULT, LANG_CPP); if (!EXPECT_TRUE(routable.get() != NULL)) { return false; } - EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_SEARCHRESULT); dm = static_cast<SearchResultMessage *>(routable.get()); -// EXPECT_EQUAL(dm->getSequenceId(), size_t(567)); EXPECT_EQUAL(dm->getHitCount(), size_t(3)); dm->getHit(0, docId, rank); dm->getSortBlob(0, buf, sz); @@ -693,90 +652,6 @@ Messages50Test::testUpdateDocumentMessage() } bool -Messages50Test::testBatchDocumentUpdateMessage() -{ - const DocumentTypeRepo &repo = getTypeRepo(); - const document::DocumentType &docType = *repo.getDocumentType("testdoc"); - - BatchDocumentUpdateMessage msg(1234); - - { - document::DocumentUpdate::SP upd; - upd.reset(new document::DocumentUpdate(repo, docType, document::DocumentId("userdoc:footype:1234:foo"))); - upd->addFieldPathUpdate(document::FieldPathUpdate::CP( - new document::RemoveFieldPathUpdate("intfield", "testdoc.intfield > 0"))); - msg.addUpdate(upd); - } - { - document::DocumentUpdate::SP upd; - upd.reset(new document::DocumentUpdate(repo, docType, document::DocumentId("orderdoc(32,17):footype:1234:123456789:foo"))); - upd->addFieldPathUpdate(document::FieldPathUpdate::CP( - new document::RemoveFieldPathUpdate("intfield", "testdoc.intfield > 0"))); - msg.addUpdate(upd); - } - try { - document::DocumentUpdate::SP upd; - upd.reset(new document::DocumentUpdate(repo, docType, document::DocumentId("userdoc:footype:5678:foo"))); - upd->addFieldPathUpdate(document::FieldPathUpdate::CP( - new document::RemoveFieldPathUpdate("intfield", "testdoc.intfield > 0"))); - msg.addUpdate(upd); - EXPECT_TRUE(false); - } catch (...) { - } - try { - document::DocumentUpdate::SP upd; - upd.reset(new document::DocumentUpdate(repo, docType, document::DocumentId("groupdoc:footype:hable:foo"))); - upd->addFieldPathUpdate(document::FieldPathUpdate::CP( - new document::RemoveFieldPathUpdate("intfield", "testdoc.intfield > 0"))); - msg.addUpdate(upd); - EXPECT_TRUE(false); - } catch (...) { - } - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 202u, serialize("BatchDocumentUpdateMessage", msg)); - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("BatchDocumentUpdateMessage", DocumentProtocol::MESSAGE_BATCHDOCUMENTUPDATE, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { - BatchDocumentUpdateMessage &ref = static_cast<BatchDocumentUpdateMessage&>(*obj); - EXPECT_EQUAL(2u, ref.getUpdates().size()); - } - } - - return true; -} - -bool -Messages50Test::testBatchDocumentUpdateReply() -{ - BatchDocumentUpdateReply reply; - reply.setHighestModificationTimestamp(30); - { - std::vector<bool> notFound(3); - notFound[0] = false; - notFound[1] = true; - notFound[2] = true; - reply.getDocumentsNotFound() = notFound; - } - - EXPECT_EQUAL(20u, serialize("BatchDocumentUpdateReply", reply)); - - for (uint32_t lang = 0; lang < NUM_LANGUAGES; ++lang) { - mbus::Routable::UP obj = deserialize("BatchDocumentUpdateReply", DocumentProtocol::REPLY_BATCHDOCUMENTUPDATE, lang); - if (EXPECT_TRUE(obj.get() != NULL)) { - BatchDocumentUpdateReply &ref = dynamic_cast<BatchDocumentUpdateReply&>(*obj); - EXPECT_EQUAL(30u, ref.getHighestModificationTimestamp()); - { - const std::vector<bool>& notFound = ref.getDocumentsNotFound(); - EXPECT_TRUE(notFound[0] == false); - EXPECT_TRUE(notFound[1] == true); - EXPECT_TRUE(notFound[2] == true); - } - } - } - return true; -} - -bool Messages50Test::testQueryResultMessage() { QueryResultMessage srm; @@ -787,18 +662,12 @@ Messages50Test::testQueryResultMessage() EXPECT_EQUAL(sr.getSerializedSize(), 20u); EXPECT_EQUAL(srm.getApproxSize(), 28u); - mbus::Blob data = encode(srm); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(32), data.size()); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + size_t(32), serialize("QueryResultMessage-1", srm)); - writeFile(getPath("5-cpp-QueryResultMessage-1.dat"), data); - // print(data); - - mbus::Routable::UP routable = decode(data); + mbus::Routable::UP routable = deserialize("QueryResultMessage-1", DocumentProtocol::MESSAGE_QUERYRESULT, LANG_CPP); if (!EXPECT_TRUE(routable.get() != NULL)) { return false; } - EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_QUERYRESULT); QueryResultMessage * dm = static_cast<QueryResultMessage *>(routable.get()); vdslib::SearchResult * dr(&dm->getSearchResult()); EXPECT_EQUAL(dm->getSequenceId(), size_t(0)); @@ -807,15 +676,11 @@ Messages50Test::testQueryResultMessage() sr.addHit(0, "doc1", 89); sr.addHit(1, "doc17", 109); - data = encode(srm); - - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 63u, data.size()); - writeFile(getPath("5-cpp-QueryResultMessage-2.dat"), data); - routable = decode(data); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 63u, serialize("QueryResultMessage-2", srm)); + routable = deserialize("QueryResultMessage-2", DocumentProtocol::MESSAGE_QUERYRESULT, LANG_CPP); if (!EXPECT_TRUE(routable.get() != NULL)) { return false; } - EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_QUERYRESULT); dm = static_cast<QueryResultMessage *>(routable.get()); dr = &dm->getSearchResult(); EXPECT_EQUAL(dr->getHitCount(), size_t(2)); @@ -830,14 +695,11 @@ Messages50Test::testQueryResultMessage() sr.sort(); - data = encode(srm); - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 63u, data.size()); - writeFile(getPath("5-cpp-QueryResultMessage-3.dat"), data); - routable = decode(data); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 63u, serialize("QueryResultMessage-3", srm)); + routable = deserialize("QueryResultMessage-3", DocumentProtocol::MESSAGE_QUERYRESULT, LANG_CPP); if (!EXPECT_TRUE(routable.get() != NULL)) { return false; } - EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_QUERYRESULT); dm = static_cast<QueryResultMessage *>(routable.get()); dr = &dm->getSearchResult(); EXPECT_EQUAL(dr->getHitCount(), size_t(2)); @@ -853,15 +715,12 @@ Messages50Test::testQueryResultMessage() sr2.addHit(0, "doc1", 89, "sortdata2", 9); sr2.addHit(1, "doc17", 109, "sortdata1", 9); sr2.addHit(2, "doc18", 90, "sortdata3", 9); - data = encode(srm2); - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 116u, data.size()); - writeFile(getPath("5-cpp-QueryResultMessage-4.dat"), data); - routable = decode(data); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 116u, serialize("QueryResultMessage-4", srm2)); + routable = deserialize("QueryResultMessage-4", DocumentProtocol::MESSAGE_QUERYRESULT, LANG_CPP); if (!EXPECT_TRUE(routable.get() != NULL)) { return false; } - EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_QUERYRESULT); dm = static_cast<QueryResultMessage *>(routable.get()); dr = &dm->getSearchResult(); EXPECT_EQUAL(dr->getHitCount(), size_t(3)); @@ -897,14 +756,11 @@ Messages50Test::testQueryResultMessage() EXPECT_EQUAL(rank, vdslib::SearchResult::RankType(90)); EXPECT_EQUAL(strcmp("doc18", docId), 0); - data = encode(srm2); - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 116u, data.size()); - writeFile(getPath("5-cpp-QueryResultMessage-5.dat"), data); - routable = decode(data); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 116u, serialize("QueryResultMessage-5", srm2)); + routable = deserialize("QueryResultMessage-5", DocumentProtocol::MESSAGE_QUERYRESULT, LANG_CPP); if (!EXPECT_TRUE(routable.get() != NULL)) { return false; } - EXPECT_EQUAL(routable->getType(), (uint32_t)DocumentProtocol::MESSAGE_QUERYRESULT); dm = static_cast<QueryResultMessage *>(routable.get()); dr = &dm->getSearchResult(); EXPECT_EQUAL(dr->getHitCount(), size_t(3)); diff --git a/documentapi/src/tests/messages/messages50test.h b/documentapi/src/tests/messages/messages50test.h index c764f814a45..96c1be78c27 100644 --- a/documentapi/src/tests/messages/messages50test.h +++ b/documentapi/src/tests/messages/messages50test.h @@ -13,8 +13,6 @@ protected: public: Messages50Test(); - bool testBatchDocumentUpdateMessage(); - bool testBatchDocumentUpdateReply(); bool testCreateVisitorMessage(); bool testCreateVisitorReply(); bool testDestroyVisitorMessage(); diff --git a/documentapi/src/tests/messages/messages51app.cpp b/documentapi/src/tests/messages/messages51app.cpp deleted file mode 100644 index 654fb932568..00000000000 --- a/documentapi/src/tests/messages/messages51app.cpp +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "messages51test.h" - -TEST_APPHOOK(Messages51Test); diff --git a/documentapi/src/tests/messages/messages52test.cpp b/documentapi/src/tests/messages/messages52test.cpp index 8f5d7381500..1da48d4aa41 100644 --- a/documentapi/src/tests/messages/messages52test.cpp +++ b/documentapi/src/tests/messages/messages52test.cpp @@ -5,6 +5,7 @@ #include "messages52test.h" #include <vespa/documentapi/documentapi.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/update/documentupdate.h> #include <vespa/document/update/fieldpathupdates.h> #include <vespa/document/datatype/documenttype.h> diff --git a/documentapi/src/tests/policies/policies_test.cpp b/documentapi/src/tests/policies/policies_test.cpp index 3e804c30415..c01cdfde30c 100644 --- a/documentapi/src/tests/policies/policies_test.cpp +++ b/documentapi/src/tests/policies/policies_test.cpp @@ -22,6 +22,7 @@ #include <vespa/document/fieldvalue/longfieldvalue.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/update/documentupdate.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/stringfmt.h> diff --git a/documentapi/src/vespa/documentapi/documentapi.h b/documentapi/src/vespa/documentapi/documentapi.h index dff4125e624..0056dd2abdf 100644 --- a/documentapi/src/vespa/documentapi/documentapi.h +++ b/documentapi/src/vespa/documentapi/documentapi.h @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.h> -#include <vespa/documentapi/messagebus/messages/batchdocumentupdatereply.h> #include <vespa/documentapi/messagebus/messages/getbucketstatemessage.h> #include <vespa/documentapi/messagebus/messages/getbucketstatereply.h> #include <vespa/documentapi/messagebus/messages/getdocumentmessage.h> diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp index d2661d0fe6c..b70a76690f9 100644 --- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.cpp @@ -1,7 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "routablefactories50.h" -#include "routablefactories51.h" #include "routablefactories52.h" #include "routablefactories60.h" #include "routingpolicyfactories.h" @@ -46,64 +44,49 @@ DocumentProtocol::DocumentProtocol(const LoadTypeSet& loadTypes, putRoutingPolicyFactory("LoadBalancer", IRoutingPolicyFactory::SP(new RoutingPolicyFactories::LoadBalancerPolicyFactory())); // Prepare version specifications to use when adding routable factories. - vespalib::VersionSpecification version50(5, 0); - vespalib::VersionSpecification version51(5, 1); vespalib::VersionSpecification version52(5, 115); vespalib::VersionSpecification version6(6, 221); - std::vector<vespalib::VersionSpecification> from50 = { version50, version51, version52, version6 }; - std::vector<vespalib::VersionSpecification> from51 = { version51, version52, version6 }; std::vector<vespalib::VersionSpecification> from52 = { version52, version6 }; std::vector<vespalib::VersionSpecification> from6 = { version6 }; - // Add 5.0 serialization - putRoutableFactory(MESSAGE_BATCHDOCUMENTUPDATE, IRoutableFactory::SP(new RoutableFactories50::BatchDocumentUpdateMessageFactory(*_repo)), from50); - putRoutableFactory(MESSAGE_CREATEVISITOR, IRoutableFactory::SP(new RoutableFactories50::CreateVisitorMessageFactory(*_repo)), from50); - putRoutableFactory(MESSAGE_DESTROYVISITOR, IRoutableFactory::SP(new RoutableFactories50::DestroyVisitorMessageFactory()), from50); - putRoutableFactory(MESSAGE_DOCUMENTLIST, IRoutableFactory::SP(new RoutableFactories50::DocumentListMessageFactory(*_repo)), from50); - putRoutableFactory(MESSAGE_DOCUMENTSUMMARY, IRoutableFactory::SP(new RoutableFactories50::DocumentSummaryMessageFactory()), from50); - putRoutableFactory(MESSAGE_EMPTYBUCKETS, IRoutableFactory::SP(new RoutableFactories50::EmptyBucketsMessageFactory()), from50); - putRoutableFactory(MESSAGE_GETBUCKETLIST, IRoutableFactory::SP(new RoutableFactories50::GetBucketListMessageFactory()), from50); - putRoutableFactory(MESSAGE_GETBUCKETSTATE, IRoutableFactory::SP(new RoutableFactories50::GetBucketStateMessageFactory()), from50); - putRoutableFactory(MESSAGE_GETDOCUMENT, IRoutableFactory::SP(new RoutableFactories50::GetDocumentMessageFactory()), from50); - putRoutableFactory(MESSAGE_MAPVISITOR, IRoutableFactory::SP(new RoutableFactories50::MapVisitorMessageFactory(*_repo)), from50); - putRoutableFactory(MESSAGE_PUTDOCUMENT, IRoutableFactory::SP(new RoutableFactories50::PutDocumentMessageFactory(*_repo)), from50); - putRoutableFactory(MESSAGE_QUERYRESULT, IRoutableFactory::SP(new RoutableFactories50::QueryResultMessageFactory()), from50); - putRoutableFactory(MESSAGE_REMOVEDOCUMENT, IRoutableFactory::SP(new RoutableFactories50::RemoveDocumentMessageFactory()), from50); - putRoutableFactory(MESSAGE_REMOVELOCATION, IRoutableFactory::SP(new RoutableFactories50::RemoveLocationMessageFactory(*_repo)), from50); - putRoutableFactory(MESSAGE_SEARCHRESULT, IRoutableFactory::SP(new RoutableFactories50::SearchResultMessageFactory()), from50); - putRoutableFactory(MESSAGE_STATBUCKET, IRoutableFactory::SP(new RoutableFactories50::StatBucketMessageFactory()), from50); - putRoutableFactory(MESSAGE_UPDATEDOCUMENT, IRoutableFactory::SP(new RoutableFactories50::UpdateDocumentMessageFactory(*_repo)), from50); - putRoutableFactory(MESSAGE_VISITORINFO, IRoutableFactory::SP(new RoutableFactories50::VisitorInfoMessageFactory()), from50); - putRoutableFactory(REPLY_BATCHDOCUMENTUPDATE, IRoutableFactory::SP(new RoutableFactories50::BatchDocumentUpdateReplyFactory()), from50); - putRoutableFactory(REPLY_CREATEVISITOR, IRoutableFactory::SP(new RoutableFactories50::CreateVisitorReplyFactory()), from50); - putRoutableFactory(REPLY_DESTROYVISITOR, IRoutableFactory::SP(new RoutableFactories50::DestroyVisitorReplyFactory()), from50); - putRoutableFactory(REPLY_DOCUMENTLIST, IRoutableFactory::SP(new RoutableFactories50::DocumentListReplyFactory()), from50); - putRoutableFactory(REPLY_DOCUMENTSUMMARY, IRoutableFactory::SP(new RoutableFactories50::DocumentSummaryReplyFactory()), from50); - putRoutableFactory(REPLY_EMPTYBUCKETS, IRoutableFactory::SP(new RoutableFactories50::EmptyBucketsReplyFactory()), from50); - putRoutableFactory(REPLY_GETBUCKETLIST, IRoutableFactory::SP(new RoutableFactories50::GetBucketListReplyFactory()), from50); - putRoutableFactory(REPLY_GETBUCKETSTATE, IRoutableFactory::SP(new RoutableFactories50::GetBucketStateReplyFactory()), from50); - putRoutableFactory(REPLY_GETDOCUMENT, IRoutableFactory::SP(new RoutableFactories50::GetDocumentReplyFactory(*_repo)), from50); - putRoutableFactory(REPLY_MAPVISITOR, IRoutableFactory::SP(new RoutableFactories50::MapVisitorReplyFactory()), from50); - putRoutableFactory(REPLY_PUTDOCUMENT, IRoutableFactory::SP(new RoutableFactories50::PutDocumentReplyFactory()), from50); - putRoutableFactory(REPLY_QUERYRESULT, IRoutableFactory::SP(new RoutableFactories50::QueryResultReplyFactory()), from50); - putRoutableFactory(REPLY_REMOVEDOCUMENT, IRoutableFactory::SP(new RoutableFactories50::RemoveDocumentReplyFactory()), from50); - putRoutableFactory(REPLY_REMOVELOCATION, IRoutableFactory::SP(new RoutableFactories50::RemoveLocationReplyFactory()), from50); - putRoutableFactory(REPLY_SEARCHRESULT, IRoutableFactory::SP(new RoutableFactories50::SearchResultReplyFactory()), from50); - putRoutableFactory(REPLY_STATBUCKET, IRoutableFactory::SP(new RoutableFactories50::StatBucketReplyFactory()), from50); - putRoutableFactory(REPLY_UPDATEDOCUMENT, IRoutableFactory::SP(new RoutableFactories50::UpdateDocumentReplyFactory()), from50); - putRoutableFactory(REPLY_VISITORINFO, IRoutableFactory::SP(new RoutableFactories50::VisitorInfoReplyFactory()), from50); - putRoutableFactory(REPLY_WRONGDISTRIBUTION, IRoutableFactory::SP(new RoutableFactories50::WrongDistributionReplyFactory()), from50); - - // Add 5.1 serialization - putRoutableFactory(MESSAGE_GETDOCUMENT, IRoutableFactory::SP(new RoutableFactories51::GetDocumentMessageFactory()), from51); - putRoutableFactory(MESSAGE_CREATEVISITOR, IRoutableFactory::SP(new RoutableFactories51::CreateVisitorMessageFactory(*_repo)), from51); - putRoutableFactory(REPLY_DOCUMENTIGNORED, IRoutableFactory::SP(new RoutableFactories51::DocumentIgnoredReplyFactory()), from51); - // Add 5.2 serialization + putRoutableFactory(MESSAGE_CREATEVISITOR, IRoutableFactory::SP(new RoutableFactories52::CreateVisitorMessageFactory(*_repo)), from52); + putRoutableFactory(MESSAGE_DESTROYVISITOR, IRoutableFactory::SP(new RoutableFactories52::DestroyVisitorMessageFactory()), from52); + putRoutableFactory(MESSAGE_DOCUMENTLIST, IRoutableFactory::SP(new RoutableFactories52::DocumentListMessageFactory(*_repo)), from52); + putRoutableFactory(MESSAGE_DOCUMENTSUMMARY, IRoutableFactory::SP(new RoutableFactories52::DocumentSummaryMessageFactory()), from52); + putRoutableFactory(MESSAGE_EMPTYBUCKETS, IRoutableFactory::SP(new RoutableFactories52::EmptyBucketsMessageFactory()), from52); + putRoutableFactory(MESSAGE_GETBUCKETLIST, IRoutableFactory::SP(new RoutableFactories52::GetBucketListMessageFactory()), from52); + putRoutableFactory(MESSAGE_GETBUCKETSTATE, IRoutableFactory::SP(new RoutableFactories52::GetBucketStateMessageFactory()), from52); + putRoutableFactory(MESSAGE_GETDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::GetDocumentMessageFactory()), from52); + putRoutableFactory(MESSAGE_MAPVISITOR, IRoutableFactory::SP(new RoutableFactories52::MapVisitorMessageFactory(*_repo)), from52); putRoutableFactory(MESSAGE_PUTDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::PutDocumentMessageFactory(*_repo)), from52); - putRoutableFactory(MESSAGE_UPDATEDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::UpdateDocumentMessageFactory(*_repo)), from52); + putRoutableFactory(MESSAGE_QUERYRESULT, IRoutableFactory::SP(new RoutableFactories52::QueryResultMessageFactory()), from52); putRoutableFactory(MESSAGE_REMOVEDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::RemoveDocumentMessageFactory()), from52); + putRoutableFactory(MESSAGE_REMOVELOCATION, IRoutableFactory::SP(new RoutableFactories52::RemoveLocationMessageFactory(*_repo)), from52); + putRoutableFactory(MESSAGE_SEARCHRESULT, IRoutableFactory::SP(new RoutableFactories52::SearchResultMessageFactory()), from52); + putRoutableFactory(MESSAGE_STATBUCKET, IRoutableFactory::SP(new RoutableFactories52::StatBucketMessageFactory()), from52); + putRoutableFactory(MESSAGE_UPDATEDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::UpdateDocumentMessageFactory(*_repo)), from52); + putRoutableFactory(MESSAGE_VISITORINFO, IRoutableFactory::SP(new RoutableFactories52::VisitorInfoMessageFactory()), from52); + putRoutableFactory(REPLY_CREATEVISITOR, IRoutableFactory::SP(new RoutableFactories52::CreateVisitorReplyFactory()), from52); + putRoutableFactory(REPLY_DESTROYVISITOR, IRoutableFactory::SP(new RoutableFactories52::DestroyVisitorReplyFactory()), from52); + putRoutableFactory(REPLY_DOCUMENTIGNORED, IRoutableFactory::SP(new RoutableFactories52::DocumentIgnoredReplyFactory()), from52); + putRoutableFactory(REPLY_DOCUMENTLIST, IRoutableFactory::SP(new RoutableFactories52::DocumentListReplyFactory()), from52); + putRoutableFactory(REPLY_DOCUMENTSUMMARY, IRoutableFactory::SP(new RoutableFactories52::DocumentSummaryReplyFactory()), from52); + putRoutableFactory(REPLY_EMPTYBUCKETS, IRoutableFactory::SP(new RoutableFactories52::EmptyBucketsReplyFactory()), from52); + putRoutableFactory(REPLY_GETBUCKETLIST, IRoutableFactory::SP(new RoutableFactories52::GetBucketListReplyFactory()), from52); + putRoutableFactory(REPLY_GETBUCKETSTATE, IRoutableFactory::SP(new RoutableFactories52::GetBucketStateReplyFactory()), from52); + putRoutableFactory(REPLY_GETDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::GetDocumentReplyFactory(*_repo)), from52); + putRoutableFactory(REPLY_MAPVISITOR, IRoutableFactory::SP(new RoutableFactories52::MapVisitorReplyFactory()), from52); + putRoutableFactory(REPLY_PUTDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::PutDocumentReplyFactory()), from52); + putRoutableFactory(REPLY_QUERYRESULT, IRoutableFactory::SP(new RoutableFactories52::QueryResultReplyFactory()), from52); + putRoutableFactory(REPLY_REMOVEDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::RemoveDocumentReplyFactory()), from52); + putRoutableFactory(REPLY_REMOVELOCATION, IRoutableFactory::SP(new RoutableFactories52::RemoveLocationReplyFactory()), from52); + putRoutableFactory(REPLY_SEARCHRESULT, IRoutableFactory::SP(new RoutableFactories52::SearchResultReplyFactory()), from52); + putRoutableFactory(REPLY_STATBUCKET, IRoutableFactory::SP(new RoutableFactories52::StatBucketReplyFactory()), from52); + putRoutableFactory(REPLY_UPDATEDOCUMENT, IRoutableFactory::SP(new RoutableFactories52::UpdateDocumentReplyFactory()), from52); + putRoutableFactory(REPLY_VISITORINFO, IRoutableFactory::SP(new RoutableFactories52::VisitorInfoReplyFactory()), from52); + putRoutableFactory(REPLY_WRONGDISTRIBUTION, IRoutableFactory::SP(new RoutableFactories52::WrongDistributionReplyFactory()), from52); // Add 6.x serialization (TODO finalize version) putRoutableFactory(MESSAGE_CREATEVISITOR, IRoutableFactory::SP(new RoutableFactories60::CreateVisitorMessageFactory(*_repo)), from6); diff --git a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h index 32e6dc1d95e..79f7d7c0ccc 100644 --- a/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h +++ b/documentapi/src/vespa/documentapi/messagebus/documentprotocol.h @@ -68,7 +68,6 @@ public: MESSAGE_EMPTYBUCKETS = DOCUMENT_MESSAGE + 23, MESSAGE_REMOVELOCATION = DOCUMENT_MESSAGE + 24, MESSAGE_QUERYRESULT = DOCUMENT_MESSAGE + 25, - MESSAGE_BATCHDOCUMENTUPDATE = DOCUMENT_MESSAGE + 26, // MESSAGE_GARBAGECOLLECT = DOCUMENT_MESSAGE + 27, DOCUMENT_REPLY = 200000, @@ -92,7 +91,6 @@ public: REPLY_EMPTYBUCKETS = DOCUMENT_REPLY + 23, REPLY_REMOVELOCATION = DOCUMENT_REPLY + 24, REPLY_QUERYRESULT = DOCUMENT_REPLY + 25, - REPLY_BATCHDOCUMENTUPDATE = DOCUMENT_REPLY + 26, // REPLY_GARBAGECOLLECT = DOCUMENT_REPLY + 27, REPLY_WRONGDISTRIBUTION = DOCUMENT_REPLY + 1000, REPLY_DOCUMENTIGNORED = DOCUMENT_REPLY + 1001 diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/CMakeLists.txt b/documentapi/src/vespa/documentapi/messagebus/messages/CMakeLists.txt index 3fa7e6086e9..7bd18a14719 100644 --- a/documentapi/src/vespa/documentapi/messagebus/messages/CMakeLists.txt +++ b/documentapi/src/vespa/documentapi/messagebus/messages/CMakeLists.txt @@ -1,8 +1,6 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(documentapi_documentapimessages OBJECT SOURCES - batchdocumentupdatemessage.cpp - batchdocumentupdatereply.cpp documentignoredreply.cpp documentmessage.cpp documentreply.cpp diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.cpp deleted file mode 100644 index 2b38f9b24ae..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.cpp +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "batchdocumentupdatemessage.h" -#include "batchdocumentupdatereply.h" -#include <vespa/documentapi/messagebus/documentprotocol.h> -#include <vespa/document/bucket/bucketidfactory.h> -#include <vespa/vespalib/util/exceptions.h> - -namespace documentapi { - -BatchDocumentUpdateMessage::BatchDocumentUpdateMessage(uint64_t userId) - : _userId(userId) -{ - setBucketId(document::UserDocIdString(vespalib::make_string("userdoc:foo:%lu:bar", _userId))); -} - -BatchDocumentUpdateMessage::BatchDocumentUpdateMessage(const string& group) - : _userId(0), - _group(group) -{ - setBucketId(document::GroupDocIdString("groupdoc:foo:" + _group + ":bar")); -} - -BatchDocumentUpdateMessage::~BatchDocumentUpdateMessage() {} - -void -BatchDocumentUpdateMessage::setBucketId(const document::IdString& idString) -{ - document::BucketIdFactory factory; - _bucketId = factory.getBucketId(document::DocumentId(idString)); -} - -void -BatchDocumentUpdateMessage::addUpdate(document::DocumentUpdate::SP update) -{ - verifyUpdate(*update); - _updates.push_back(update); -} - -void -BatchDocumentUpdateMessage::verifyUpdate(const document::DocumentUpdate& update) { - const document::IdString& idString = update.getId().getScheme(); - - if (_group.length()) { - string group; - - if (idString.hasGroup()) { - group = idString.getGroup(); - } else { - throw vespalib::IllegalArgumentException("Batch update message can only contain groupdoc or orderdoc items"); - } - - if (group != _group) { - throw vespalib::IllegalArgumentException(vespalib::make_string("Batch update message can not contain messages from group %s, only group %s", group.c_str(), _group.c_str())); - } - } else { - uint64_t userId; - - if (idString.hasNumber()) { - userId = idString.getNumber(); - } else { - throw vespalib::IllegalArgumentException("Batch update message can only contain userdoc or orderdoc items"); - } - - if (userId != _userId) { - throw vespalib::IllegalArgumentException(vespalib::make_string("Batch update message can not contain messages from user %llu, only user %llu", (long long unsigned)userId, (long long unsigned)_userId)); - } - } -} - -DocumentReply::UP -BatchDocumentUpdateMessage::doCreateReply() const -{ - return DocumentReply::UP(new BatchDocumentUpdateReply()); -} - -uint32_t -BatchDocumentUpdateMessage::getType() const { - return DocumentProtocol::MESSAGE_BATCHDOCUMENTUPDATE; -} - -} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.h b/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.h deleted file mode 100644 index 54599c2627b..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.h +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "documentmessage.h" -#include "writedocumentreply.h" -#include <vespa/document/update/documentupdate.h> -#include <vespa/document/bucket/bucketid.h> -#include <vespa/document/base/idstring.h> - -namespace documentapi { - -/** - Message to use to send multiple updates for documents - belonging to the same user or group to Vespa. Using this - message improves performance in VDS mainly. -*/ -class BatchDocumentUpdateMessage : public DocumentMessage -{ -public: - typedef std::vector<document::DocumentUpdate::SP > UpdateList; - - /** - Creates a batch update message that can contain only updates - for documents belonging to the given user. - */ - BatchDocumentUpdateMessage(uint64_t userId); - - /** - Creates a batch update message that can contain only updates - for documents belonging to the given group. - */ - BatchDocumentUpdateMessage(const string& group); - ~BatchDocumentUpdateMessage(); - - /** - @return Returns a list of the updates to be performed. - */ - const UpdateList& getUpdates() const { return _updates; }; - - /** - Adds an update to be performed. - */ - void addUpdate(document::DocumentUpdate::SP update); - - /** - Returns the user id that this batch can contain. - Only valid if this object was created with the first constructor. - */ - uint64_t getUserId() const { return _userId; }; - - /** - Returns the grouo that this batch can contain. - Only valid if this object was created with the second constructor. - */ - const string& getGroup() const { return _group; } - - uint32_t getType() const override; - - /** - Returns a bucket id suitable for routing this message. - */ - const document::BucketId& getBucketId() const { return _bucketId; } - - string toString() const override { return "batchdocumentupdatemessage"; } - -protected: - DocumentReply::UP doCreateReply() const override; - -private: - uint64_t _userId; - string _group; - - UpdateList _updates; - document::BucketId _bucketId; - - void verifyUpdate(const document::DocumentUpdate& update); - void setBucketId(const document::IdString& idString); -}; - -} - diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatereply.cpp b/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatereply.cpp deleted file mode 100644 index 483740233c0..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatereply.cpp +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "batchdocumentupdatereply.h" -#include <vespa/documentapi/messagebus/documentprotocol.h> - -namespace documentapi { - -BatchDocumentUpdateReply::BatchDocumentUpdateReply() - : WriteDocumentReply(DocumentProtocol::REPLY_BATCHDOCUMENTUPDATE) -{ } - -BatchDocumentUpdateReply::~BatchDocumentUpdateReply() {} - -} diff --git a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatereply.h b/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatereply.h deleted file mode 100644 index be1f5bc75d5..00000000000 --- a/documentapi/src/vespa/documentapi/messagebus/messages/batchdocumentupdatereply.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "writedocumentreply.h" - -namespace documentapi { - -class BatchDocumentUpdateReply : public WriteDocumentReply -{ - /** - * If all documents to update are found, this vector will be empty. If - * one or more documents are not found, this vector will have the size of - * the initial number of updates, with entries set to true where the - * corresponding update was not found. - */ - std::vector<bool> _documentsNotFound; -public: - typedef std::unique_ptr<BatchDocumentUpdateReply> UP; - typedef std::shared_ptr<BatchDocumentUpdateReply> SP; - - BatchDocumentUpdateReply(); - ~BatchDocumentUpdateReply(); - - const std::vector<bool>& getDocumentsNotFound() const { return _documentsNotFound; } - std::vector<bool>& getDocumentsNotFound() { return _documentsNotFound; } - - string toString() const override { return "batchdocumentupdatereply"; } -}; - -} diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp index 42120b8052a..62135150d98 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/documentrouteselectorpolicy.cpp @@ -4,7 +4,6 @@ #include <vespa/document/bucket/bucketidfactory.h> #include <vespa/document/select/parser.h> #include <vespa/documentapi/messagebus/documentprotocol.h> -#include <vespa/documentapi/messagebus/messages/batchdocumentupdatemessage.h> #include <vespa/documentapi/messagebus/messages/putdocumentmessage.h> #include <vespa/documentapi/messagebus/messages/updatedocumentmessage.h> #include <vespa/documentapi/messagebus/messages/documentignoredreply.h> @@ -138,17 +137,6 @@ DocumentRouteSelectorPolicy::select(mbus::RoutingContext &context, const vespali return true; } } - - case DocumentProtocol::MESSAGE_BATCHDOCUMENTUPDATE: - { - const BatchDocumentUpdateMessage& mom = static_cast<const BatchDocumentUpdateMessage&>(msg); - for (uint32_t i = 0; i < mom.getUpdates().size(); i++) { - if (it->second->contains(*mom.getUpdates()[i]) == Result::False) { - return false; - } - } - return true; - } default: return true; } diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp index 723c6fc836a..472440d7863 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/storagepolicy.cpp @@ -2,6 +2,7 @@ #include "storagepolicy.h" #include <vespa/document/base/documentid.h> +#include <vespa/document/update/documentupdate.h> #include <vespa/messagebus/emptyreply.h> #include <vespa/messagebus/routing/verbatimdirective.h> #include <vespa/documentapi/documentapi.h> @@ -150,10 +151,6 @@ StoragePolicy::doSelect(mbus::RoutingContext &context) id = static_cast<const RemoveLocationMessage&>(msg).getBucketId(); break; - case DocumentProtocol::MESSAGE_BATCHDOCUMENTUPDATE: - id = static_cast<const BatchDocumentUpdateMessage&>(msg).getBucketId(); - break; - default: LOG(error, "Message type '%d' not supported.", msg.getType()); return; diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp index 26d85b57522..120ee1facd0 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.cpp @@ -4,6 +4,7 @@ #include <vespa/document/bucket/bucketidfactory.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/document/select/parser.h> +#include <vespa/document/update/documentupdate.h> #include <vespa/documentapi/documentapi.h> #include <vespa/documentapi/loadtypes/loadtypeset.h> #include <vespa/vespalib/objects/nbostream.h> @@ -67,69 +68,6 @@ RoutableFactories50::DocumentReplyFactory::decode(document::ByteBuffer &in, cons //////////////////////////////////////////////////////////////////////////////// DocumentMessage::UP -RoutableFactories50::BatchDocumentUpdateMessageFactory::doDecode(document::ByteBuffer &buf) const -{ - uint64_t userId = (uint64_t)decodeLong(buf); - string group = decodeString(buf); - - auto msg = (group.length()) - ? std::make_unique<BatchDocumentUpdateMessage>(group) - : std::make_unique<BatchDocumentUpdateMessage>(userId); - - uint32_t len = decodeInt(buf); - for (uint32_t i = 0; i < len; i++) { - document::DocumentUpdate::SP upd = document::DocumentUpdate::createHEAD(_repo, buf); - msg->addUpdate(upd); - } - - return msg; -} - -bool -RoutableFactories50::BatchDocumentUpdateMessageFactory::doEncode(const DocumentMessage &obj, vespalib::GrowableByteBuffer &buf) const -{ - const BatchDocumentUpdateMessage &msg = static_cast<const BatchDocumentUpdateMessage&>(obj); - - buf.putLong(msg.getUserId()); - buf.putString(msg.getGroup()); - buf.putInt(msg.getUpdates().size()); - - vespalib::nbostream stream; - for (const auto & update : msg.getUpdates()) { - update->serializeHEAD(stream); - } - buf.putBytes(stream.c_str(), stream.size()); - - return true; -} - -DocumentReply::UP -RoutableFactories50::BatchDocumentUpdateReplyFactory::doDecode(document::ByteBuffer &buf) const -{ - auto reply = std::make_unique<BatchDocumentUpdateReply>(); - reply->setHighestModificationTimestamp(decodeLong(buf)); - std::vector<bool>& notFound = reply->getDocumentsNotFound(); - notFound.resize(decodeInt(buf)); - for (std::size_t i = 0; i < notFound.size(); ++i) { - notFound[i] = decodeBoolean(buf); - } - return reply; -} - -bool -RoutableFactories50::BatchDocumentUpdateReplyFactory::doEncode(const DocumentReply &obj, vespalib::GrowableByteBuffer &buf) const -{ - const BatchDocumentUpdateReply& reply = static_cast<const BatchDocumentUpdateReply&>(obj); - buf.putLong(reply.getHighestModificationTimestamp()); - const std::vector<bool>& notFoundV = reply.getDocumentsNotFound(); - buf.putInt(notFoundV.size()); - for (bool notFound : notFoundV) { - buf.putBoolean(notFound); - } - return true; -} - -DocumentMessage::UP RoutableFactories50::CreateVisitorMessageFactory::doDecode(document::ByteBuffer &buf) const { auto msg = std::make_unique<CreateVisitorMessage>(); diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.h b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.h index 12d5c560786..f96b9641800 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories50.h +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories50.h @@ -157,19 +157,6 @@ public: // Factories // //////////////////////////////////////////////////////////////////////////////// - - class BatchDocumentUpdateMessageFactory : public DocumentMessageFactory { - const document::DocumentTypeRepo &_repo; - DocumentMessage::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentMessage &msg, vespalib::GrowableByteBuffer &buf) const override; - public: - BatchDocumentUpdateMessageFactory(const document::DocumentTypeRepo &r) : _repo(r) {} - }; - class BatchDocumentUpdateReplyFactory : public DocumentReplyFactory { - protected: - DocumentReply::UP doDecode(document::ByteBuffer &buf) const override; - bool doEncode(const DocumentReply &reply, vespalib::GrowableByteBuffer &buf) const override; - }; class CreateVisitorMessageFactory : public DocumentMessageFactory { const document::DocumentTypeRepo &_repo; protected: diff --git a/documentapi/src/vespa/documentapi/messagebus/routablefactories51.h b/documentapi/src/vespa/documentapi/messagebus/routablefactories51.h index fc20324b418..b71c478c540 100644 --- a/documentapi/src/vespa/documentapi/messagebus/routablefactories51.h +++ b/documentapi/src/vespa/documentapi/messagebus/routablefactories51.h @@ -10,7 +10,7 @@ namespace documentapi { * This class encapsulates all the {@link RoutableFactory} classes needed to implement factories for the document * routable. When adding new factories to this class, please KEEP THE THEM ORDERED alphabetically like they are now. */ -class RoutableFactories51 { +class RoutableFactories51 : public RoutableFactories50 { public: RoutableFactories51() = delete; diff --git a/documentapi/test/crosslanguagefiles/5-cpp-BatchDocumentUpdateMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-BatchDocumentUpdateMessage.dat Binary files differdeleted file mode 100644 index 95e663088c3..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-BatchDocumentUpdateMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-BatchDocumentUpdateReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-BatchDocumentUpdateReply.dat Binary files differdeleted file mode 100644 index 216db17f80e..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-BatchDocumentUpdateReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-BatchMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-BatchMessage.dat Binary files differdeleted file mode 100644 index c1c43e8081b..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-BatchMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-BatchReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-BatchReply.dat Binary files differdeleted file mode 100644 index be3a9ba1913..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-BatchReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-CreateVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-CreateVisitorMessage.dat Binary files differdeleted file mode 100644 index a506fc760bd..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-CreateVisitorMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-CreateVisitorReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-CreateVisitorReply.dat Binary files differdeleted file mode 100644 index e13917227d1..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-CreateVisitorReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-DestroyVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-DestroyVisitorMessage.dat Binary files differdeleted file mode 100644 index f39b31217e6..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-DestroyVisitorMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-DestroyVisitorReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-DestroyVisitorReply.dat Binary files differdeleted file mode 100644 index 1468f027b15..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-DestroyVisitorReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-DocumentListMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-DocumentListMessage.dat Binary files differdeleted file mode 100644 index 2d8d12d2704..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-DocumentListMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-DocumentListReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-DocumentListReply.dat Binary files differdeleted file mode 100644 index c8a1cd888f0..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-DocumentListReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryReply.dat Binary files differdeleted file mode 100644 index 16b1e4bc4ef..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-EmptyBucketsMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-EmptyBucketsMessage.dat Binary files differdeleted file mode 100644 index b9df278fd7a..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-EmptyBucketsMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-EmptyBucketsReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-EmptyBucketsReply.dat Binary files differdeleted file mode 100644 index 05510dd8c1e..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-EmptyBucketsReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-GetBucketListMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-GetBucketListMessage.dat Binary files differdeleted file mode 100644 index fa3de45ac5b..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-GetBucketListMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-GetBucketListReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-GetBucketListReply.dat Binary files differdeleted file mode 100644 index 830994ed785..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-GetBucketListReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-GetBucketStateMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-GetBucketStateMessage.dat Binary files differdeleted file mode 100644 index aa2d206ca3a..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-GetBucketStateMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-GetBucketStateReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-GetBucketStateReply.dat Binary files differdeleted file mode 100644 index 0f91e3759f8..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-GetBucketStateReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-GetDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-GetDocumentMessage.dat Binary files differdeleted file mode 100644 index 1444d617c22..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-GetDocumentMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-GetDocumentReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-GetDocumentReply.dat Binary files differdeleted file mode 100644 index c1ad7920a2e..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-GetDocumentReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-MapVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-MapVisitorMessage.dat Binary files differdeleted file mode 100644 index 1a8a837ea16..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-MapVisitorMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-MapVisitorReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-MapVisitorReply.dat Binary files differdeleted file mode 100644 index 541cd718a66..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-MapVisitorReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-MultiOperationMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-MultiOperationMessage.dat Binary files differdeleted file mode 100644 index 6efc54a6d09..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-MultiOperationMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-MultiOperationReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-MultiOperationReply.dat Binary files differdeleted file mode 100644 index 8ad31a95bd5..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-MultiOperationReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-PutDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-PutDocumentMessage.dat Binary files differdeleted file mode 100644 index 9e4a5c5f82f..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-PutDocumentMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-PutDocumentReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-PutDocumentReply.dat Binary files differdeleted file mode 100644 index 480544045bb..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-PutDocumentReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-QueryResultReply.dat Binary files differdeleted file mode 100644 index 003f35d63a7..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-RemoveDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-RemoveDocumentMessage.dat Binary files differdeleted file mode 100644 index 78b7972eb5c..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-RemoveDocumentMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-RemoveDocumentReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-RemoveDocumentReply.dat Binary files differdeleted file mode 100644 index bf5db8761e2..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-RemoveDocumentReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessage.dat Binary files differdeleted file mode 100644 index 16850a6aff3..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessageGroup.dat b/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessageGroup.dat Binary files differdeleted file mode 100644 index abd648184d7..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessageGroup.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessageUser.dat b/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessageUser.dat Binary files differdeleted file mode 100644 index c2a63cb94c0..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationMessageUser.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationReply.dat Binary files differdeleted file mode 100644 index 752c4dba399..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-RemoveLocationReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-SearchResultReply.dat Binary files differdeleted file mode 100644 index cce9c6f8d14..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-StatBucketMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-StatBucketMessage.dat Binary files differdeleted file mode 100644 index 1fc2b1cf3c1..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-StatBucketMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-StatBucketReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-StatBucketReply.dat Binary files differdeleted file mode 100644 index 0b98e240018..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-StatBucketReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-UpdateDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-UpdateDocumentMessage.dat Binary files differdeleted file mode 100644 index f1bca25ceca..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-UpdateDocumentMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-UpdateDocumentReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-UpdateDocumentReply.dat Binary files differdeleted file mode 100644 index c7151299366..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-UpdateDocumentReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-VisitorInfoMessage.dat b/documentapi/test/crosslanguagefiles/5-cpp-VisitorInfoMessage.dat Binary files differdeleted file mode 100644 index 80b44e0c6fd..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-VisitorInfoMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-VisitorInfoReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-VisitorInfoReply.dat Binary files differdeleted file mode 100644 index 57a656c9b2d..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-VisitorInfoReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-WrongDistributionReply.dat b/documentapi/test/crosslanguagefiles/5-cpp-WrongDistributionReply.dat Binary files differdeleted file mode 100644 index 0dbe13225ae..00000000000 --- a/documentapi/test/crosslanguagefiles/5-cpp-WrongDistributionReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-BatchDocumentUpdateMessage.dat b/documentapi/test/crosslanguagefiles/5-java-BatchDocumentUpdateMessage.dat Binary files differdeleted file mode 100644 index 95e663088c3..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-BatchDocumentUpdateMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-BatchDocumentUpdateReply.dat b/documentapi/test/crosslanguagefiles/5-java-BatchDocumentUpdateReply.dat Binary files differdeleted file mode 100644 index 216db17f80e..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-BatchDocumentUpdateReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-BatchMessage.dat b/documentapi/test/crosslanguagefiles/5-java-BatchMessage.dat Binary files differdeleted file mode 100644 index c1c43e8081b..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-BatchMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-BatchReply.dat b/documentapi/test/crosslanguagefiles/5-java-BatchReply.dat Binary files differdeleted file mode 100644 index be3a9ba1913..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-BatchReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-CreateVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5-java-CreateVisitorMessage.dat Binary files differdeleted file mode 100644 index a506fc760bd..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-CreateVisitorMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-CreateVisitorReply.dat b/documentapi/test/crosslanguagefiles/5-java-CreateVisitorReply.dat Binary files differdeleted file mode 100644 index e13917227d1..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-CreateVisitorReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-DestroyVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5-java-DestroyVisitorMessage.dat Binary files differdeleted file mode 100644 index f39b31217e6..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-DestroyVisitorMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-DestroyVisitorReply.dat b/documentapi/test/crosslanguagefiles/5-java-DestroyVisitorReply.dat Binary files differdeleted file mode 100644 index 1468f027b15..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-DestroyVisitorReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-DocumentListMessage.dat b/documentapi/test/crosslanguagefiles/5-java-DocumentListMessage.dat Binary files differdeleted file mode 100644 index 2d8d12d2704..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-DocumentListMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-DocumentListReply.dat b/documentapi/test/crosslanguagefiles/5-java-DocumentListReply.dat Binary files differdeleted file mode 100644 index c8a1cd888f0..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-DocumentListReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-DocumentSummaryReply.dat b/documentapi/test/crosslanguagefiles/5-java-DocumentSummaryReply.dat Binary files differdeleted file mode 100644 index 16b1e4bc4ef..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-DocumentSummaryReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-EmptyBucketsMessage.dat b/documentapi/test/crosslanguagefiles/5-java-EmptyBucketsMessage.dat Binary files differdeleted file mode 100644 index b9df278fd7a..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-EmptyBucketsMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-EmptyBucketsReply.dat b/documentapi/test/crosslanguagefiles/5-java-EmptyBucketsReply.dat Binary files differdeleted file mode 100644 index 05510dd8c1e..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-EmptyBucketsReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-GetBucketListMessage.dat b/documentapi/test/crosslanguagefiles/5-java-GetBucketListMessage.dat Binary files differdeleted file mode 100644 index fa3de45ac5b..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-GetBucketListMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-GetBucketListReply.dat b/documentapi/test/crosslanguagefiles/5-java-GetBucketListReply.dat Binary files differdeleted file mode 100644 index 830994ed785..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-GetBucketListReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-GetBucketStateMessage.dat b/documentapi/test/crosslanguagefiles/5-java-GetBucketStateMessage.dat Binary files differdeleted file mode 100644 index aa2d206ca3a..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-GetBucketStateMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-GetBucketStateReply.dat b/documentapi/test/crosslanguagefiles/5-java-GetBucketStateReply.dat Binary files differdeleted file mode 100644 index 0f91e3759f8..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-GetBucketStateReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-GetDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5-java-GetDocumentMessage.dat Binary files differdeleted file mode 100644 index 1444d617c22..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-GetDocumentMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-GetDocumentReply.dat b/documentapi/test/crosslanguagefiles/5-java-GetDocumentReply.dat Binary files differdeleted file mode 100644 index c1ad7920a2e..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-GetDocumentReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-MapVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5-java-MapVisitorMessage.dat Binary files differdeleted file mode 100644 index 1a8a837ea16..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-MapVisitorMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-MapVisitorReply.dat b/documentapi/test/crosslanguagefiles/5-java-MapVisitorReply.dat Binary files differdeleted file mode 100644 index 541cd718a66..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-MapVisitorReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-MultiOperationMessage.dat b/documentapi/test/crosslanguagefiles/5-java-MultiOperationMessage.dat Binary files differdeleted file mode 100644 index 6efc54a6d09..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-MultiOperationMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-MultiOperationReply.dat b/documentapi/test/crosslanguagefiles/5-java-MultiOperationReply.dat Binary files differdeleted file mode 100644 index 8ad31a95bd5..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-MultiOperationReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-PutDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5-java-PutDocumentMessage.dat Binary files differdeleted file mode 100644 index 9e4a5c5f82f..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-PutDocumentMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-PutDocumentReply.dat b/documentapi/test/crosslanguagefiles/5-java-PutDocumentReply.dat Binary files differdeleted file mode 100644 index 480544045bb..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-PutDocumentReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-QueryResultReply.dat b/documentapi/test/crosslanguagefiles/5-java-QueryResultReply.dat Binary files differdeleted file mode 100644 index 003f35d63a7..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-QueryResultReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-RemoveDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5-java-RemoveDocumentMessage.dat Binary files differdeleted file mode 100644 index 78b7972eb5c..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-RemoveDocumentMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-RemoveDocumentReply.dat b/documentapi/test/crosslanguagefiles/5-java-RemoveDocumentReply.dat Binary files differdeleted file mode 100644 index bf5db8761e2..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-RemoveDocumentReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessage.dat b/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessage.dat Binary files differdeleted file mode 100644 index 16850a6aff3..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessageGroup.dat b/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessageGroup.dat Binary files differdeleted file mode 100644 index 9d7901a8b6e..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessageGroup.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessageUser.dat b/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessageUser.dat Binary files differdeleted file mode 100644 index c2a63cb94c0..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-RemoveLocationMessageUser.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-RemoveLocationReply.dat b/documentapi/test/crosslanguagefiles/5-java-RemoveLocationReply.dat Binary files differdeleted file mode 100644 index 752c4dba399..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-RemoveLocationReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-SearchResultReply.dat b/documentapi/test/crosslanguagefiles/5-java-SearchResultReply.dat Binary files differdeleted file mode 100644 index cce9c6f8d14..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-SearchResultReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-StatBucketMessage.dat b/documentapi/test/crosslanguagefiles/5-java-StatBucketMessage.dat Binary files differdeleted file mode 100644 index 1fc2b1cf3c1..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-StatBucketMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-StatBucketReply.dat b/documentapi/test/crosslanguagefiles/5-java-StatBucketReply.dat Binary files differdeleted file mode 100644 index 0b98e240018..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-StatBucketReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-UpdateDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5-java-UpdateDocumentMessage.dat Binary files differdeleted file mode 100644 index f1bca25ceca..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-UpdateDocumentMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-UpdateDocumentReply.dat b/documentapi/test/crosslanguagefiles/5-java-UpdateDocumentReply.dat Binary files differdeleted file mode 100644 index c7151299366..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-UpdateDocumentReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-VisitorInfoMessage.dat b/documentapi/test/crosslanguagefiles/5-java-VisitorInfoMessage.dat Binary files differdeleted file mode 100644 index 80b44e0c6fd..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-VisitorInfoMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-VisitorInfoReply.dat b/documentapi/test/crosslanguagefiles/5-java-VisitorInfoReply.dat Binary files differdeleted file mode 100644 index 57a656c9b2d..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-VisitorInfoReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-java-WrongDistributionReply.dat b/documentapi/test/crosslanguagefiles/5-java-WrongDistributionReply.dat Binary files differdeleted file mode 100644 index 0dbe13225ae..00000000000 --- a/documentapi/test/crosslanguagefiles/5-java-WrongDistributionReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-BatchDocumentUpdateMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-BatchDocumentUpdateMessage.dat Binary files differdeleted file mode 100644 index 95e663088c3..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-BatchDocumentUpdateMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-BatchDocumentUpdateReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-BatchDocumentUpdateReply.dat Binary files differdeleted file mode 100644 index 216db17f80e..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-BatchDocumentUpdateReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-BatchMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-BatchMessage.dat Binary files differdeleted file mode 100644 index c1c43e8081b..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-BatchMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-BatchReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-BatchReply.dat Binary files differdeleted file mode 100644 index be3a9ba1913..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-BatchReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-CreateVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-CreateVisitorMessage.dat Binary files differdeleted file mode 100644 index 4ca8648e702..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-CreateVisitorMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-CreateVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-CreateVisitorReply.dat Binary files differdeleted file mode 100644 index e13917227d1..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-CreateVisitorReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DestroyVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DestroyVisitorMessage.dat Binary files differdeleted file mode 100644 index f39b31217e6..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-DestroyVisitorMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DestroyVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DestroyVisitorReply.dat Binary files differdeleted file mode 100644 index 1468f027b15..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-DestroyVisitorReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentIgnoredReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentIgnoredReply.dat Binary files differdeleted file mode 100644 index 15a7afe2a59..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentIgnoredReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentListMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentListMessage.dat Binary files differdeleted file mode 100644 index 2d8d12d2704..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentListMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentListReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentListReply.dat Binary files differdeleted file mode 100644 index c8a1cd888f0..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentListReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-1.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-1.dat Binary files differdeleted file mode 100644 index 0107dd5f350..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-1.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-2.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-2.dat Binary files differdeleted file mode 100644 index 57187093f28..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-2.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-3.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-3.dat Binary files differdeleted file mode 100644 index 6a516d38d17..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryMessage-3.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryReply.dat Binary files differdeleted file mode 100644 index 16b1e4bc4ef..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-DocumentSummaryReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-EmptyBucketsMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-EmptyBucketsMessage.dat Binary files differdeleted file mode 100644 index b9df278fd7a..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-EmptyBucketsMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-EmptyBucketsReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-EmptyBucketsReply.dat Binary files differdeleted file mode 100644 index 05510dd8c1e..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-EmptyBucketsReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketListMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketListMessage.dat Binary files differdeleted file mode 100644 index fa3de45ac5b..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketListMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketListReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketListReply.dat Binary files differdeleted file mode 100644 index 830994ed785..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketListReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketStateMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketStateMessage.dat Binary files differdeleted file mode 100644 index aa2d206ca3a..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketStateMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketStateReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketStateReply.dat Binary files differdeleted file mode 100644 index 0f91e3759f8..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-GetBucketStateReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-GetDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-GetDocumentMessage.dat Binary files differdeleted file mode 100644 index 3df64ed657f..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-GetDocumentMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-GetDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-GetDocumentReply.dat Binary files differdeleted file mode 100644 index c1ad7920a2e..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-GetDocumentReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-MapVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-MapVisitorMessage.dat Binary files differdeleted file mode 100644 index 1a8a837ea16..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-MapVisitorMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-MapVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-MapVisitorReply.dat Binary files differdeleted file mode 100644 index 541cd718a66..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-MapVisitorReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-MultiOperationMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-MultiOperationMessage.dat Binary files differdeleted file mode 100644 index 6efc54a6d09..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-MultiOperationMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-MultiOperationReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-MultiOperationReply.dat Binary files differdeleted file mode 100644 index 8ad31a95bd5..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-MultiOperationReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-PutDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-PutDocumentMessage.dat Binary files differdeleted file mode 100644 index 9e4a5c5f82f..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-PutDocumentMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-PutDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-PutDocumentReply.dat Binary files differdeleted file mode 100644 index 480544045bb..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-PutDocumentReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-1.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-1.dat Binary files differdeleted file mode 100644 index dbf830c9365..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-1.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-2.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-2.dat Binary files differdeleted file mode 100644 index 094143cf78d..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-2.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-3.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-3.dat Binary files differdeleted file mode 100644 index 3341d74052b..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-3.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-4.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-4.dat Binary files differdeleted file mode 100644 index 8aaaefff491..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-4.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-5.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-5.dat Binary files differdeleted file mode 100644 index e66ed1f07d4..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultMessage-5.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultReply.dat Binary files differdeleted file mode 100644 index 003f35d63a7..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-QueryResultReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveDocumentMessage.dat Binary files differdeleted file mode 100644 index 78b7972eb5c..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveDocumentMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveDocumentReply.dat Binary files differdeleted file mode 100644 index bf5db8761e2..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveDocumentReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessage.dat Binary files differdeleted file mode 100644 index 16850a6aff3..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessageGroup.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessageGroup.dat Binary files differdeleted file mode 100644 index abd648184d7..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessageGroup.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessageUser.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessageUser.dat Binary files differdeleted file mode 100644 index c2a63cb94c0..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationMessageUser.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationReply.dat Binary files differdeleted file mode 100644 index 752c4dba399..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-RemoveLocationReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-1.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-1.dat Binary files differdeleted file mode 100644 index 988f9fdab1f..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-1.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-2.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-2.dat Binary files differdeleted file mode 100644 index ac277d09643..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-2.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-3.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-3.dat Binary files differdeleted file mode 100644 index 03b49c8a0ac..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-3.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-4.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-4.dat Binary files differdeleted file mode 100644 index d52e574ea44..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-4.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-5.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-5.dat Binary files differdeleted file mode 100644 index e68654e9941..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultMessage-5.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultReply.dat Binary files differdeleted file mode 100644 index cce9c6f8d14..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-SearchResultReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-StatBucketMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-StatBucketMessage.dat Binary files differdeleted file mode 100644 index 1fc2b1cf3c1..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-StatBucketMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-StatBucketReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-StatBucketReply.dat Binary files differdeleted file mode 100644 index 0b98e240018..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-StatBucketReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-UpdateDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-UpdateDocumentMessage.dat Binary files differdeleted file mode 100644 index f1bca25ceca..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-UpdateDocumentMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-UpdateDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-UpdateDocumentReply.dat Binary files differdeleted file mode 100644 index c7151299366..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-UpdateDocumentReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-VisitorInfoMessage.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-VisitorInfoMessage.dat Binary files differdeleted file mode 100644 index 80b44e0c6fd..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-VisitorInfoMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-VisitorInfoReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-VisitorInfoReply.dat Binary files differdeleted file mode 100644 index 57a656c9b2d..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-VisitorInfoReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-cpp-WrongDistributionReply.dat b/documentapi/test/crosslanguagefiles/5.1-cpp-WrongDistributionReply.dat Binary files differdeleted file mode 100644 index 0dbe13225ae..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-cpp-WrongDistributionReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-BatchDocumentUpdateMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-BatchDocumentUpdateMessage.dat Binary files differdeleted file mode 100644 index 95e663088c3..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-BatchDocumentUpdateMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-BatchDocumentUpdateReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-BatchDocumentUpdateReply.dat Binary files differdeleted file mode 100644 index 216db17f80e..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-BatchDocumentUpdateReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-BatchMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-BatchMessage.dat Binary files differdeleted file mode 100644 index c1c43e8081b..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-BatchMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-BatchReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-BatchReply.dat Binary files differdeleted file mode 100644 index be3a9ba1913..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-BatchReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-CreateVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-CreateVisitorMessage.dat Binary files differdeleted file mode 100644 index 4ca8648e702..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-CreateVisitorMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-CreateVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-CreateVisitorReply.dat Binary files differdeleted file mode 100644 index e13917227d1..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-CreateVisitorReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-DestroyVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-DestroyVisitorMessage.dat Binary files differdeleted file mode 100644 index f39b31217e6..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-DestroyVisitorMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-DestroyVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-DestroyVisitorReply.dat Binary files differdeleted file mode 100644 index 1468f027b15..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-DestroyVisitorReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-DocumentIgnoredReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-DocumentIgnoredReply.dat Binary files differdeleted file mode 100644 index 15a7afe2a59..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-DocumentIgnoredReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-DocumentListMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-DocumentListMessage.dat Binary files differdeleted file mode 100644 index 2d8d12d2704..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-DocumentListMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-DocumentListReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-DocumentListReply.dat Binary files differdeleted file mode 100644 index c8a1cd888f0..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-DocumentListReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-DocumentSummaryReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-DocumentSummaryReply.dat Binary files differdeleted file mode 100644 index 16b1e4bc4ef..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-DocumentSummaryReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-EmptyBucketsMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-EmptyBucketsMessage.dat Binary files differdeleted file mode 100644 index b9df278fd7a..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-EmptyBucketsMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-EmptyBucketsReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-EmptyBucketsReply.dat Binary files differdeleted file mode 100644 index 05510dd8c1e..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-EmptyBucketsReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-GetBucketListMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-GetBucketListMessage.dat Binary files differdeleted file mode 100644 index fa3de45ac5b..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-GetBucketListMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-GetBucketListReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-GetBucketListReply.dat Binary files differdeleted file mode 100644 index 830994ed785..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-GetBucketListReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-GetBucketStateMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-GetBucketStateMessage.dat Binary files differdeleted file mode 100644 index aa2d206ca3a..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-GetBucketStateMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-GetBucketStateReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-GetBucketStateReply.dat Binary files differdeleted file mode 100644 index 0f91e3759f8..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-GetBucketStateReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-GetDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-GetDocumentMessage.dat Binary files differdeleted file mode 100644 index 3df64ed657f..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-GetDocumentMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-GetDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-GetDocumentReply.dat Binary files differdeleted file mode 100644 index c1ad7920a2e..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-GetDocumentReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-MapVisitorMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-MapVisitorMessage.dat Binary files differdeleted file mode 100644 index 1a8a837ea16..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-MapVisitorMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-MapVisitorReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-MapVisitorReply.dat Binary files differdeleted file mode 100644 index 541cd718a66..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-MapVisitorReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-MultiOperationMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-MultiOperationMessage.dat Binary files differdeleted file mode 100644 index 6efc54a6d09..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-MultiOperationMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-MultiOperationReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-MultiOperationReply.dat Binary files differdeleted file mode 100644 index 8ad31a95bd5..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-MultiOperationReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-PutDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-PutDocumentMessage.dat Binary files differdeleted file mode 100644 index 9e4a5c5f82f..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-PutDocumentMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-PutDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-PutDocumentReply.dat Binary files differdeleted file mode 100644 index 480544045bb..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-PutDocumentReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-QueryResultReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-QueryResultReply.dat Binary files differdeleted file mode 100644 index 003f35d63a7..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-QueryResultReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-RemoveDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-RemoveDocumentMessage.dat Binary files differdeleted file mode 100644 index 78b7972eb5c..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-RemoveDocumentMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-RemoveDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-RemoveDocumentReply.dat Binary files differdeleted file mode 100644 index bf5db8761e2..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-RemoveDocumentReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessage.dat Binary files differdeleted file mode 100644 index 16850a6aff3..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessageGroup.dat b/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessageGroup.dat Binary files differdeleted file mode 100644 index 9d7901a8b6e..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessageGroup.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessageUser.dat b/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessageUser.dat Binary files differdeleted file mode 100644 index c2a63cb94c0..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationMessageUser.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationReply.dat Binary files differdeleted file mode 100644 index 752c4dba399..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-RemoveLocationReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-SearchResultReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-SearchResultReply.dat Binary files differdeleted file mode 100644 index cce9c6f8d14..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-SearchResultReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-StatBucketMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-StatBucketMessage.dat Binary files differdeleted file mode 100644 index 1fc2b1cf3c1..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-StatBucketMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-StatBucketReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-StatBucketReply.dat Binary files differdeleted file mode 100644 index 0b98e240018..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-StatBucketReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-UpdateDocumentMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-UpdateDocumentMessage.dat Binary files differdeleted file mode 100644 index f1bca25ceca..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-UpdateDocumentMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-UpdateDocumentReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-UpdateDocumentReply.dat Binary files differdeleted file mode 100644 index c7151299366..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-UpdateDocumentReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-VisitorInfoMessage.dat b/documentapi/test/crosslanguagefiles/5.1-java-VisitorInfoMessage.dat Binary files differdeleted file mode 100644 index 80b44e0c6fd..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-VisitorInfoMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-VisitorInfoReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-VisitorInfoReply.dat Binary files differdeleted file mode 100644 index 57a656c9b2d..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-VisitorInfoReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.1-java-WrongDistributionReply.dat b/documentapi/test/crosslanguagefiles/5.1-java-WrongDistributionReply.dat Binary files differdeleted file mode 100644 index 0dbe13225ae..00000000000 --- a/documentapi/test/crosslanguagefiles/5.1-java-WrongDistributionReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-BatchDocumentUpdateMessage.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-BatchDocumentUpdateMessage.dat Binary files differdeleted file mode 100644 index 95e663088c3..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-BatchDocumentUpdateMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-cpp-BatchDocumentUpdateReply.dat b/documentapi/test/crosslanguagefiles/5.115-cpp-BatchDocumentUpdateReply.dat Binary files differdeleted file mode 100644 index 216db17f80e..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-cpp-BatchDocumentUpdateReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-BatchDocumentUpdateMessage.dat b/documentapi/test/crosslanguagefiles/5.115-java-BatchDocumentUpdateMessage.dat Binary files differdeleted file mode 100644 index 95e663088c3..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-BatchDocumentUpdateMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5.115-java-BatchDocumentUpdateReply.dat b/documentapi/test/crosslanguagefiles/5.115-java-BatchDocumentUpdateReply.dat Binary files differdeleted file mode 100644 index 216db17f80e..00000000000 --- a/documentapi/test/crosslanguagefiles/5.115-java-BatchDocumentUpdateReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-BatchDocumentUpdateMessage.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-BatchDocumentUpdateMessage.dat Binary files differdeleted file mode 100644 index 95e663088c3..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-BatchDocumentUpdateMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-BatchDocumentUpdateReply.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-BatchDocumentUpdateReply.dat Binary files differdeleted file mode 100644 index 216db17f80e..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-BatchDocumentUpdateReply.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryMessage-1.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-1.dat Binary files differindex 0107dd5f350..0107dd5f350 100644 --- a/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryMessage-1.dat +++ b/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-1.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryMessage-2.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-2.dat Binary files differindex 57187093f28..57187093f28 100644 --- a/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryMessage-2.dat +++ b/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-2.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryMessage-3.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-3.dat Binary files differindex 6a516d38d17..6a516d38d17 100644 --- a/documentapi/test/crosslanguagefiles/5-cpp-DocumentSummaryMessage-3.dat +++ b/documentapi/test/crosslanguagefiles/6.221-cpp-DocumentSummaryMessage-3.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-1.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-1.dat Binary files differindex dbf830c9365..dbf830c9365 100644 --- a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-1.dat +++ b/documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-1.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-2.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-2.dat Binary files differindex 094143cf78d..094143cf78d 100644 --- a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-2.dat +++ b/documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-2.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-3.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-3.dat Binary files differindex 3341d74052b..3341d74052b 100644 --- a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-3.dat +++ b/documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-3.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-4.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-4.dat Binary files differindex 8aaaefff491..8aaaefff491 100644 --- a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-4.dat +++ b/documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-4.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-5.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-5.dat Binary files differindex e66ed1f07d4..e66ed1f07d4 100644 --- a/documentapi/test/crosslanguagefiles/5-cpp-QueryResultMessage-5.dat +++ b/documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-5.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-1.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-1.dat Binary files differindex 988f9fdab1f..988f9fdab1f 100644 --- a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-1.dat +++ b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-1.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-2.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-2.dat Binary files differindex ac277d09643..ac277d09643 100644 --- a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-2.dat +++ b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-2.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-3.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-3.dat Binary files differindex 03b49c8a0ac..03b49c8a0ac 100644 --- a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-3.dat +++ b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-3.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-4.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-4.dat Binary files differindex d52e574ea44..d52e574ea44 100644 --- a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-4.dat +++ b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-4.dat diff --git a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-5.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-5.dat Binary files differindex e68654e9941..e68654e9941 100644 --- a/documentapi/test/crosslanguagefiles/5-cpp-SearchResultMessage-5.dat +++ b/documentapi/test/crosslanguagefiles/6.221-cpp-SearchResultMessage-5.dat diff --git a/documentapi/test/crosslanguagefiles/6.221-java-BatchDocumentUpdateMessage.dat b/documentapi/test/crosslanguagefiles/6.221-java-BatchDocumentUpdateMessage.dat Binary files differdeleted file mode 100644 index 95e663088c3..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-java-BatchDocumentUpdateMessage.dat +++ /dev/null diff --git a/documentapi/test/crosslanguagefiles/6.221-java-BatchDocumentUpdateReply.dat b/documentapi/test/crosslanguagefiles/6.221-java-BatchDocumentUpdateReply.dat Binary files differdeleted file mode 100644 index 216db17f80e..00000000000 --- a/documentapi/test/crosslanguagefiles/6.221-java-BatchDocumentUpdateReply.dat +++ /dev/null diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java index 252f6c5bd12..43ccd6d48a8 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/SimpleAdapterFactory.java @@ -48,23 +48,31 @@ public class SimpleAdapterFactory implements AdapterFactory { DocumentId docId = upd.getId(); Document complete = new Document(docType, upd.getId()); for (FieldPathUpdate fieldUpd : upd) { - if (FieldPathUpdateHelper.isComplete(fieldUpd)) { - // A 'complete' field path update is basically a regular top-level field update - // in wolf's clothing. Convert it to a regular field update to be friendlier - // towards the search core backend. - FieldPathUpdateHelper.applyUpdate(fieldUpd, complete); - } else { - ret.add(new IdentityFieldPathUpdateAdapter(fieldUpd, newDocumentAdapter(complete, true))); + try { + if (FieldPathUpdateHelper.isComplete(fieldUpd)) { + // A 'complete' field path update is basically a regular top-level field update + // in wolf's clothing. Convert it to a regular field update to be friendlier + // towards the search core backend. + FieldPathUpdateHelper.applyUpdate(fieldUpd, complete); + } else { + ret.add(new IdentityFieldPathUpdateAdapter(fieldUpd, newDocumentAdapter(complete, true))); + } + } catch (NullPointerException e) { + throw new IllegalArgumentException("Exception during handling of update '" + fieldUpd + "' to field '" + fieldUpd.getFieldPath() + "'", e); } } for (FieldUpdate fieldUpd : upd.getFieldUpdates()) { Field field = fieldUpd.getField(); for (ValueUpdate valueUpd : fieldUpd.getValueUpdates()) { - if (FieldUpdateHelper.isComplete(field, valueUpd)) { - FieldUpdateHelper.applyUpdate(field, valueUpd, complete); - } else { - Document partial = FieldUpdateHelper.newPartialDocument(docType, docId, field, valueUpd); - ret.add(FieldUpdateAdapter.fromPartialUpdate(expressionSelector.selectExpression(docType, field.getName()),newDocumentAdapter(partial, true), valueUpd)); + try { + if (FieldUpdateHelper.isComplete(field, valueUpd)) { + FieldUpdateHelper.applyUpdate(field, valueUpd, complete); + } else { + Document partial = FieldUpdateHelper.newPartialDocument(docType, docId, field, valueUpd); + ret.add(FieldUpdateAdapter.fromPartialUpdate(expressionSelector.selectExpression(docType, field.getName()), newDocumentAdapter(partial, true), valueUpd)); + } + } catch (NullPointerException e) { + throw new IllegalArgumentException("Exception during handling of update '" + valueUpd + "' to field '" + field + "'", e); } } } diff --git a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JerseyJaxRsClientFactory.java b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JerseyJaxRsClientFactory.java index 5a3ccfed490..9321f8e290d 100644 --- a/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JerseyJaxRsClientFactory.java +++ b/jaxrs_client_utils/src/main/java/com/yahoo/vespa/jaxrs/client/JerseyJaxRsClientFactory.java @@ -8,6 +8,7 @@ import org.glassfish.jersey.client.proxy.WebResourceFactory; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLContext; +import javax.ws.rs.client.Client; import javax.ws.rs.client.ClientBuilder; import javax.ws.rs.client.ClientRequestFilter; import javax.ws.rs.client.WebTarget; @@ -16,17 +17,17 @@ import javax.ws.rs.core.UriBuilder; import java.util.Collections; /** - * @author bakksjo + * Factory for creating Jersey clients from a JAX-RS resource interface. + * + * @author Oyvind Bakksjo */ public class JerseyJaxRsClientFactory implements JaxRsClientFactory { + private static final int DEFAULT_CONNECT_TIMEOUT_MS = 30000; private static final int DEFAULT_READ_TIMEOUT_MS = 30000; - private final int connectTimeoutMs; - private final int readTimeoutMs; - private final SSLContext sslContext; - private final String userAgent; - private final HostnameVerifier hostnameVerifier; + // Client is a heavy-weight object with a finalizer so we create only one and re-use it + private final Client client; public JerseyJaxRsClientFactory() { this(DEFAULT_CONNECT_TIMEOUT_MS, DEFAULT_READ_TIMEOUT_MS); @@ -36,33 +37,23 @@ public class JerseyJaxRsClientFactory implements JaxRsClientFactory { this(DEFAULT_CONNECT_TIMEOUT_MS, DEFAULT_READ_TIMEOUT_MS, sslContext, hostnameVerifier, userAgent); } - public JerseyJaxRsClientFactory(final int connectTimeoutMs, final int readTimeoutMs) { + public JerseyJaxRsClientFactory(int connectTimeoutMs, int readTimeoutMs) { this(connectTimeoutMs, readTimeoutMs, null, null, null); } public JerseyJaxRsClientFactory(int connectTimeoutMs, int readTimeoutMs, SSLContext sslContext, HostnameVerifier hostnameVerifier, String userAgent) { - this.connectTimeoutMs = connectTimeoutMs; - this.readTimeoutMs = readTimeoutMs; - this.sslContext = sslContext; - this.hostnameVerifier = hostnameVerifier; - this.userAgent = userAgent; - } - - /** - * Contains some workarounds for HTTP/JAX-RS/Jersey issues. See: - * https://jersey.java.net/apidocs/latest/jersey/org/glassfish/jersey/client/ClientProperties.html#SUPPRESS_HTTP_COMPLIANCE_VALIDATION - * https://jersey.java.net/apidocs/latest/jersey/org/glassfish/jersey/client/HttpUrlConnectorProvider.html#SET_METHOD_WORKAROUND - */ - @Override - public <T> T createClient(final Class<T> apiClass, final HostName hostName, final int port, final String pathPrefix, String scheme) { - final UriBuilder uriBuilder = UriBuilder.fromPath(pathPrefix).host(hostName.s()).port(port).scheme(scheme); + /* + * Configure client with some workarounds for HTTP/JAX-RS/Jersey issues. See: + * https://jersey.java.net/apidocs/latest/jersey/org/glassfish/jersey/client/ClientProperties.html#SUPPRESS_HTTP_COMPLIANCE_VALIDATION + * https://jersey.java.net/apidocs/latest/jersey/org/glassfish/jersey/client/HttpUrlConnectorProvider.html#SET_METHOD_WORKAROUND + */ ClientBuilder builder = ClientBuilder.newBuilder() - .property(ClientProperties.CONNECT_TIMEOUT, connectTimeoutMs) - .property(ClientProperties.READ_TIMEOUT, readTimeoutMs) - .property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true) // Allow empty PUT. TODO: Fix API. - .property(HttpUrlConnectorProvider.SET_METHOD_WORKAROUND, true) // Allow e.g. PATCH method. - .property(ClientProperties.FOLLOW_REDIRECTS, true); + .property(ClientProperties.CONNECT_TIMEOUT, connectTimeoutMs) + .property(ClientProperties.READ_TIMEOUT, readTimeoutMs) + .property(ClientProperties.SUPPRESS_HTTP_COMPLIANCE_VALIDATION, true) // Allow empty PUT. TODO: Fix API. + .property(HttpUrlConnectorProvider.SET_METHOD_WORKAROUND, true) // Allow e.g. PATCH method. + .property(ClientProperties.FOLLOW_REDIRECTS, true); if (sslContext != null) { builder.sslContext(sslContext); } @@ -70,11 +61,16 @@ public class JerseyJaxRsClientFactory implements JaxRsClientFactory { builder.hostnameVerifier(hostnameVerifier); } if (userAgent != null) { - builder.register((ClientRequestFilter) context -> - context.getHeaders().put(HttpHeaders.USER_AGENT, Collections.singletonList(userAgent))); + builder.register((ClientRequestFilter) context -> context.getHeaders().put(HttpHeaders.USER_AGENT, Collections.singletonList(userAgent))); } - final WebTarget target = builder.build().target(uriBuilder); - // TODO: Check if this fills up non-heap memory with loaded classes. + this.client = builder.build(); + } + + @Override + public <T> T createClient(Class<T> apiClass, HostName hostName, int port, String pathPrefix, String scheme) { + UriBuilder uriBuilder = UriBuilder.fromPath(pathPrefix).host(hostName.s()).port(port).scheme(scheme); + WebTarget target = client.target(uriBuilder); return WebResourceFactory.newResource(apiClass, target); } + } diff --git a/jdisc_core/pom.xml b/jdisc_core/pom.xml index efa4e62affd..1a0c27c71a6 100644 --- a/jdisc_core/pom.xml +++ b/jdisc_core/pom.xml @@ -232,6 +232,7 @@ <configuration> <mainClass>com.yahoo.jdisc.core.ExportPackages</mainClass> <classpathScope>test</classpathScope> + <cleanupDaemonThreads>false</cleanupDaemonThreads> <arguments> <argument>${exportPackagesFile}</argument> <argument>${project.build.directory}/dependency/commons-daemon.jar</argument> diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java index 86ab016bded..e8e6373f2ed 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java @@ -1,20 +1,31 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.core; -import org.apache.felix.framework.util.Util; +import com.yahoo.io.IOUtils; +import com.yahoo.yolean.Exceptions; +import org.apache.felix.framework.Felix; +import org.osgi.framework.BundleException; import org.osgi.framework.Constants; +import org.osgi.framework.launch.Framework; import java.io.File; import java.io.FileInputStream; import java.io.FileWriter; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; import java.util.jar.JarInputStream; +import java.util.logging.Logger; /** * @author Simon Thoresen Hult + * @author gjoranv */ public class ExportPackages { + private static final Logger log = Logger.getLogger(ExportPackages.class.getName()); public static final String PROPERTIES_FILE = "/exportPackages.properties"; public static final String EXPORT_PACKAGES = "exportPackages"; @@ -75,7 +86,38 @@ public class ExportPackages { } public static String getSystemPackages() { - return Util.getDefaultProperty(null, "org.osgi.framework.system.packages"); + File cache; + try { + cache = Files.createTempDirectory("felix-cache").toAbsolutePath().toFile(); + } catch (IOException e) { + throw new RuntimeException("Could not create temp bundle-cache.", e); + } + Framework framework = new Felix(felixCacheParams(cache.getAbsolutePath())); + try { + framework.init(); + framework.start(); + return framework.getHeaders().get(Constants.EXPORT_PACKAGE); + } catch (BundleException e) { + throw new RuntimeException("Failed retrieving exported system packages. ", e); + } finally { + try { + framework.stop(); + framework.waitForStop(10000); + } catch (BundleException | InterruptedException e) { + log.warning("Failed to stop Felix framework:\n" + Exceptions.toMessageString(e)); + } + if (! IOUtils.recursiveDeleteDir(cache)) { + log.warning("Failed to delete temp dir, must be deleted manually: " + cache.getAbsolutePath()); + } + } + } + + private static Map<String, Object> felixCacheParams(String cache) { + Map<String, Object> params = new HashMap<>(); + params.put("felix.cache.profiledir", cache); + params.put("felix.cache.dir", cache); + params.put(Constants.FRAMEWORK_STORAGE, cache); + return params; } private static String getExportedPackages(String argument) throws IOException { @@ -92,4 +134,5 @@ public class ExportPackages { return jar.getManifest().getMainAttributes().getValue(Constants.EXPORT_PACKAGE); } } + } diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/core/FelixParamsTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/FelixParamsTestCase.java index 55fca97841a..6836ebde496 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/core/FelixParamsTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/core/FelixParamsTestCase.java @@ -29,17 +29,19 @@ public class FelixParamsTestCase { @Test public void requireThatSystemPackagesAreNotReplaced() { + String systemPackages = ExportPackages.getSystemPackages(); + FelixParams params = new FelixParams(); Map<String, String> config = params.toConfig(); assertNotNull(config); String str = config.get(Constants.FRAMEWORK_SYSTEMPACKAGES); assertNotNull(str); - assertTrue(str.contains(ExportPackages.getSystemPackages())); + assertTrue(str.contains(systemPackages)); params.exportPackage("foo"); assertNotNull(config = params.toConfig()); assertNotNull(str = config.get(Constants.FRAMEWORK_SYSTEMPACKAGES)); - assertTrue(str.contains(ExportPackages.getSystemPackages())); + assertTrue(str.contains(systemPackages)); assertTrue(str.contains("foo")); } diff --git a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/application/BundleActivatorIntegrationTest.java b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/application/BundleActivatorIntegrationTest.java index 6388643148c..03e56d7c26d 100644 --- a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/application/BundleActivatorIntegrationTest.java +++ b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/application/BundleActivatorIntegrationTest.java @@ -22,7 +22,7 @@ public class BundleActivatorIntegrationTest { OsgiFramework osgi = driver.osgiFramework(); BundleInstaller installer = new BundleInstaller(driver.osgiFramework()); Bundle bundle = installer.installAndStart("my-bundle-activator.jar").get(0); - Class<?> serviceClass = bundle.loadClass("com.yahoo.jdisc.bundle.MyService"); + Class<?> serviceClass = bundle.loadClass("com.yahoo.jdisc.bundle.my_act.MyService"); assertNotNull(serviceClass); BundleContext ctx = osgi.bundleContext(); ServiceReference<?> serviceRef = ctx.getServiceReference(serviceClass.getName()); @@ -37,7 +37,7 @@ public class BundleActivatorIntegrationTest { public void requireThatApplicationBundleActivatorHasAccessToCurrentContainer() throws Exception { TestDriver driver = TestDriver.newApplicationBundleInstance("app-g-act.jar", false); OsgiFramework osgi = driver.osgiFramework(); - Class<?> serviceClass = osgi.bundles().get(1).loadClass("com.yahoo.jdisc.bundle.MyService"); + Class<?> serviceClass = osgi.bundles().get(1).loadClass("com.yahoo.jdisc.bundle.g_act.MyService"); assertNotNull(serviceClass); BundleContext ctx = osgi.bundleContext(); ServiceReference<?> serviceRef = ctx.getServiceReference(serviceClass.getName()); diff --git a/jdisc_core_test/test_bundles/app-g-act/pom.xml b/jdisc_core_test/test_bundles/app-g-act/pom.xml index b91b95f9fdd..aaa740efade 100644 --- a/jdisc_core_test/test_bundles/app-g-act/pom.xml +++ b/jdisc_core_test/test_bundles/app-g-act/pom.xml @@ -23,13 +23,13 @@ <configuration> <instructions> <X-JDisc-Application> - com.yahoo.jdisc.bundle.ApplicationG + com.yahoo.jdisc.bundle.g_act.ApplicationG </X-JDisc-Application> <Bundle-Activator> - com.yahoo.jdisc.bundle.MyBundleActivator + com.yahoo.jdisc.bundle.g_act.MyBundleActivator </Bundle-Activator> <Export-Package> - com.yahoo.jdisc.bundle + com.yahoo.jdisc.bundle.g_act </Export-Package> </instructions> </configuration> diff --git a/jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/ApplicationG.java b/jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/g_act/ApplicationG.java index 9b4d7501ac0..02a9cd5a542 100644 --- a/jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/ApplicationG.java +++ b/jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/g_act/ApplicationG.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.bundle; +package com.yahoo.jdisc.bundle.g_act; import com.yahoo.jdisc.application.Application; diff --git a/jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/MyBundleActivator.java b/jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/g_act/MyBundleActivator.java index 51a95b8e3a9..c60f4939409 100644 --- a/jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/MyBundleActivator.java +++ b/jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/g_act/MyBundleActivator.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.bundle; +package com.yahoo.jdisc.bundle.g_act; import com.yahoo.jdisc.service.CurrentContainer; import org.osgi.framework.BundleActivator; diff --git a/jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/MyService.java b/jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/g_act/MyService.java index a5cb9be9a47..518f7304100 100644 --- a/jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/MyService.java +++ b/jdisc_core_test/test_bundles/app-g-act/src/main/java/com/yahoo/jdisc/bundle/g_act/MyService.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.bundle; +package com.yahoo.jdisc.bundle.g_act; import com.yahoo.jdisc.service.AbstractServerProvider; import com.yahoo.jdisc.service.CurrentContainer; diff --git a/jdisc_core_test/test_bundles/my-bundle-activator/pom.xml b/jdisc_core_test/test_bundles/my-bundle-activator/pom.xml index 531af08b915..8931c26a73e 100644 --- a/jdisc_core_test/test_bundles/my-bundle-activator/pom.xml +++ b/jdisc_core_test/test_bundles/my-bundle-activator/pom.xml @@ -23,10 +23,10 @@ <configuration> <instructions> <Bundle-Activator> - com.yahoo.jdisc.bundle.MyBundleActivator + com.yahoo.jdisc.bundle.my_act.MyBundleActivator </Bundle-Activator> <Export-Package> - com.yahoo.jdisc.bundle + com.yahoo.jdisc.bundle.my_act </Export-Package> </instructions> </configuration> diff --git a/jdisc_core_test/test_bundles/my-bundle-activator/src/main/java/com/yahoo/jdisc/bundle/MyBundleActivator.java b/jdisc_core_test/test_bundles/my-bundle-activator/src/main/java/com/yahoo/jdisc/bundle/my_act/MyBundleActivator.java index b4af03bdf60..4212fc2fd09 100644 --- a/jdisc_core_test/test_bundles/my-bundle-activator/src/main/java/com/yahoo/jdisc/bundle/MyBundleActivator.java +++ b/jdisc_core_test/test_bundles/my-bundle-activator/src/main/java/com/yahoo/jdisc/bundle/my_act/MyBundleActivator.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.bundle; +package com.yahoo.jdisc.bundle.my_act; import com.yahoo.jdisc.service.CurrentContainer; import org.osgi.framework.BundleActivator; diff --git a/jdisc_core_test/test_bundles/my-bundle-activator/src/main/java/com/yahoo/jdisc/bundle/MyService.java b/jdisc_core_test/test_bundles/my-bundle-activator/src/main/java/com/yahoo/jdisc/bundle/my_act/MyService.java index 8eb1543776d..b24ee137f59 100644 --- a/jdisc_core_test/test_bundles/my-bundle-activator/src/main/java/com/yahoo/jdisc/bundle/MyService.java +++ b/jdisc_core_test/test_bundles/my-bundle-activator/src/main/java/com/yahoo/jdisc/bundle/my_act/MyService.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.bundle; +package com.yahoo.jdisc.bundle.my_act; /** * @author Simon Thoresen Hult diff --git a/jrt/src/com/yahoo/jrt/CryptoEngine.java b/jrt/src/com/yahoo/jrt/CryptoEngine.java index 2ef936ec7ed..c27aba73873 100644 --- a/jrt/src/com/yahoo/jrt/CryptoEngine.java +++ b/jrt/src/com/yahoo/jrt/CryptoEngine.java @@ -2,10 +2,10 @@ package com.yahoo.jrt; -import com.yahoo.security.tls.TransportSecurityOptions; +import com.yahoo.security.tls.TransportSecurityUtils; +import com.yahoo.security.tls.TransportSecurityUtils.MixedMode; import java.nio.channels.SocketChannel; -import java.nio.file.Paths; /** @@ -16,12 +16,22 @@ import java.nio.file.Paths; **/ public interface CryptoEngine { public CryptoSocket createCryptoSocket(SocketChannel channel, boolean isServer); - static public CryptoEngine createDefault() { // TODO Move this logic to a dedicated factory class - String tlsConfigParameter = System.getenv("VESPA_TLS_CONFIG_FILE"); - if (tlsConfigParameter != null && !tlsConfigParameter.isEmpty()) { - return new TlsCryptoEngine(TransportSecurityOptions.fromJsonFile(Paths.get(tlsConfigParameter))); - } else { + static public CryptoEngine createDefault() { + if (!TransportSecurityUtils.isTransportSecurityEnabled()) { return new NullCryptoEngine(); } + TlsCryptoEngine tlsCryptoEngine = new TlsCryptoEngine(TransportSecurityUtils.getOptions().get()); + if (!TransportSecurityUtils.isInsecureMixedModeEnabled()) { + return tlsCryptoEngine; + } + MixedMode mixedMode = TransportSecurityUtils.getInsecureMixedMode().get(); + switch (mixedMode) { + case PLAINTEXT_CLIENT_MIXED_SERVER: + return new MaybeTlsCryptoEngine(tlsCryptoEngine, false); + case TLS_CLIENT_MIXED_SERVER: + return new MaybeTlsCryptoEngine(tlsCryptoEngine, true); + default: + throw new IllegalArgumentException(mixedMode.toString()); + } } } diff --git a/jrt/src/com/yahoo/jrt/MaybeTlsCryptoEngine.java b/jrt/src/com/yahoo/jrt/MaybeTlsCryptoEngine.java index 8cb560246e8..6664934799c 100644 --- a/jrt/src/com/yahoo/jrt/MaybeTlsCryptoEngine.java +++ b/jrt/src/com/yahoo/jrt/MaybeTlsCryptoEngine.java @@ -7,28 +7,28 @@ import java.nio.channels.SocketChannel; * A crypto engine that supports both tls encrypted connections and * unencrypted connections. The use of tls for incoming connections is * auto-detected using clever heuristics. The use of tls for outgoing - * connections is controlled by the useTls flag given to the + * connections is controlled by the useTlsWhenClient flag given to the * constructor. **/ public class MaybeTlsCryptoEngine implements CryptoEngine { private final TlsCryptoEngine tlsEngine; - private final boolean useTls; + private final boolean useTlsWhenClient; - public MaybeTlsCryptoEngine(TlsCryptoEngine tlsEngine, boolean useTls) { + public MaybeTlsCryptoEngine(TlsCryptoEngine tlsEngine, boolean useTlsWhenClient) { this.tlsEngine = tlsEngine; - this.useTls = useTls; + this.useTlsWhenClient = useTlsWhenClient; } @Override public CryptoSocket createCryptoSocket(SocketChannel channel, boolean isServer) { if (isServer) { return new MaybeTlsCryptoSocket(channel, tlsEngine); - } else if (useTls) { + } else if (useTlsWhenClient) { return tlsEngine.createCryptoSocket(channel, false); } else { return new NullCryptoSocket(channel); } } - @Override public String toString() { return "MaybeTlsCryptoEngine(useTls:" + useTls + ")"; } + @Override public String toString() { return "MaybeTlsCryptoEngine(useTlsWhenClient:" + useTlsWhenClient + ")"; } } diff --git a/jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java b/jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java index 7cedbcda9a1..1cf3dfd1261 100644 --- a/jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java +++ b/jrt/src/com/yahoo/jrt/MaybeTlsCryptoSocket.java @@ -16,13 +16,13 @@ import java.nio.channels.SocketChannel; **/ public class MaybeTlsCryptoSocket implements CryptoSocket { - private static final int snoop_size = 9; + private static final int SNOOP_SIZE = 9; private CryptoSocket socket; // 'data' is the first 9 bytes received from the client public static boolean looksLikeTlsToMe(byte[] data) { - if (data.length != snoop_size) { + if (data.length != SNOOP_SIZE) { return false; // wrong data size for tls detection } if (data[0] != 22) { @@ -67,13 +67,13 @@ public class MaybeTlsCryptoSocket implements CryptoSocket { @Override public HandshakeResult handshake() throws IOException { if (factory != null) { - channel().read(buffer.getWritable(snoop_size)); - if (buffer.bytes() < snoop_size) { + channel().read(buffer.getWritable(SNOOP_SIZE)); + if (buffer.bytes() < SNOOP_SIZE) { return HandshakeResult.NEED_READ; } - byte[] data = new byte[snoop_size]; + byte[] data = new byte[SNOOP_SIZE]; ByteBuffer src = buffer.getReadable(); - for (int i = 0; i < snoop_size; i++) { + for (int i = 0; i < SNOOP_SIZE; i++) { data[i] = src.get(i); } if (looksLikeTlsToMe(data)) { diff --git a/jrt/tests/com/yahoo/jrt/TlsDetectionTest.java b/jrt/tests/com/yahoo/jrt/TlsDetectionTest.java index 9bd37e25772..47a8a20deab 100644 --- a/jrt/tests/com/yahoo/jrt/TlsDetectionTest.java +++ b/jrt/tests/com/yahoo/jrt/TlsDetectionTest.java @@ -1,20 +1,18 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jrt; +import static org.junit.Assert.assertEquals; + public class TlsDetectionTest { - static private String message(byte[] data, boolean actual) { - String msg = "["; + static private String message(byte[] data) { + String msg = "isTls(["; String delimiter = ""; for (byte b: data) { msg += delimiter + (b & 0xff); delimiter = ", "; } - if (actual) { - msg += "] wrongfully detected as tls"; - } else { - msg += "] wrongfully rejected as not tls"; - } + msg += "])"; return msg; } @@ -23,10 +21,7 @@ public class TlsDetectionTest { for (int i = 0; i < data.length; i++) { data[i] = (byte) values[i]; } - boolean actual = MaybeTlsCryptoSocket.looksLikeTlsToMe(data); - if(actual != expect) { - throw new AssertionError(message(data, actual)); - } + assertEquals(message(data), expect, MaybeTlsCryptoSocket.looksLikeTlsToMe(data)); } @org.junit.Test public void testValidHandshake() { diff --git a/metrics/src/vespa/metrics/countmetric.h b/metrics/src/vespa/metrics/countmetric.h index dd4c9e6fc92..912fc52449b 100644 --- a/metrics/src/vespa/metrics/countmetric.h +++ b/metrics/src/vespa/metrics/countmetric.h @@ -78,11 +78,6 @@ public: void set(T value); void inc(T value = 1); void dec(T value = 1); - CountMetric & operator++() { inc(); return *this; } - CountMetric & operator--() { dec(); return *this; } - - CountMetric operator++(int); - CountMetric operator--(int); CountMetric & operator+=(const CountMetric &); CountMetric & operator-=(const CountMetric &); diff --git a/metrics/src/vespa/metrics/countmetric.hpp b/metrics/src/vespa/metrics/countmetric.hpp index 516f6c660b6..e150ce0d632 100644 --- a/metrics/src/vespa/metrics/countmetric.hpp +++ b/metrics/src/vespa/metrics/countmetric.hpp @@ -75,22 +75,6 @@ CountMetric<T, SumOnAdd>::operator-=(const CountMetric<T, SumOnAdd>& other) } template <typename T, bool SumOnAdd> -CountMetric<T, SumOnAdd> -CountMetric<T, SumOnAdd>:: operator++(int) { - CountMetric tmp(*this); - inc(); - return tmp; -} - -template <typename T, bool SumOnAdd> -CountMetric<T, SumOnAdd> -CountMetric<T, SumOnAdd>::operator--(int) { - CountMetric tmp(*this); - inc(); - return tmp; -} - -template <typename T, bool SumOnAdd> void CountMetric<T, SumOnAdd>::set(T value) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerNetworking.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerNetworking.java index 7678ad8169a..7f0ade2e500 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerNetworking.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerNetworking.java @@ -28,7 +28,7 @@ public enum DockerNetworking { } public static DockerNetworking from(String cloud, NodeType nodeType, boolean hostAdmin) { - if (cloud.equals("AWS")) { + if (cloud.equalsIgnoreCase("aws")) { return DockerNetworking.NPT; } else if (nodeType == NodeType.confighost || nodeType == NodeType.proxyhost) { return DockerNetworking.HOST_NETWORK; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java index 4a19e5fe215..6d1d51ead43 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperations.java @@ -30,18 +30,21 @@ public interface DockerOperations { ProcessResult executeCommandInNetworkNamespace(ContainerName containerName, String... command); - void resumeNode(ContainerName containerName); - - void restartVespaOnNode(ContainerName containerName); - void stopServicesOnNode(ContainerName containerName); + /** Resume node. Resuming a node means that it is ready to take on traffic. */ + void resumeNode(ContainerName containerName); /** - * Try to suspend node. Suspending a node means the node should be taken offline, - * such that maintenance can be done of the node (upgrading, rebooting, etc), - * and such that we will start serving again as soon as possible afterwards. + * Suspend node. Suspending a node means the node should be taken temporarly offline, + * such that maintenance of the node can be done (upgrading, rebooting, etc). */ - void trySuspendNode(ContainerName containerName); + void suspendNode(ContainerName containerName); + + void restartVespa(ContainerName containerName); + + void startServices(ContainerName containerName); + + void stopServices(ContainerName containerName); Optional<ContainerStats> getContainerStats(ContainerName containerName); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index 45c2e93c93e..7809f263ad9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -13,7 +13,6 @@ import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.DockerImage; import com.yahoo.vespa.hosted.dockerapi.DockerNetworkCreator; import com.yahoo.vespa.hosted.dockerapi.ProcessResult; -import com.yahoo.vespa.hosted.dockerapi.exception.ContainerNotFoundException; import com.yahoo.vespa.hosted.node.admin.component.Environment; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.nodeagent.ContainerData; @@ -292,29 +291,26 @@ public class DockerOperationsImpl implements DockerOperations { } @Override - public void restartVespaOnNode(ContainerName containerName) { + public void suspendNode(ContainerName containerName) { + executeCommandInContainer(containerName, nodeProgram, "suspend"); + } + + @Override + public void restartVespa(ContainerName containerName) { executeCommandInContainer(containerName, nodeProgram, "restart-vespa"); } @Override - public void stopServicesOnNode(ContainerName containerName) { - executeCommandInContainer(containerName, nodeProgram, "stop"); + public void startServices(ContainerName containerName) { + executeCommandInContainer(containerName, nodeProgram, "start"); } @Override - public void trySuspendNode(ContainerName containerName) { - try { - executeCommandInContainer(containerName, nodeProgram, "suspend"); - } catch (ContainerNotFoundException e) { - throw e; - } catch (RuntimeException e) { - PrefixLogger logger = PrefixLogger.getNodeAgentLogger(DockerOperationsImpl.class, containerName); - // It's bad to continue as-if nothing happened, but on the other hand if we do not proceed to - // remove container, we will not be able to upgrade to fix any problems in the suspend logic! - logger.warning("Failed trying to suspend container " + containerName.asString(), e); - } + public void stopServices(ContainerName containerName) { + executeCommandInContainer(containerName, nodeProgram, "stop"); } + @Override public Optional<ContainerStats> getContainerStats(ContainerName containerName) { return docker.getContainerStats(containerName); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index ba8a2e55587..12c1b9bcf11 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -145,7 +145,10 @@ public class NodeAdminImpl implements NodeAdmin { // Each container may spend 1-1:30 minutes stopping nodeAgentsByHostname.values().parallelStream() .filter(nodeAgent -> hostnames.contains(nodeAgent.getHostname())) - .forEach(NodeAgent::stopServices); + .forEach(nodeAgent -> { + nodeAgent.suspend(); + nodeAgent.stopServices(); + }); } public int getNumberOfNodeAgents() { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java index 92c44969d5e..9b759b208eb 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgent.java @@ -18,9 +18,18 @@ public interface NodeAgent { */ boolean setFrozen(boolean frozen); + /** + * Stop services running on node. Depending on the state of the node, {@link #suspend()} might need to be + * called before calling this method. + */ void stopServices(); /** + * Suspend node. Take node offline (e.g. take node out of VIP, drain traffic, prepare for restart etc.) + */ + void suspend(); + + /** * Returns a map containing all relevant NodeAgent variables and their current values. */ Map<String, Object> debugInfo(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index ad38306547d..36a2672b1a6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -88,7 +88,8 @@ public class NodeAgentImpl implements NodeAgent { private Consumer<String> serviceRestarter; private Optional<Future<?>> currentFilebeatRestarter = Optional.empty(); - private boolean resumeScriptRun = false; + private boolean hasResumedNode = false; + private boolean hasStartedServices = false; /** * ABSENT means container is definitely absent - A container that was absent will not suddenly appear without @@ -216,17 +217,26 @@ public class NodeAgentImpl implements NodeAgent { */ protected void verifyHealth(NodeSpec node) { } - void runLocalResumeScriptIfNeeded(NodeSpec node) { - if (! resumeScriptRun) { - storageMaintainer.writeMetricsConfig(containerName, node); - storageMaintainer.writeFilebeatConfig(containerName, node); - stopFilebeatSchedulerIfNeeded(); - currentFilebeatRestarter = Optional.of(filebeatRestarter.scheduleWithFixedDelay( - () -> serviceRestarter.accept("filebeat"), 1, 1, TimeUnit.DAYS)); + void startServicesIfNeeded() { + if (!hasStartedServices) { + logger.info("Starting services"); + dockerOperations.startServices(containerName); + hasStartedServices = true; + } + } + + void resumeNodeIfNeeded(NodeSpec node) { + if (!hasResumedNode) { + if (!currentFilebeatRestarter.isPresent()) { + storageMaintainer.writeMetricsConfig(containerName, node); + storageMaintainer.writeFilebeatConfig(containerName, node); + currentFilebeatRestarter = Optional.of(filebeatRestarter.scheduleWithFixedDelay( + () -> serviceRestarter.accept("filebeat"), 1, 1, TimeUnit.DAYS)); + } logger.debug("Starting optional node program resume command"); dockerOperations.resumeNode(containerName); - resumeScriptRun = true; + hasResumedNode = true; } } @@ -260,7 +270,8 @@ public class NodeAgentImpl implements NodeAgent { dockerOperations.startContainer(containerName); lastCpuMetric = new CpuUsageReporter(); - resumeScriptRun = false; + hasStartedServices = true; // Automatically started with the container + hasResumedNode = false; logger.info("Container successfully started, new containerState is " + containerState); } @@ -293,7 +304,7 @@ public class NodeAgentImpl implements NodeAgent { logger.info("Restarting services"); // Since we are restarting the services we need to suspend the node. orchestratorSuspendNode(); - dockerOperations.restartVespaOnNode(containerName); + dockerOperations.restartVespa(containerName); } } @@ -302,13 +313,29 @@ public class NodeAgentImpl implements NodeAgent { logger.info("Stopping services"); if (containerState == ABSENT) return; try { - dockerOperations.trySuspendNode(containerName); - dockerOperations.stopServicesOnNode(containerName); + hasStartedServices = hasResumedNode = false; + dockerOperations.stopServices(containerName); } catch (ContainerNotFoundException e) { containerState = ABSENT; } } + @Override + public void suspend() { + logger.info("Suspending services on node"); + if (containerState == ABSENT) return; + try { + hasResumedNode = false; + dockerOperations.suspendNode(containerName); + } catch (ContainerNotFoundException e) { + containerState = ABSENT; + } catch (RuntimeException e) { + // It's bad to continue as-if nothing happened, but on the other hand if we do not proceed to + // remove container, we will not be able to upgrade to fix any problems in the suspend logic! + logger.warning("Failed trying to suspend container " + containerName.asString(), e); + } + } + private Optional<String> shouldRemoveContainer(NodeSpec node, Container existingContainer) { final Node.State nodeState = node.getState(); if (nodeState == Node.State.dirty || nodeState == Node.State.provisioned) { @@ -344,6 +371,9 @@ public class NodeAgentImpl implements NodeAgent { } try { + if (node.getState() != Node.State.dirty) { + suspend(); + } stopServices(); } catch (Exception e) { logger.info("Failed stopping services, ignoring", e); @@ -484,7 +514,8 @@ public class NodeAgentImpl implements NodeAgent { } verifyHealth(node); - runLocalResumeScriptIfNeeded(node); + startServicesIfNeeded(); + resumeNodeIfNeeded(node); athenzCredentialsMaintainer.converge(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java index 319207f9e95..a46defc991b 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/RunInContainerTest.java @@ -194,9 +194,9 @@ public class RunInContainerTest { // Allow stopping services in active nodes doNothing().when(dockerOperationsMock) - .trySuspendNode(eq(new ContainerName("host1"))); + .suspendNode(eq(new ContainerName("host1"))); doNothing().when(dockerOperationsMock) - .stopServicesOnNode(eq(new ContainerName("host1"))); + .stopServices(eq(new ContainerName("host1"))); assertTrue(verifyWithRetries("suspend", false)); assertTrue(verifyWithRetries("suspend", true)); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java index 56373dda2f8..9d5555e1f92 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java @@ -49,13 +49,13 @@ public class AclMaintainerTest { @Before public void before() { when(dockerOperations.getAllManagedContainers()).thenReturn(containerList); - when(env.getCloud()).thenReturn("AWS"); + when(env.getCloud()).thenReturn("aws"); when(env.getDockerNetworking()).thenReturn(DockerNetworking.NPT); } @Test public void no_redirect_in_yahoo() { - when(env.getCloud()).thenReturn("YAHOO"); + when(env.getCloud()).thenReturn("yahoo"); when(env.getDockerNetworking()).thenReturn(DockerNetworking.MACVLAN); Container container = addContainer("container1", "container1.host.com", Container.State.RUNNING); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index e0031f9b9b3..635b50246f4 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -131,6 +131,7 @@ public class NodeAgentImplTest { final InOrder inOrder = inOrder(dockerOperations, orchestrator, nodeRepository); // TODO: Verify this isn't run unless 1st time + inOrder.verify(dockerOperations, times(1)).startServices(eq(containerName)); inOrder.verify(dockerOperations, times(1)).resumeNode(eq(containerName)); inOrder.verify(orchestrator).resume(hostName); } @@ -159,6 +160,41 @@ public class NodeAgentImplTest { verify(storageMaintainer, times(1)).removeOldFilesFromNode(eq(containerName)); } + @Test + public void startsAfterStoppingServices() { + final InOrder inOrder = inOrder(dockerOperations); + final NodeSpec node = nodeBuilder + .wantedDockerImage(dockerImage) + .currentDockerImage(dockerImage) + .state(Node.State.active) + .wantedVespaVersion(vespaVersion) + .vespaVersion(vespaVersion) + .build(); + + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); + when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); + when(storageMaintainer.getDiskUsageFor(eq(containerName))).thenReturn(Optional.of(187500000000L)); + + nodeAgent.converge(); + inOrder.verify(dockerOperations, times(1)).startServices(eq(containerName)); + inOrder.verify(dockerOperations, times(1)).resumeNode(eq(containerName)); + + nodeAgent.suspend(); + nodeAgent.converge(); + inOrder.verify(dockerOperations, never()).startServices(eq(containerName)); + inOrder.verify(dockerOperations, times(1)).resumeNode(eq(containerName)); // Expect a resume, but no start services + + // No new suspends/stops, so no need to resume/start + nodeAgent.converge(); + inOrder.verify(dockerOperations, never()).startServices(eq(containerName)); + inOrder.verify(dockerOperations, never()).resumeNode(eq(containerName)); + + nodeAgent.suspend(); + nodeAgent.stopServices(); + nodeAgent.converge(); + inOrder.verify(dockerOperations, times(1)).startServices(eq(containerName)); + inOrder.verify(dockerOperations, times(1)).resumeNode(eq(containerName)); + } @Test public void absentContainerCausesStart() throws Exception { @@ -184,6 +220,7 @@ public class NodeAgentImplTest { nodeAgent.converge(); verify(dockerOperations, never()).removeContainer(any()); + verify(dockerOperations, never()).startServices(any()); verify(orchestrator, never()).suspend(any(String.class)); final InOrder inOrder = inOrder(dockerOperations, orchestrator, nodeRepository, aclMaintainer); @@ -426,6 +463,8 @@ public class NodeAgentImplTest { verify(dockerOperations, never()).createContainer(eq(containerName), any(), any()); verify(dockerOperations, never()).startContainer(eq(containerName)); + verify(dockerOperations, never()).suspendNode(eq(containerName)); + verify(dockerOperations, times(1)).stopServices(eq(containerName)); verify(orchestrator, never()).resume(any(String.class)); verify(orchestrator, never()).suspend(any(String.class)); // current Docker image and vespa version should be cleared @@ -579,7 +618,7 @@ public class NodeAgentImplTest { verify(dockerOperations, never()).removeContainer(any()); verify(dockerOperations, times(1)).createContainer(eq(containerName), eq(node), any()); verify(dockerOperations, times(1)).startContainer(eq(containerName)); - verify(nodeAgent, never()).runLocalResumeScriptIfNeeded(any()); + verify(nodeAgent, never()).resumeNodeIfNeeded(any()); // The docker container was actually started and is running, but subsequent exec calls to set up // networking failed @@ -589,7 +628,7 @@ public class NodeAgentImplTest { verify(dockerOperations, times(1)).removeContainer(any()); verify(dockerOperations, times(2)).createContainer(eq(containerName), eq(node), any()); verify(dockerOperations, times(2)).startContainer(eq(containerName)); - verify(nodeAgent, times(1)).runLocalResumeScriptIfNeeded(any()); + verify(nodeAgent, times(1)).resumeNodeIfNeeded(any()); } @Test diff --git a/parent/pom.xml b/parent/pom.xml index 96075974567..891eabd9c83 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -461,7 +461,7 @@ <artifactId>commons-pool</artifactId> <version>1.5.6</version> </dependency> - <!-- Explicitly included to get Zookeeper version 3.4.12, + <!-- Explicitly included to get Zookeeper version 3.4.13, can be excluded if you want the Zookeeper version used by curator by default --> @@ -673,6 +673,11 @@ <artifactId>language-detector</artifactId> <version>0.6</version> </dependency> + <dependency> + <groupId>net.java.dev.jna</groupId> + <artifactId>jna</artifactId> + <version>${jna.version}</version> + </dependency> </dependencies> </dependencyManagement> @@ -680,6 +685,7 @@ <antlr.version>3.5.2</antlr.version> <antlr4.version>4.5</antlr4.version> <asm.version>6.2</asm.version> + <jna.version>4.5.2</jna.version> <!-- Athenz dependencies. Make sure these dependencies matches those in Vespa's internal repositories --> <athenz.version>1.7.43</athenz.version> <commons-lang.version>2.6</commons-lang.version> diff --git a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp index ea2cc00c642..f4b80f7961c 100644 --- a/storage/src/tests/distributor/twophaseupdateoperationtest.cpp +++ b/storage/src/tests/distributor/twophaseupdateoperationtest.cpp @@ -11,7 +11,6 @@ #include <vespa/storage/distributor/externaloperationhandler.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storage/distributor/operations/external/twophaseupdateoperation.h> -#include <vespa/storageapi/message/batch.h> #include <tests/distributor/distributortestutil.h> #include <vespa/document/test/make_document_bucket.h> #include <vespa/storage/distributor/distributor.h> diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 369e820f987..61c55248370 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -12,12 +12,12 @@ #include <vespa/storage/persistence/filestorage/filestormanager.h> #include <vespa/storage/persistence/filestorage/modifiedbucketchecker.h> #include <vespa/document/update/assignvalueupdate.h> +#include <vespa/document/update/documentupdate.h> #include <vespa/document/select/parser.h> #include <vespa/vdslib/state/random.h> #include <vespa/storageapi/message/bucketsplitting.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> #include <vespa/persistence/spi/test.h> -#include <vespa/storageapi/message/batch.h> #include <vespa/config/common/exceptions.h> #include <vespa/fastos/file.h> diff --git a/storage/src/tests/storageserver/documentapiconvertertest.cpp b/storage/src/tests/storageserver/documentapiconvertertest.cpp index 40d561bd589..7f8b1b8f34a 100644 --- a/storage/src/tests/storageserver/documentapiconvertertest.cpp +++ b/storage/src/tests/storageserver/documentapiconvertertest.cpp @@ -8,11 +8,11 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/select/parser.h> #include <vespa/document/test/make_document_bucket.h> +#include <vespa/document/update/documentupdate.h> #include <vespa/documentapi/documentapi.h> #include <vespa/messagebus/emptyreply.h> #include <vespa/storage/common/bucket_resolver.h> #include <vespa/storage/storageserver/documentapiconverter.h> -#include <vespa/storageapi/message/batch.h> #include <vespa/storageapi/message/datagram.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/removelocation.h> @@ -115,7 +115,6 @@ struct DocumentApiConverterTest : public CppUnit::TestFixture void testCreateVisitorReplyLastBucket(); void testDestroyVisitor(); void testVisitorInfo(); - void testBatchDocumentUpdate(); void testStatBucket(); void testGetBucketList(); void testRemoveLocation(); @@ -133,7 +132,6 @@ struct DocumentApiConverterTest : public CppUnit::TestFixture CPPUNIT_TEST(testCreateVisitorReplyLastBucket); CPPUNIT_TEST(testDestroyVisitor); CPPUNIT_TEST(testVisitorInfo); - CPPUNIT_TEST(testBatchDocumentUpdate); CPPUNIT_TEST(testStatBucket); CPPUNIT_TEST(testGetBucketList); CPPUNIT_TEST(testRemoveLocation); @@ -321,58 +319,6 @@ DocumentApiConverterTest::testVisitorInfo() } void -DocumentApiConverterTest::testBatchDocumentUpdate() -{ - std::vector<document::DocumentUpdate::SP > updates; - - { - document::DocumentId docId(document::UserDocIdString("userdoc:test:1234:test1")); - auto update = std::make_shared<document::DocumentUpdate>(*_repo, _html_type, docId); - updates.push_back(update); - } - - { - document::DocumentId docId(document::UserDocIdString("userdoc:test:1234:test2")); - auto update = std::make_shared<document::DocumentUpdate>(*_repo, _html_type, docId); - updates.push_back(update); - } - - { - document::DocumentId docId(document::UserDocIdString("userdoc:test:1234:test3")); - auto update = std::make_shared<document::DocumentUpdate>(*_repo, _html_type, docId); - updates.push_back(update); - } - - auto msg = std::make_shared<documentapi::BatchDocumentUpdateMessage>(1234); - for (std::size_t i = 0; i < updates.size(); ++i) { - msg->addUpdate(updates[i]); - } - - auto batchCmd = toStorageAPI<api::BatchDocumentUpdateCommand>(*msg); - CPPUNIT_ASSERT_EQUAL(updates.size(), batchCmd->getUpdates().size()); - for (std::size_t i = 0; i < updates.size(); ++i) { - CPPUNIT_ASSERT_EQUAL(*updates[i], *batchCmd->getUpdates()[i]); - } - - api::BatchDocumentUpdateReply batchReply(*batchCmd); - batchReply.getDocumentsNotFound().resize(3); - batchReply.getDocumentsNotFound()[0] = true; - batchReply.getDocumentsNotFound()[2] = true; - - std::unique_ptr<mbus::Reply> mbusReply = msg->createReply(); - documentapi::BatchDocumentUpdateReply* mbusBatchReply( - dynamic_cast<documentapi::BatchDocumentUpdateReply*>(mbusReply.get())); - CPPUNIT_ASSERT(mbusBatchReply != 0); - - _converter->transferReplyState(batchReply, *mbusReply); - - CPPUNIT_ASSERT_EQUAL(std::size_t(3), mbusBatchReply->getDocumentsNotFound().size()); - CPPUNIT_ASSERT(mbusBatchReply->getDocumentsNotFound()[0] == true); - CPPUNIT_ASSERT(mbusBatchReply->getDocumentsNotFound()[1] == false); - CPPUNIT_ASSERT(mbusBatchReply->getDocumentsNotFound()[2] == true); -} - -void DocumentApiConverterTest::testStatBucket() { documentapi::StatBucketMessage msg(BucketId(123), ""); diff --git a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp index 0837b0d1463..d553b41de7c 100644 --- a/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp +++ b/storage/src/vespa/storage/bucketdb/storagebucketdbinitializer.cpp @@ -438,7 +438,7 @@ StorageBucketDBInitializer::registerBucket(const document::Bucket &bucket, _system._nodeState, _system._nodeIndex, bucketId.stripUnused(), lib::Distribution::IDEAL_DISK_EVEN_IF_DOWN)); if (disk != partition) { - ++_metrics._wrongDisk; + _metrics._wrongDisk.inc(); } _metrics._insertedCount.inc(); diff --git a/storage/src/vespa/storage/common/messagebucket.cpp b/storage/src/vespa/storage/common/messagebucket.cpp index ecbad310a58..4a6f638262d 100644 --- a/storage/src/vespa/storage/common/messagebucket.cpp +++ b/storage/src/vespa/storage/common/messagebucket.cpp @@ -9,7 +9,6 @@ #include <vespa/storageapi/message/removelocation.h> #include <vespa/storage/persistence/messages.h> #include <vespa/storageapi/message/stat.h> -#include <vespa/storageapi/message/batch.h> #include <vespa/vespalib/util/exceptions.h> @@ -31,8 +30,6 @@ getStorageMessageBucket(const api::StorageMessage& msg) return static_cast<const api::RevertCommand&>(msg).getBucket(); case api::MessageType::STATBUCKET_ID: return static_cast<const api::StatBucketCommand&>(msg).getBucket(); - case api::MessageType::BATCHPUTREMOVE_ID: - return static_cast<const api::BatchPutRemoveCommand&>(msg).getBucket(); case api::MessageType::REMOVELOCATION_ID: return static_cast<const api::RemoveLocationCommand&>(msg).getBucket(); case api::MessageType::CREATEBUCKET_ID: diff --git a/storage/src/vespa/storage/distributor/distributor.cpp b/storage/src/vespa/storage/distributor/distributor.cpp index 2d490ea9923..1664dd0d9a1 100644 --- a/storage/src/vespa/storage/distributor/distributor.cpp +++ b/storage/src/vespa/storage/distributor/distributor.cpp @@ -616,8 +616,6 @@ bool is_client_request(const api::StorageMessage& msg) noexcept { case api::MessageType::STATBUCKET_ID: case api::MessageType::UPDATE_ID: case api::MessageType::REMOVELOCATION_ID: - case api::MessageType::BATCHPUTREMOVE_ID: // Deprecated - case api::MessageType::BATCHDOCUMENTUPDATE_ID: // Deprecated return true; default: return false; diff --git a/storage/src/vespa/storage/distributor/externaloperationhandler.cpp b/storage/src/vespa/storage/distributor/externaloperationhandler.cpp index bb8c2b0608a..b22592af327 100644 --- a/storage/src/vespa/storage/distributor/externaloperationhandler.cpp +++ b/storage/src/vespa/storage/distributor/externaloperationhandler.cpp @@ -15,7 +15,6 @@ #include <vespa/document/util/stringutil.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/removelocation.h> -#include <vespa/storageapi/message/batch.h> #include <vespa/storageapi/message/stat.h> #include "distributor_bucket_space_repo.h" #include "distributor_bucket_space.h" @@ -79,11 +78,11 @@ ExternalOperationHandler::checkTimestampMutationPreconditions(api::StorageComman LOG(debug, "Distributor manager received %s, bucket %s with wrong distribution", cmd.toString().c_str(), bucket.toString().c_str()); - persistenceMetrics.failures.wrongdistributor++; + persistenceMetrics.failures.wrongdistributor.inc(); return false; } if (!checkSafeTimeReached(cmd)) { - persistenceMetrics.failures.safe_time_not_reached++; + persistenceMetrics.failures.safe_time_not_reached.inc(); return false; } return true; @@ -97,7 +96,7 @@ ExternalOperationHandler::makeConcurrentMutationRejectionReply(api::StorageComma auto err_msg = vespalib::make_string("A mutating operation for document '%s' is already in progress", docId.toString().c_str()); LOG(debug, "Aborting incoming %s operation: %s", cmd.getType().toString().c_str(), err_msg.c_str()); - persistenceMetrics.failures.concurrent_mutations++; + persistenceMetrics.failures.concurrent_mutations.inc(); api::StorageReply::UP reply(cmd.makeReply()); reply->setResult(api::ReturnCode(api::ReturnCode::BUSY, err_msg)); return std::shared_ptr<api::StorageMessage>(reply.release()); @@ -190,7 +189,7 @@ IMPL_MSG_COMMAND_H(ExternalOperationHandler, RemoveLocation) if (!checkDistribution(*cmd, bucket)) { LOG(debug, "Distributor manager received %s with wrong distribution", cmd->toString().c_str()); - getMetrics().removelocations[cmd->getLoadType()].failures.wrongdistributor++; + getMetrics().removelocations[cmd->getLoadType()].failures.wrongdistributor.inc(); return true; } @@ -206,7 +205,7 @@ IMPL_MSG_COMMAND_H(ExternalOperationHandler, Get) LOG(debug, "Distributor manager received get for %s, bucket %s with wrong distribution", cmd->getDocumentId().toString().c_str(), bucket.toString().c_str()); - getMetrics().gets[cmd->getLoadType()].failures.wrongdistributor++; + getMetrics().gets[cmd->getLoadType()].failures.wrongdistributor.inc(); return true; } diff --git a/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp index 5b1e585a6f8..04fcd4bd01c 100644 --- a/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp @@ -193,19 +193,19 @@ GetOperation::sendReply(DistributorMessageSender& sender) repl->setResult(_returnCode); if (_returnCode.success()) { - ++_metric.ok; + _metric.ok.inc(); } else if (_returnCode.getResult() == api::ReturnCode::TIMEOUT) { - ++_metric.failures.timeout; + _metric.failures.timeout.inc(); } else if (_returnCode.isBusy()) { - ++_metric.failures.busy; + _metric.failures.busy.inc(); } else if (_returnCode.isNodeDownOrNetwork()) { - ++_metric.failures.notconnected; + _metric.failures.notconnected.inc(); } else { - ++_metric.failures.storagefailure; + _metric.failures.storagefailure.inc(); } if (!_doc.get()) { - ++_metric.failures.notfound; + _metric.failures.notfound.inc(); } _metric.latency.addValue(_operationTimer.getElapsedTimeAsDouble()); diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp index 0b0fc05763f..c652f787b2e 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp @@ -6,8 +6,8 @@ #include "updateoperation.h" #include <vespa/storage/distributor/distributor_bucket_space.h> #include <vespa/storageapi/message/persistence.h> -#include <vespa/storageapi/message/batch.h> #include <vespa/document/datatype/documenttype.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/document/select/parser.h> #include <vespa/vespalib/stllike/hash_map.hpp> diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h index e3fb6c93a3a..6efce913e70 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.h @@ -15,7 +15,6 @@ namespace storage { namespace api { class UpdateCommand; -class BatchDocumentUpdateCommand; class CreateBucketReply; } diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp index 3475e3e12ba..784ea7253b6 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp @@ -308,9 +308,7 @@ constexpr std::array<uint32_t, 7> WRITE_FEED_MESSAGE_TYPES {{ api::MessageType::PUT_ID, api::MessageType::REMOVE_ID, api::MessageType::UPDATE_ID, - api::MessageType::REMOVELOCATION_ID, - api::MessageType::BATCHPUTREMOVE_ID, - api::MessageType::BATCHDOCUMENTUPDATE_ID + api::MessageType::REMOVELOCATION_ID }}; } diff --git a/storage/src/vespa/storage/distributor/persistence_operation_metric_set.cpp b/storage/src/vespa/storage/distributor/persistence_operation_metric_set.cpp index 88b28941e65..d5f73a9e916 100644 --- a/storage/src/vespa/storage/distributor/persistence_operation_metric_set.cpp +++ b/storage/src/vespa/storage/distributor/persistence_operation_metric_set.cpp @@ -79,21 +79,21 @@ void PersistenceOperationMetricSet::updateFromResult(const api::ReturnCode& result) { if (result.success()) { - ++ok; + ok.inc(); } else if (result.getResult() == api::ReturnCode::WRONG_DISTRIBUTION) { - ++failures.wrongdistributor; + failures.wrongdistributor.inc(); } else if (result.getResult() == api::ReturnCode::TIMEOUT) { - ++failures.timeout; + failures.timeout.inc(); } else if (result.isBusy()) { - ++failures.busy; + failures.busy.inc(); } else if (result.isBucketDisappearance()) { // Bucket not found/deleted codes imply that replicas are transiently // inconsistent in our DB or across replica nodes. - ++failures.inconsistent_bucket; + failures.inconsistent_bucket.inc(); } else if (result.isNodeDownOrNetwork()) { - ++failures.notconnected; + failures.notconnected.inc(); } else { - ++failures.storagefailure; + failures.storagefailure.inc(); } } diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index f9571228ef9..3ccb35ef5a7 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -12,7 +12,6 @@ #include <vespa/storage/common/messagebucket.h> #include <vespa/storage/persistence/messages.h> #include <vespa/storageapi/message/stat.h> -#include <vespa/storageapi/message/batch.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/exceptions.h> @@ -249,8 +248,6 @@ FileStorHandlerImpl::messageMayBeAborted(const api::StorageMessage& msg) case api::MessageType::JOINBUCKETS_ID: case api::MessageType::UPDATE_ID: case api::MessageType::REMOVELOCATION_ID: - case api::MessageType::BATCHPUTREMOVE_ID: - case api::MessageType::BATCHDOCUMENTUPDATE_ID: case api::MessageType::SETBUCKETSTATE_ID: return true; default: @@ -587,7 +584,6 @@ FileStorHandlerImpl::remapMessage(api::StorageMessage& msg, const document::Buck break; } case api::MessageType::STAT_ID: - case api::MessageType::BATCHPUTREMOVE_ID: case api::MessageType::REVERT_ID: case api::MessageType::REMOVELOCATION_ID: case api::MessageType::SETBUCKETSTATE_ID: diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index bf0244255c1..e7323d07480 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -11,7 +11,6 @@ #include <vespa/storage/config/config-stor-server.h> #include <vespa/storage/persistence/bucketownershipnotifier.h> #include <vespa/storage/persistence/persistencethread.h> -#include <vespa/storageapi/message/batch.h> #include <vespa/storageapi/message/bucketsplitting.h> #include <vespa/storageapi/message/state.h> #include <vespa/vespalib/stllike/hash_map.hpp> @@ -325,16 +324,6 @@ FileStorManager::onRevert(const shared_ptr<api::RevertCommand>& cmd) } bool -FileStorManager::onBatchPutRemove(const std::shared_ptr<api::BatchPutRemoveCommand>& cmd) -{ - StorBucketDatabase::WrappedEntry entry(mapOperationToBucketAndDisk(*cmd, 0)); - if (entry.exist()) { - handlePersistenceMessage(cmd, entry->disk); - } - return true; -} - -bool FileStorManager::onRemoveLocation(const std::shared_ptr<api::RemoveLocationCommand>& cmd) { StorBucketDatabase::WrappedEntry entry(mapOperationToDisk(*cmd, cmd->getBucket())); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h index 4bf2c1049cf..5c52e6c6a23 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h @@ -151,7 +151,6 @@ private: bool onGet(const std::shared_ptr<api::GetCommand>&) override; bool onRemove(const std::shared_ptr<api::RemoveCommand>&) override; bool onRevert(const std::shared_ptr<api::RevertCommand>&) override; - bool onBatchPutRemove(const std::shared_ptr<api::BatchPutRemoveCommand>&) override; bool onStatBucket(const std::shared_ptr<api::StatBucketCommand>&) override; // Bucket operations diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp index c3dc7cba80a..26cfb0e6566 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.cpp +++ b/storage/src/vespa/storage/persistence/mergehandler.cpp @@ -1301,7 +1301,7 @@ void MergeHandler::handleGetBucketDiffReply(api::GetBucketDiffReply& reply, MessageSender& sender) { - ++_env._metrics.getBucketDiffReply; + _env._metrics.getBucketDiffReply.inc(); spi::Bucket bucket(reply.getBucket(), spi::PartitionId(_env._partition)); LOG(debug, "GetBucketDiffReply(%s)", bucket.toString().c_str()); @@ -1479,7 +1479,7 @@ void MergeHandler::handleApplyBucketDiffReply(api::ApplyBucketDiffReply& reply, MessageSender& sender) { - ++_env._metrics.applyBucketDiffReply; + _env._metrics.applyBucketDiffReply.inc(); spi::Bucket bucket(reply.getBucket(), spi::PartitionId(_env._partition)); std::vector<api::ApplyBucketDiffCommand::Entry>& diff(reply.getDiff()); LOG(debug, "%s", reply.toString().c_str()); diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index 0eeb4eeb84d..1b221ea0b7c 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -137,7 +137,7 @@ PersistenceThread::handleRemove(api::RemoveCommand& cmd) tracker->setReply(std::make_shared<api::RemoveReply>(cmd, response.wasFound() ? cmd.getTimestamp() : 0)); } if (!response.wasFound()) { - ++_env._metrics.remove[cmd.getLoadType()].notFound; + _env._metrics.remove[cmd.getLoadType()].notFound.inc(); } return tracker; } @@ -177,7 +177,7 @@ PersistenceThread::handleGet(api::GetCommand& cmd) if (checkForError(result, *tracker)) { if (!result.hasDocument()) { - ++_env._metrics.get[cmd.getLoadType()].notFound; + _env._metrics.get[cmd.getLoadType()].notFound.inc(); } tracker->setReply(std::make_shared<api::GetReply>(cmd, result.getDocumentPtr(), result.getTimestamp())); } @@ -202,7 +202,7 @@ PersistenceThread::handleRepairBucket(RepairBucketCommand& cmd) reply->setAltered(!(after == before)); if (reply->bucketAltered()) { notifyGuard.notifyAlways(cmd.getBucket(), after); - ++_env._metrics.repairFixed; + _env._metrics.repairFixed.inc(); } _env.updateBucketDatabase(cmd.getBucket(), after); @@ -810,7 +810,7 @@ PersistenceThread::processMessage(api::StorageMessage& msg) { MBUS_TRACE(msg.getTrace(), 5, "PersistenceThread: Processing message in persistence layer"); - ++_env._metrics.operations; + _env._metrics.operations.inc(); if (msg.getType().isReply()) { try{ LOG(debug, "Handling reply: %s", msg.toString().c_str()); @@ -837,7 +837,7 @@ PersistenceThread::processMessage(api::StorageMessage& msg) && tracker->getReply()->getResult().failed()) || tracker->getResult().failed()) { - ++_env._metrics.failedOperations; + _env._metrics.failedOperations.inc(); } } diff --git a/storage/src/vespa/storage/persistence/persistenceutil.cpp b/storage/src/vespa/storage/persistence/persistenceutil.cpp index 888dc93dd82..4bee5df4d04 100644 --- a/storage/src/vespa/storage/persistence/persistenceutil.cpp +++ b/storage/src/vespa/storage/persistence/persistenceutil.cpp @@ -23,7 +23,7 @@ MessageTracker::MessageTracker(FileStorThreadMetrics::Op& metric, _result(api::ReturnCode::OK), _timer(clock) { - ++_metric.count; + _metric.count.inc(); } MessageTracker::~MessageTracker() @@ -53,7 +53,7 @@ MessageTracker::generateReply(api::StorageCommand& cmd) } if (!_reply->getResult().success()) { - ++_metric.failed; + _metric.failed.inc(); LOGBP(debug, "Failed to handle command %s: %s", cmd.toString().c_str(), _result.toString().c_str()); diff --git a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp index 00bc395b536..b6ac2a4b219 100644 --- a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp +++ b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp @@ -3,9 +3,9 @@ #include "documentapiconverter.h" #include "priorityconverter.h" #include <vespa/document/bucket/bucketidfactory.h> +#include <vespa/document/update/documentupdate.h> #include <vespa/documentapi/documentapi.h> #include <vespa/storage/common/bucket_resolver.h> -#include <vespa/storageapi/message/batch.h> #include <vespa/storageapi/message/datagram.h> #include <vespa/storageapi/message/documentsummary.h> #include <vespa/storageapi/message/persistence.h> @@ -100,12 +100,6 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg) toMsg = std::make_unique<api::DestroyVisitorCommand>(from.getInstanceId()); break; } - case DocumentProtocol::MESSAGE_BATCHDOCUMENTUPDATE: - { - documentapi::BatchDocumentUpdateMessage& from(static_cast<documentapi::BatchDocumentUpdateMessage&>(fromMsg)); - toMsg = std::make_unique<api::BatchDocumentUpdateCommand>(from.getUpdates()); - break; - } case DocumentProtocol::MESSAGE_STATBUCKET: { documentapi::StatBucketMessage& from(static_cast<documentapi::StatBucketMessage&>(fromMsg)); @@ -377,10 +371,6 @@ DocumentApiConverter::transferReplyState(api::StorageReply& fromMsg, mbus::Reply documentapi::CreateVisitorReply& to(static_cast<documentapi::CreateVisitorReply&>(toMsg)); to.setLastBucket(from.getLastBucket()); to.setVisitorStatistics(from.getVisitorStatistics()); - } else if (toMsg.getType() == DocumentProtocol::REPLY_BATCHDOCUMENTUPDATE) { - api::BatchDocumentUpdateReply& from(static_cast<api::BatchDocumentUpdateReply&>(fromMsg)); - documentapi::BatchDocumentUpdateReply& to(static_cast<documentapi::BatchDocumentUpdateReply&>(toMsg)); - to.getDocumentsNotFound() = from.getDocumentsNotFound(); } } diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.cpp b/storage/src/vespa/storage/storageserver/mergethrottler.cpp index 22af1a73633..a9d54c196c9 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.cpp +++ b/storage/src/vespa/storage/storageserver/mergethrottler.cpp @@ -471,33 +471,33 @@ MergeThrottler::updateOperationMetrics( { switch (result.getResult()) { case api::ReturnCode::OK: - ++metrics.ok; + metrics.ok.inc(); break; case api::ReturnCode::NOT_READY: - ++metrics.failures.notready; + metrics.failures.notready.inc(); break; case api::ReturnCode::TIMEOUT: - ++metrics.failures.timeout; + metrics.failures.timeout.inc(); break; case api::ReturnCode::ABORTED: - ++metrics.failures.aborted; + metrics.failures.aborted.inc(); break; case api::ReturnCode::WRONG_DISTRIBUTION: - ++metrics.failures.wrongdistribution; + metrics.failures.wrongdistribution.inc(); break; case api::ReturnCode::EXISTS: - ++metrics.failures.exists; + metrics.failures.exists.inc(); break; case api::ReturnCode::REJECTED: - ++metrics.failures.rejected; + metrics.failures.rejected.inc(); break; default: if (result.isBusy()) { - ++metrics.failures.busy; + metrics.failures.busy.inc(); } else if (result.isBucketDisappearance()) { - ++metrics.failures.bucketnotfound; + metrics.failures.bucketnotfound.inc(); } else { - ++metrics.failures.other; + metrics.failures.other.inc(); } } } diff --git a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp index cfc69bcde45..da7e8cb743e 100644 --- a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp +++ b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp @@ -5,7 +5,6 @@ #include <vespa/storageapi/message/bucketsplitting.h> #include <vespa/storageapi/message/internal.h> #include <vespa/storageapi/message/removelocation.h> -#include <vespa/storageapi/message/batch.h> #include <vespa/storageapi/mbusprot/storageprotocol.h> #include <vespa/storageapi/mbusprot/storagecommand.h> #include <vespa/storageapi/mbusprot/storagereply.h> @@ -89,7 +88,6 @@ struct StorageProtocolTest : public CppUnit::TestFixture { void testSplitBucket51(); void testSplitBucketChain51(); void testJoinBuckets51(); - void testBatchPutRemove51(); void testCreateVisitor51(); void testDestroyVisitor51(); void testRemoveLocation51(); @@ -133,7 +131,6 @@ struct StorageProtocolTest : public CppUnit::TestFixture { CPPUNIT_TEST(testCreateVisitor51); CPPUNIT_TEST(testDestroyVisitor51); CPPUNIT_TEST(testRemoveLocation51); - CPPUNIT_TEST(testBatchPutRemove51); CPPUNIT_TEST(testInternalMessage); CPPUNIT_TEST(testSetBucketState51); @@ -782,55 +779,6 @@ StorageProtocolTest::testApplyBucketDiff51() recordSerialization50(); } -void -StorageProtocolTest::testBatchPutRemove51() -{ - ScopedName test("testBatchPutRemove51"); - - document::BucketId bucketId(20, 0xf1f1f1f1f1ull); - document::Bucket bucket(makeDocumentBucket(bucketId)); - BatchPutRemoveCommand::SP cmd(new BatchPutRemoveCommand(bucket)); - cmd->addPut(_testDoc, 100); - cmd->addHeaderUpdate(_testDoc, 101, 1234); - cmd->addRemove(_testDoc->getId(), 102); - cmd->forceMsgId(556677); - BatchPutRemoveCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(bucketId, cmd2->getBucketId()); - CPPUNIT_ASSERT_EQUAL(3, (int)cmd2->getOperationCount()); - CPPUNIT_ASSERT_EQUAL(*_testDoc, *(dynamic_cast<const BatchPutRemoveCommand::PutOperation&>(cmd2->getOperation(0)).document)); - CPPUNIT_ASSERT_EQUAL((uint64_t)100, cmd2->getOperation(0).timestamp); - { - vespalib::nbostream header; - _testDoc->serializeHeader(header); - document::Document headerDoc(_docMan.getTypeRepo(), header); - CPPUNIT_ASSERT_EQUAL( - headerDoc, - *(dynamic_cast<const BatchPutRemoveCommand::HeaderUpdateOperation&>( - cmd2->getOperation(1)).document)); - } - CPPUNIT_ASSERT_EQUAL((uint64_t)101, cmd2->getOperation(1).timestamp); - CPPUNIT_ASSERT_EQUAL(1234, (int)dynamic_cast<const BatchPutRemoveCommand::HeaderUpdateOperation&>(cmd2->getOperation(1)).timestampToUpdate); - CPPUNIT_ASSERT_EQUAL(_testDoc->getId(), dynamic_cast<const BatchPutRemoveCommand::RemoveOperation&>(cmd2->getOperation(2)).documentId); - CPPUNIT_ASSERT_EQUAL((uint64_t)102, cmd2->getOperation(2).timestamp); - CPPUNIT_ASSERT_EQUAL(uint64_t(556677), cmd2->getMsgId()); - - BatchPutRemoveReply::SP reply(new BatchPutRemoveReply(*cmd2)); - reply->getDocumentsNotFound().push_back(document::DocumentId("userdoc:footype:1234:foo1")); - reply->getDocumentsNotFound().push_back(document::DocumentId("userdoc:footype:1234:foo2")); - reply->getDocumentsNotFound().push_back(document::DocumentId("userdoc:footype:1234:foo3")); - - BatchPutRemoveReply::SP reply2(copyReply(reply)); - - CPPUNIT_ASSERT_EQUAL(3, (int)reply2->getDocumentsNotFound().size()); - CPPUNIT_ASSERT_EQUAL(document::DocumentId("userdoc:footype:1234:foo1"), reply2->getDocumentsNotFound()[0]); - CPPUNIT_ASSERT_EQUAL(document::DocumentId("userdoc:footype:1234:foo2"), reply2->getDocumentsNotFound()[1]); - CPPUNIT_ASSERT_EQUAL(document::DocumentId("userdoc:footype:1234:foo3"), reply2->getDocumentsNotFound()[2]); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); -} - namespace { struct MyCommand : public api::InternalCommand { MyCommand() : InternalCommand(101) {} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp index d08464da715..172cd6c8de5 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp @@ -7,7 +7,6 @@ #include <vespa/storageapi/message/bucketsplitting.h> #include <vespa/storageapi/message/visitor.h> #include <vespa/storageapi/message/removelocation.h> -#include <vespa/storageapi/message/batch.h> #include <vespa/vespalib/util/exceptions.h> @@ -132,12 +131,6 @@ ProtocolSerialization::encode(const api::StorageMessage& msg) const case api::MessageType::REMOVELOCATION_REPLY_ID: onEncode(buf, static_cast<const api::RemoveLocationReply&>(msg)); break; - case api::MessageType::BATCHPUTREMOVE_ID: - onEncode(buf, static_cast<const api::BatchPutRemoveCommand&>(msg)); - break; - case api::MessageType::BATCHPUTREMOVE_REPLY_ID: - onEncode(buf, static_cast<const api::BatchPutRemoveReply&>(msg)); - break; case api::MessageType::SETBUCKETSTATE_ID: onEncode(buf, static_cast<const api::SetBucketStateCommand&>(msg)); break; @@ -205,8 +198,6 @@ ProtocolSerialization::decodeCommand(mbus::BlobRef data) const cmd = onDecodeDestroyVisitorCommand(buf); break; case api::MessageType::REMOVELOCATION_ID: cmd = onDecodeRemoveLocationCommand(buf); break; - case api::MessageType::BATCHPUTREMOVE_ID: - cmd = onDecodeBatchPutRemoveCommand(buf); break; case api::MessageType::SETBUCKETSTATE_ID: cmd = onDecodeSetBucketStateCommand(buf); break; default: @@ -269,8 +260,6 @@ ProtocolSerialization::decodeReply(mbus::BlobRef data, const api::StorageCommand reply = onDecodeDestroyVisitorReply(cmd, buf); break; case api::MessageType::REMOVELOCATION_REPLY_ID: reply = onDecodeRemoveLocationReply(cmd, buf); break; - case api::MessageType::BATCHPUTREMOVE_REPLY_ID: - reply = onDecodeBatchPutRemoveReply(cmd, buf); break; case api::MessageType::SETBUCKETSTATE_REPLY_ID: reply = onDecodeSetBucketStateReply(cmd, buf); break; default: diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h index e7d80a0614f..9c3ddb88bdf 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h @@ -50,10 +50,6 @@ class SetBucketStateReply; class CreateVisitorCommand; class RemoveLocationCommand; class RemoveLocationReply; -class BatchPutRemoveCommand; -class BatchPutRemoveReply; -class BatchDocumentUpdateCommand; -class BatchDocumentUpdateReply; } namespace storage::mbusprot { @@ -125,8 +121,6 @@ protected: virtual void onEncode(GBBuf&, const api::DestroyVisitorReply&) const = 0; virtual void onEncode(GBBuf&, const api::RemoveLocationCommand&) const = 0; virtual void onEncode(GBBuf&, const api::RemoveLocationReply&) const = 0; - virtual void onEncode(GBBuf&, const api::BatchPutRemoveCommand&) const = 0; - virtual void onEncode(GBBuf&, const api::BatchPutRemoveReply&) const = 0; virtual SCmd::UP onDecodePutCommand(BBuf&) const = 0; virtual SRep::UP onDecodePutReply(const SCmd&, BBuf&) const = 0; @@ -166,8 +160,6 @@ protected: virtual SRep::UP onDecodeDestroyVisitorReply(const SCmd&, BBuf&) const = 0; virtual SCmd::UP onDecodeRemoveLocationCommand(BBuf&) const = 0; virtual SRep::UP onDecodeRemoveLocationReply(const SCmd&, BBuf&) const = 0; - virtual SCmd::UP onDecodeBatchPutRemoveCommand(BBuf&) const = 0; - virtual SRep::UP onDecodeBatchPutRemoveReply(const SCmd&, BBuf&) const = 0; virtual document::Bucket getBucket(document::ByteBuffer& buf) const = 0; virtual void putBucket(const document::Bucket& bucket, vespalib::GrowableByteBuffer& buf) const = 0; diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp index 2532f76f3a0..74a0c964d19 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp @@ -7,7 +7,6 @@ #include "storagereply.h" #include <vespa/storageapi/message/bucketsplitting.h> -#include <vespa/storageapi/message/batch.h> #include <vespa/storageapi/message/visitor.h> #include <vespa/storageapi/message/removelocation.h> #include <vespa/vespalib/util/exceptions.h> @@ -25,104 +24,6 @@ ProtocolSerialization4_2::ProtocolSerialization4_2( { } -void -ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::BatchPutRemoveCommand& msg) const -{ - // Serialization format - allow different types of serialization depending on source. - buf.putByte(0); - putBucket(msg.getBucket(), buf); - buf.putInt(msg.getOperationCount()); - - for (uint32_t i = 0; i < msg.getOperationCount(); i++) { - const api::BatchPutRemoveCommand::Operation& op = msg.getOperation(i); - buf.putByte((uint8_t)op.type); - buf.putLong(op.timestamp); - - switch (op.type) { - case api::BatchPutRemoveCommand::Operation::REMOVE: - buf.putString(static_cast<const api::BatchPutRemoveCommand::RemoveOperation&>(op).documentId.toString()); - break; - case api::BatchPutRemoveCommand::Operation::HEADERUPDATE: - { - buf.putLong(static_cast<const api::BatchPutRemoveCommand::HeaderUpdateOperation&>(op).timestampToUpdate); - - vespalib::nbostream stream; - static_cast<const api::BatchPutRemoveCommand::HeaderUpdateOperation&>(op).document->serializeHeader(stream); - buf.putInt(stream.size()); - buf.putBytes(stream.peek(), stream.size()); - break; - } - case api::BatchPutRemoveCommand::Operation::PUT: - SH::putDocument(static_cast<const api::BatchPutRemoveCommand::PutOperation&>(op).document.get(), buf); - break; - } - } - onEncodeBucketInfoCommand(buf, msg); -} - -api::StorageCommand::UP -ProtocolSerialization4_2::onDecodeBatchPutRemoveCommand(BBuf& buf) const -{ - SH::getByte(buf); - document::Bucket bucket = getBucket(buf); - std::unique_ptr<api::BatchPutRemoveCommand> cmd(new api::BatchPutRemoveCommand(bucket)); - int length = SH::getInt(buf); - - for (int i = 0; i < length; i++) { - int type = SH::getByte(buf); - long timestamp = SH::getLong(buf); - - switch (type) { - case api::BatchPutRemoveCommand::Operation::REMOVE: - cmd->addRemove(document::DocumentId(SH::getString(buf)), timestamp); - break; - case api::BatchPutRemoveCommand::Operation::HEADERUPDATE: - { - long newTimestamp = SH::getLong(buf); - cmd->addHeaderUpdate(document::Document::SP( - SH::getDocument(buf, getTypeRepo())), - timestamp, newTimestamp); - break; - } - case api::BatchPutRemoveCommand::Operation::PUT: - cmd->addPut(document::Document::SP(SH::getDocument( - buf, getTypeRepo())), timestamp); - break; - } - } - - onDecodeBucketInfoCommand(buf, *cmd); - - return api::StorageCommand::UP(cmd.release()); -} - -void ProtocolSerialization4_2::onEncode( - GBBuf& buf, const api::BatchPutRemoveReply& msg) const -{ - buf.putInt(msg.getDocumentsNotFound().size()); - for (uint32_t i = 0; i < msg.getDocumentsNotFound().size(); i++) { - buf.putString(msg.getDocumentsNotFound()[i].toString()); - } - - onEncodeBucketInfoReply(buf, msg); -} - -api::StorageReply::UP -ProtocolSerialization4_2::onDecodeBatchPutRemoveReply(const SCmd& cmd, - BBuf& buf) const -{ - api::BatchPutRemoveReply::UP msg(new api::BatchPutRemoveReply( - static_cast<const api::BatchPutRemoveCommand&>(cmd))); - uint32_t count = SH::getInt(buf); - for (uint32_t i = 0; i < count; i++) { - msg->getDocumentsNotFound().push_back(document::DocumentId(SH::getString(buf))); - } - - onDecodeBucketInfoReply(buf, *msg); - return api::StorageReply::UP(msg.release()); -} - void ProtocolSerialization4_2::onEncode( GBBuf& buf, const api::GetCommand& msg) const { diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h index 3a6f77e46a3..56aa3d4ed30 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h @@ -28,8 +28,6 @@ protected: void onEncode(GBBuf&, const api::RemoveLocationReply&) const override; // Not supported on 4.2, but implemented here for simplicity. - void onEncode(GBBuf&, const api::BatchPutRemoveCommand&) const override; - void onEncode(GBBuf&, const api::BatchPutRemoveReply&) const override; void onEncode(GBBuf&, const api::SetBucketStateCommand&) const override; void onEncode(GBBuf&, const api::SetBucketStateReply&) const override; @@ -59,10 +57,6 @@ protected: SCmd::UP onDecodeRemoveLocationCommand(BBuf&) const override; SRep::UP onDecodeRemoveLocationReply(const SCmd&, BBuf&) const override; - // Not supported on 4.2, but implemented here for simplicity. - SCmd::UP onDecodeBatchPutRemoveCommand(BBuf&) const override; - SRep::UP onDecodeBatchPutRemoveReply(const SCmd&, BBuf&) const override; - virtual void onDecodeBucketInfoCommand(BBuf&, api::BucketInfoCommand&) const; virtual void onDecodeBucketInfoReply(BBuf&, api::BucketInfoReply&) const = 0; virtual void onDecodeCommand(BBuf& buf, api::StorageCommand& msg) const = 0; diff --git a/storageapi/src/vespa/storageapi/message/CMakeLists.txt b/storageapi/src/vespa/storageapi/message/CMakeLists.txt index dbbaad8eed1..cde9183482f 100644 --- a/storageapi/src/vespa/storageapi/message/CMakeLists.txt +++ b/storageapi/src/vespa/storageapi/message/CMakeLists.txt @@ -12,7 +12,6 @@ vespa_add_library(storageapi_message OBJECT stat.cpp removelocation.cpp queryresult.cpp - batch.cpp internal.cpp DEPENDS ) diff --git a/storageapi/src/vespa/storageapi/message/batch.cpp b/storageapi/src/vespa/storageapi/message/batch.cpp deleted file mode 100644 index de8ac849dee..00000000000 --- a/storageapi/src/vespa/storageapi/message/batch.cpp +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// -#include "batch.h" -#include <vespa/document/bucket/bucketidfactory.h> -#include <ostream> - -using namespace storage::api; -using document::BucketSpace; - -IMPLEMENT_COMMAND(BatchPutRemoveCommand, BatchPutRemoveReply) -IMPLEMENT_REPLY(BatchPutRemoveReply) -IMPLEMENT_COMMAND(BatchDocumentUpdateCommand, BatchDocumentUpdateReply) -IMPLEMENT_REPLY(BatchDocumentUpdateReply) - - -BatchPutRemoveCommand::Operation::Operation(uint64_t ts, Type tp) - : timestamp(ts), - type(tp) -{ -} - -BatchPutRemoveCommand::PutOperation::PutOperation(document::Document::SP doc, uint64_t ts) - : Operation(ts, PUT), - document(doc) -{ -} - -BatchPutRemoveCommand::HeaderUpdateOperation::HeaderUpdateOperation(document::Document::SP doc, uint64_t newTimestamp, uint64_t timestampToUpdate_) - : Operation(newTimestamp, HEADERUPDATE), - document(doc), - timestampToUpdate(timestampToUpdate_) -{ -} - -BatchPutRemoveCommand::RemoveOperation::RemoveOperation(const document::DocumentId& docId, uint64_t ts) - : Operation(ts, REMOVE), - documentId(docId) -{ -} - -BatchPutRemoveCommand::BatchPutRemoveCommand(const document::Bucket &bucket) - : BucketInfoCommand(MessageType::BATCHPUTREMOVE, bucket), - _approxSize(0) -{ -} - -void -BatchPutRemoveCommand::addPut(document::Document::SP document, uint64_t ts) -{ - _operations.push_back(std::unique_ptr<Operation>(new PutOperation(document, ts))); - _approxSize += document->serialize()->getLength(); -} - -void -BatchPutRemoveCommand::addHeaderUpdate(document::Document::SP document, uint64_t ts, uint64_t timestampToUpdate) -{ - _operations.push_back(std::unique_ptr<Operation>(new HeaderUpdateOperation(document, ts, timestampToUpdate))); - _approxSize += document->serialize()->getLength(); -} - -void -BatchPutRemoveCommand::addRemove(const document::DocumentId& docId, uint64_t ts) -{ - _operations.push_back(std::unique_ptr<Operation>(new RemoveOperation(docId, ts))); - _approxSize += docId.toString().length(); -} - -void -BatchPutRemoveCommand::addOperation(const Operation& op, bool cloneDocument) -{ - switch (op.type) { - case Operation::PUT: - { - document::Document::SP doc; - if (!cloneDocument) { - doc = static_cast<const PutOperation&>(op).document; - } else { - doc.reset(static_cast<const PutOperation&>(op).document->clone()); - } - addPut(doc, op.timestamp); - break; - } - case Operation::REMOVE: - addRemove(static_cast<const RemoveOperation&>(op).documentId, op.timestamp); - break; - case Operation::HEADERUPDATE: - { - const HeaderUpdateOperation& hup = static_cast<const HeaderUpdateOperation&>(op); - document::Document::SP doc; - if (!cloneDocument) { - doc = hup.document; - } else { - doc.reset(hup.document->clone()); - } - addHeaderUpdate(doc, op.timestamp, hup.timestampToUpdate); - break; - } - } -} - -void -BatchPutRemoveCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const { - out << "BatchPutRemove(" << getBucketId() << ", " << _operations.size() << " operations)"; - - if (verbose) { - out << " : "; - BucketInfoCommand::print(out, verbose, indent); - } -} - -BatchPutRemoveReply::BatchPutRemoveReply(const BatchPutRemoveCommand& cmd) - : BucketInfoReply(cmd) -{ -} - -void -BatchPutRemoveReply::print(std::ostream& out, bool verbose, - const std::string& indent) const { - out << "BatchPutRemoveReply("; - out << _documentsNotFound.size() << " documents not found)"; - - if (verbose) { - out << " {"; - for (std::vector<document::DocumentId>::const_iterator it = - _documentsNotFound.begin(); - it != _documentsNotFound.end(); ++it) - { - out << "\n" << indent << " " << (*it); - } - out << "\n" << indent << "} : "; - BucketInfoReply::print(out, verbose, indent); - } -} - -BatchDocumentUpdateCommand::BatchDocumentUpdateCommand(const UpdateList& updates) - : StorageCommand(MessageType::BATCHDOCUMENTUPDATE), - _updates(updates), - _bucket(BucketSpace::placeHolder(), document::BucketId()) -{ - document::BucketIdFactory factory; - _bucket = document::Bucket(BucketSpace::placeHolder(), factory.getBucketId(updates[0]->getId())); -} - -void -BatchDocumentUpdateCommand::print(std::ostream& out, bool verbose, - const std::string& indent) const { - out << "BatchDocumentUpdate(" << _updates.size() << " operations)"; - - if (verbose) { - out << " : "; - StorageCommand::print(out, verbose, indent); - } -} - -BatchDocumentUpdateReply::BatchDocumentUpdateReply(const BatchDocumentUpdateCommand& cmd) - : StorageReply(cmd), - _documentsNotFound() -{ -} - -void -BatchDocumentUpdateReply::print(std::ostream& out, bool verbose, - const std::string& indent) const { - out << "BatchDocumentUpdateReply(" - << std::count(_documentsNotFound.begin(), _documentsNotFound.end(), true) - << " not found)"; - - if (verbose) { - out << " : "; - StorageReply::print(out, verbose, indent); - } -} diff --git a/storageapi/src/vespa/storageapi/message/batch.h b/storageapi/src/vespa/storageapi/message/batch.h deleted file mode 100644 index 56cca3cfe38..00000000000 --- a/storageapi/src/vespa/storageapi/message/batch.h +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/document/fieldvalue/document.h> -#include <vespa/document/update/documentupdate.h> -#include <vespa/storageapi/messageapi/bucketinfocommand.h> -#include <vespa/storageapi/messageapi/bucketinforeply.h> - -namespace storage::api { - -/** - * @class BatchPutRemoveCommand - * @ingroup message - * - * @brief Sends a batch of puts and removes - */ -class BatchPutRemoveCommand : public BucketInfoCommand { -public: - class Operation { - public: - enum Type { - REMOVE, // Removes a document - HEADERUPDATE, // Updates the header of a document, if it already exists. - PUT // Inserts a new document. - }; - - Operation(uint64_t ts, Type type); - virtual ~Operation() {}; - - uint64_t timestamp; - Type type; - - virtual const document::DocumentId& getDocumentId() const = 0; - }; - - explicit BatchPutRemoveCommand(const document::Bucket &bucket); - - class PutOperation : public Operation { - public: - PutOperation(document::Document::SP document, uint64_t timestamp); - - document::Document::SP document; - - const document::DocumentId& getDocumentId() const override { - return document->getId(); - } - }; - - class HeaderUpdateOperation : public Operation { - public: - HeaderUpdateOperation(document::Document::SP document, uint64_t newTimestamp, uint64_t timestampToUpdate); - - document::Document::SP document; - uint64_t timestampToUpdate; - - const document::DocumentId& getDocumentId() const override { - return document->getId(); - } - }; - - class RemoveOperation : public Operation { - public: - RemoveOperation(const document::DocumentId& docId, uint64_t timestamp); - - document::DocumentId documentId; - - const document::DocumentId& getDocumentId() const override { - return documentId; - } - }; - - /** - Adds a PUT operation to be performed. - */ - void addPut(document::Document::SP document, uint64_t timestamp); - - /** - Adds a PUT operation to be performed. - */ - void addHeaderUpdate(document::Document::SP document, uint64_t newTimestamp, uint64_t timestampToUpdate); - - /** - Adds a REMOVE operation to be performed. - */ - void addRemove(const document::DocumentId& docId, uint64_t timestamp); - - /** - * Adds an operation to be performed. Optionally deep-clones the - * operation's document. - */ - void addOperation(const Operation& op, bool cloneDocument = false); - - /** - Returns the number of operations in this batch. - */ - uint32_t getOperationCount() const { return _operations.size(); } - - /** - Returns the nth operation in this batch. - */ - const Operation& getOperation(uint32_t index) const { return *_operations[index]; } - - /** - Returns the nth operation in this batch. - */ - Operation& getOperation(uint32_t index) { return *_operations[index]; } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - DECLARE_STORAGECOMMAND(BatchPutRemoveCommand, onBatchPutRemove) - -private: - std::vector<std::unique_ptr<Operation> > _operations; - uint32_t _approxSize; -}; - -/** - * @class BatchPutRemoveReply - * @ingroup message - * - * @brief Confirm that a given docoperations have been received. - */ -class BatchPutRemoveReply : public BucketInfoReply { -private: - std::vector<document::DocumentId> _documentsNotFound; - -public: - explicit BatchPutRemoveReply(const BatchPutRemoveCommand&); - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - const std::vector<document::DocumentId>& getDocumentsNotFound() const { return _documentsNotFound; } - std::vector<document::DocumentId>& getDocumentsNotFound() { return _documentsNotFound; } - - DECLARE_STORAGEREPLY(BatchPutRemoveReply, onBatchPutRemoveReply) -}; - -class BatchDocumentUpdateCommand : public StorageCommand -{ -public: - typedef std::vector<document::DocumentUpdate::SP > UpdateList; - - /** - Creates a batch update message containing the given updates. - */ - BatchDocumentUpdateCommand(const UpdateList& updates); - - /** - @return Returns a list of the updates to be performed. - */ - const UpdateList& getUpdates() const { return _updates; }; - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - /** - Returns a bucket suitable for routing this message. - */ - document::Bucket getBucket() const override { return _bucket; } - bool hasSingleBucketId() const override { return true; } - - DECLARE_STORAGECOMMAND(BatchDocumentUpdateCommand, onBatchDocumentUpdate) - -private: - UpdateList _updates; - document::Bucket _bucket; -}; - -/** - * @class BatchDocumentUpdateReply - * @ingroup message - * - * @brief Confirm that a given docoperations have been received. - */ -class BatchDocumentUpdateReply : public StorageReply { - // 1-1 mapping of found/not found state for documents - std::vector<bool> _documentsNotFound; -public: - explicit BatchDocumentUpdateReply(const BatchDocumentUpdateCommand&); - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - const std::vector<bool>& getDocumentsNotFound() const { return _documentsNotFound; } - std::vector<bool>& getDocumentsNotFound() { return _documentsNotFound; } - - DECLARE_STORAGEREPLY(BatchDocumentUpdateReply, onBatchDocumentUpdateReply) -}; - -} diff --git a/storageapi/src/vespa/storageapi/messageapi/messagehandler.h b/storageapi/src/vespa/storageapi/messageapi/messagehandler.h index e7bd2b5bf27..a9c1dfb8f26 100644 --- a/storageapi/src/vespa/storageapi/messageapi/messagehandler.h +++ b/storageapi/src/vespa/storageapi/messageapi/messagehandler.h @@ -25,8 +25,6 @@ class PutCommand; // Add document class UpdateCommand; // Update document class RemoveCommand; // Remove document class RevertCommand; // Revert put/remove operation -class BatchPutRemoveCommand; -class BatchDocumentUpdateCommand; class CreateVisitorCommand; // Create a new visitor class DestroyVisitorCommand; // Destroy a running visitor @@ -64,8 +62,6 @@ class PutReply; class UpdateReply; class RemoveReply; class RevertReply; -class BatchPutRemoveReply; -class BatchDocumentUpdateReply; class CreateVisitorReply; class DestroyVisitorReply; @@ -143,18 +139,6 @@ public: { return false; } virtual bool onRevertReply(const std::shared_ptr<api::RevertReply>&) { return false; } - virtual bool onBatchPutRemove( - const std::shared_ptr<api::BatchPutRemoveCommand>&) - { return false; } - virtual bool onBatchPutRemoveReply( - const std::shared_ptr<api::BatchPutRemoveReply>&) - { return false; } - virtual bool onBatchDocumentUpdate( - const std::shared_ptr<api::BatchDocumentUpdateCommand>&) - { return false; } - virtual bool onBatchDocumentUpdateReply( - const std::shared_ptr<api::BatchDocumentUpdateReply>&) - { return false; } // Visiting virtual bool onCreateVisitor( diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp index 563a2aab7c1..bab475eea32 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp +++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp @@ -105,10 +105,6 @@ const MessageType MessageType::REMOVELOCATION("Removelocation", REMOVELOCATION_I const MessageType MessageType::REMOVELOCATION_REPLY("Removelocation Reply", REMOVELOCATION_REPLY_ID, &MessageType::REMOVELOCATION); const MessageType MessageType::QUERYRESULT("QueryResult", QUERYRESULT_ID); const MessageType MessageType::QUERYRESULT_REPLY("QueryResult reply", QUERYRESULT_REPLY_ID, &MessageType::QUERYRESULT); -const MessageType MessageType::BATCHPUTREMOVE("BatchPutRemove", BATCHPUTREMOVE_ID); -const MessageType MessageType::BATCHPUTREMOVE_REPLY("BatchPutRemove reply", BATCHPUTREMOVE_REPLY_ID, &MessageType::BATCHPUTREMOVE); -const MessageType MessageType::BATCHDOCUMENTUPDATE("BatchDocumentUpdate", BATCHDOCUMENTUPDATE_ID); -const MessageType MessageType::BATCHDOCUMENTUPDATE_REPLY("BatchDocumentUpdate reply", BATCHDOCUMENTUPDATE_REPLY_ID, &MessageType::BATCHDOCUMENTUPDATE); const MessageType MessageType::SETBUCKETSTATE("SetBucketState", SETBUCKETSTATE_ID); const MessageType MessageType::SETBUCKETSTATE_REPLY("SetBucketStateReply", SETBUCKETSTATE_REPLY_ID, &MessageType::SETBUCKETSTATE); diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h index c81ee51ae22..c9f6e737a47 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h +++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h @@ -147,10 +147,6 @@ public: REMOVELOCATION_REPLY_ID = 87, QUERYRESULT_ID = 88, QUERYRESULT_REPLY_ID = 89, - BATCHPUTREMOVE_ID = 90, - BATCHPUTREMOVE_REPLY_ID = 91, - BATCHDOCUMENTUPDATE_ID = 92, - BATCHDOCUMENTUPDATE_REPLY_ID = 93, SETBUCKETSTATE_ID = 94, SETBUCKETSTATE_REPLY_ID = 95, MESSAGETYPE_MAX_ID @@ -233,10 +229,6 @@ public: static const MessageType REMOVELOCATION_REPLY; static const MessageType QUERYRESULT; static const MessageType QUERYRESULT_REPLY; - static const MessageType BATCHPUTREMOVE; - static const MessageType BATCHPUTREMOVE_REPLY; - static const MessageType BATCHDOCUMENTUPDATE; - static const MessageType BATCHDOCUMENTUPDATE_REPLY; static const MessageType SETBUCKETSTATE; static const MessageType SETBUCKETSTATE_REPLY; diff --git a/vespabase/src/start-cbinaries.sh b/vespabase/src/start-cbinaries.sh index e61a3d8c8b1..02eac2f42bc 100755 --- a/vespabase/src/start-cbinaries.sh +++ b/vespabase/src/start-cbinaries.sh @@ -92,7 +92,7 @@ check_bname_in_value () { configure_valgrind () { no_valgrind=true - if which valgrind >/dev/null; then + if which valgrind >/dev/null 2>&1; then if check_bname_in_value $VESPA_USE_VALGRIND; then no_valgrind=false valgrind_log=$VESPA_HOME/tmp/valgrind.$bname.log.$$ diff --git a/vespajlib/pom.xml b/vespajlib/pom.xml index 5b9c143a447..741b4b58cc9 100644 --- a/vespajlib/pom.xml +++ b/vespajlib/pom.xml @@ -31,6 +31,10 @@ <groupId>org.apache.commons</groupId> <artifactId>commons-exec</artifactId> </dependency> + <dependency> + <groupId>net.java.dev.jna</groupId> + <artifactId>jna</artifactId> + </dependency> <!-- provided scope --> diff --git a/vespajlib/src/main/java/com/yahoo/io/NativeIO.java b/vespajlib/src/main/java/com/yahoo/io/NativeIO.java new file mode 100644 index 00000000000..f08e01070d7 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/io/NativeIO.java @@ -0,0 +1,104 @@ +package com.yahoo.io; + +import java.io.File; +import java.io.FileDescriptor; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.SyncFailedException; +import java.lang.reflect.Field; +import java.util.logging.Logger; + +import com.sun.jna.LastErrorException; +import com.sun.jna.Native; +import com.sun.jna.Platform; + +/** + * Provides functionality only possible through native C library. + */ +public class NativeIO { + private final static Logger logger = Logger.getLogger(NativeIO.class.getName()); + private static final int POSIX_FADV_DONTNEED = 4; // See /usr/include/linux/fadvise.h + private static boolean initialized = false; + private static Throwable initError = null; + static { + try { + if (Platform.isLinux()) { + Native.register(Platform.C_LIBRARY_NAME); + initialized = true; + } + } catch (Throwable throwable) { + initError = throwable; + } + } + + private static final Field fieldFD = getField(FileDescriptor.class, "fd"); + + + private static native int posix_fadvise(int fd, long offset, long len, int flag) throws LastErrorException; + + public NativeIO() { + if (!initialized) { + logger.warning("native IO not possible due to " + getError().getMessage()); + } + } + + public boolean valid() { return initialized; } + public Throwable getError() { + if (initError != null) { + return initError; + } else { + return new RuntimeException("Platform is unsúpported. Only supported on linux."); + } + } + + /** + * Will hint the OS that this is will not be accessed again and should hence be dropped from the buffer cache. + * @param fd The file descriptor to drop from buffer cache. + */ + public void dropFileFromCache(FileDescriptor fd) { + try { + fd.sync(); + } catch (SyncFailedException e) { + logger.warning("Sync failed while dropping cache: " + e.getMessage()); + } + if (initialized) { + posix_fadvise(getNativeFD(fd), 0, 0, POSIX_FADV_DONTNEED); + } + } + + /** + * Will hint the OS that this is will not be accessed again and should hence be dropped from the buffer cache. + * @param file File to drop from buffer cache + */ + public void dropFileFromCache(File file) { + try { + dropFileFromCache(new FileInputStream(file).getFD()); + } catch (FileNotFoundException e) { + logger.warning("No point in dropping a non-existing file from the buffer cache: " + e.getMessage()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + + private static Field getField(Class<?> clazz, String fieldName) { + Field field; + try { + field = clazz.getDeclaredField(fieldName); + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); + } + field.setAccessible(true); + return field; + } + + private static int getNativeFD(FileDescriptor fd) { + try { + return fieldFD.getInt(fd); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/vespajlib/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java b/vespajlib/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java new file mode 100644 index 00000000000..5595d33a9b5 --- /dev/null +++ b/vespajlib/src/main/java/com/yahoo/security/tls/TransportSecurityUtils.java @@ -0,0 +1,66 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.security.tls; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Optional; + +/** + * Utility class for retrieving {@link TransportSecurityOptions} from the system. + * + * @author bjorncs + */ +public class TransportSecurityUtils { + + public static final String CONFIG_FILE_ENVIRONMENT_VARIABLE = "VESPA_TLS_CONFIG_FILE"; + public static final String INSECURE_MIXED_MODE_ENVIRONMENT_VARIABLE = "VESPA_TLS_INSECURE_MIXED_MODE"; + + public enum MixedMode { + PLAINTEXT_CLIENT_MIXED_SERVER("plaintext_client_mixed_server"), + TLS_CLIENT_MIXED_SERVER("tls_client_mixed_server"); + + final String configValue; + + MixedMode(String configValue) { + this.configValue = configValue; + } + + static MixedMode fromConfigValue(String configValue) { + return Arrays.stream(values()) + .filter(v -> v.configValue.equals(configValue)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("Unknown value: " + configValue)); + } + } + + private TransportSecurityUtils() {} + + public static boolean isTransportSecurityEnabled() { + return getConfigFile().isPresent(); + } + + public static boolean isInsecureMixedModeEnabled() { + return getInsecureMixedMode().isPresent(); + } + + public static Optional<MixedMode> getInsecureMixedMode() { + if (!isTransportSecurityEnabled()) return Optional.empty(); + return getEnvironmentVariable(INSECURE_MIXED_MODE_ENVIRONMENT_VARIABLE) + .map(MixedMode::fromConfigValue); + } + + public static Optional<Path> getConfigFile() { + return getEnvironmentVariable(CONFIG_FILE_ENVIRONMENT_VARIABLE).map(Paths::get); + } + + public static Optional<TransportSecurityOptions> getOptions() { + return getConfigFile() + .map(TransportSecurityOptions::fromJsonFile); + } + + private static Optional<String> getEnvironmentVariable(String environmentVariable) { + return Optional.ofNullable(System.getenv(environmentVariable)) + .filter(var -> !var.isEmpty()); + } +} diff --git a/vespajlib/src/test/java/com/yahoo/io/NativeIOTestCase.java b/vespajlib/src/test/java/com/yahoo/io/NativeIOTestCase.java new file mode 100644 index 00000000000..6018a195b6a --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/io/NativeIOTestCase.java @@ -0,0 +1,35 @@ +package com.yahoo.io; + +import com.sun.jna.Platform; +import org.junit.Test; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +public class NativeIOTestCase { + + @Test + public void requireThatDropFileFromCacheDoesNotThrow() throws IOException { + File testFile = new File("testfile"); + FileOutputStream output = new FileOutputStream(testFile); + output.write('t'); + output.flush(); + output.close(); + NativeIO nativeIO = new NativeIO(); + if (Platform.isLinux()) { + assertTrue(nativeIO.valid()); + } else { + assertFalse(nativeIO.valid()); + assertEquals("Platform is unsúpported. Only supported on linux.", nativeIO.getError().getMessage()); + } + nativeIO.dropFileFromCache(output.getFD()); + nativeIO.dropFileFromCache(testFile); + testFile.delete(); + nativeIO.dropFileFromCache(new File("file.that.does.not.exist")); + } +} diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index b32c875cb26..663d281179d 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -12,6 +12,7 @@ vespa_define_module( src/apps/make_fixture_macros src/apps/vespa-detect-hostname src/apps/vespa-validate-hostname + src/apps/vespa-drop-file-from-cache TESTS src/tests/alignedmemory @@ -36,6 +37,7 @@ vespa_define_module( src/tests/data/smart_buffer src/tests/delegatelist src/tests/dotproduct + src/tests/drop-file-from-cache src/tests/dual_merge_director src/tests/eventbarrier src/tests/exception_classes @@ -59,6 +61,7 @@ vespa_define_module( src/tests/net/socket_spec src/tests/net/tls/direct_buffer_bio src/tests/net/tls/openssl_impl + src/tests/net/tls/protocol_snooping src/tests/net/tls/transport_options src/tests/objects/nbostream src/tests/optimized diff --git a/vespalib/src/apps/vespa-drop-file-from-cache/.gitignore b/vespalib/src/apps/vespa-drop-file-from-cache/.gitignore new file mode 100644 index 00000000000..24dc558b6bc --- /dev/null +++ b/vespalib/src/apps/vespa-drop-file-from-cache/.gitignore @@ -0,0 +1 @@ +vespa-drop-file-from-cache diff --git a/vespalib/src/apps/vespa-drop-file-from-cache/CMakeLists.txt b/vespalib/src/apps/vespa-drop-file-from-cache/CMakeLists.txt new file mode 100644 index 00000000000..409fd5bb725 --- /dev/null +++ b/vespalib/src/apps/vespa-drop-file-from-cache/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_vespa-drop-file-from-cache_app + SOURCES + drop_file_from_cache.cpp + OUTPUT_NAME vespa-drop-file-from-cache + INSTALL bin + DEPENDS + vespalib +) diff --git a/vespalib/src/apps/vespa-drop-file-from-cache/drop_file_from_cache.cpp b/vespalib/src/apps/vespa-drop-file-from-cache/drop_file_from_cache.cpp new file mode 100644 index 00000000000..042681a90fe --- /dev/null +++ b/vespalib/src/apps/vespa-drop-file-from-cache/drop_file_from_cache.cpp @@ -0,0 +1,32 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <cstdio> +#include <cerrno> +#include <cstring> +#include <fcntl.h> +#include <unistd.h> + +int main(int argc, char **argv) { + char errorBuf[200]; + if (argc != 2) { + fprintf(stderr, "%s <filename>\n", argv[0]); + return 1; + } + const char *fileName = argv[1]; + int fh = open(fileName, O_RDONLY); + if (fh == -1) { + const char *errorString = strerror_r(errno, errorBuf, sizeof(errorBuf)); + fprintf(stderr, "Failed opening file %s: %s\n", fileName, errorString); + return 2; + } + + int retval = 0; + int err = posix_fadvise(fh, 0, 0, POSIX_FADV_DONTNEED); + if (err != 0) { + const char *errorString = strerror_r(errno, errorBuf, sizeof(errorBuf)); + fprintf(stderr, "posix_fadvise failed: %s\n", errorString); + retval = 3; + } + close(fh); + return retval; +} diff --git a/vespalib/src/tests/drop-file-from-cache/CMakeLists.txt b/vespalib/src/tests/drop-file-from-cache/CMakeLists.txt new file mode 100644 index 00000000000..d6f4a51a4c5 --- /dev/null +++ b/vespalib/src/tests/drop-file-from-cache/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_drop_file_from_cache_test_app TEST + SOURCES + drop_file_from_cache_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_drop_file_from_cache_test_app COMMAND vespalib_drop_file_from_cache_test_app) diff --git a/vespalib/src/tests/drop-file-from-cache/drop_file_from_cache_test.cpp b/vespalib/src/tests/drop-file-from-cache/drop_file_from_cache_test.cpp new file mode 100644 index 00000000000..63defa58c41 --- /dev/null +++ b/vespalib/src/tests/drop-file-from-cache/drop_file_from_cache_test.cpp @@ -0,0 +1,25 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/util/slaveproc.h> + +using vespalib::SlaveProc; + +TEST("no arguments") { + SlaveProc drop("../../apps/vespa-drop-file-from-cache/vespa-drop-file-from-cache"); + drop.wait(); + EXPECT_EQUAL(1, drop.getExitCode()); +} + +TEST("file does not exist") { + SlaveProc drop("../../apps/vespa-drop-file-from-cache/vespa-drop-file-from-cache not_exist"); + drop.wait(); + EXPECT_EQUAL(2, drop.getExitCode()); +} + +TEST("All is well") { + SlaveProc drop("../../apps/vespa-drop-file-from-cache/vespa-drop-file-from-cache vespalib_drop_file_from_cache_test_app"); + drop.wait(); + EXPECT_EQUAL(0, drop.getExitCode()); +} + +TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/net/tls/protocol_snooping/CMakeLists.txt b/vespalib/src/tests/net/tls/protocol_snooping/CMakeLists.txt new file mode 100644 index 00000000000..1489859fe48 --- /dev/null +++ b/vespalib/src/tests/net/tls/protocol_snooping/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_net_tls_protocol_snooping_test_app TEST + SOURCES + protocol_snooping_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_net_tls_protocol_snooping_test_app COMMAND vespalib_net_tls_protocol_snooping_test_app) + diff --git a/vespalib/src/tests/net/tls/protocol_snooping/protocol_snooping_test.cpp b/vespalib/src/tests/net/tls/protocol_snooping/protocol_snooping_test.cpp new file mode 100644 index 00000000000..2d203047835 --- /dev/null +++ b/vespalib/src/tests/net/tls/protocol_snooping/protocol_snooping_test.cpp @@ -0,0 +1,78 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/net/tls/protocol_snooping.h> + +using namespace vespalib; +using namespace vespalib::net::tls::snooping; + +TEST("min_header_bytes_to_observe() is 8") { + EXPECT_EQUAL(8u, min_header_bytes_to_observe()); +} + +TlsSnoopingResult do_snoop(const unsigned char* buf) { + return snoop_client_hello_header(reinterpret_cast<const char*>(buf)); +} + +TEST("Well-formed TLSv1.0 packet returns ProbablyTls") { + const unsigned char buf[] = { 22, 3, 1, 10, 255, 1, 0, 10 }; + EXPECT_EQUAL(TlsSnoopingResult::ProbablyTls, do_snoop(buf)); +} + +TEST("Well-formed TLSv1.2 packet returns ProbablyTls") { + const unsigned char buf[] = { 22, 3, 3, 10, 255, 1, 0, 10 }; + EXPECT_EQUAL(TlsSnoopingResult::ProbablyTls, do_snoop(buf)); +} + +TEST("Mismatching handshake header byte 1 returns HandshakeMismatch") { + const unsigned char buf[] = { 23, 3, 1, 10, 255, 1, 0, 10 }; + EXPECT_EQUAL(TlsSnoopingResult::HandshakeMismatch, do_snoop(buf)); +} + +TEST("Mismatching major version byte returns ProtocolVersionMismatch") { + const unsigned char buf[] = { 22, 2, 1, 10, 255, 1, 0, 10 }; + EXPECT_EQUAL(TlsSnoopingResult::ProtocolVersionMismatch, do_snoop(buf)); +} + +TEST("Mismatching minor version byte returns ProtocolVersionMismatch") { + const unsigned char buf[] = { 22, 3, 0, 10, 255, 1, 0, 10 }; + EXPECT_EQUAL(TlsSnoopingResult::ProtocolVersionMismatch, do_snoop(buf)); +} + +TEST("Oversized record returns RecordSizeRfcViolation") { + const unsigned char buf1[] = { 22, 3, 1, 255, 255, 1, 0, 10 }; // 64k + // ^^^^^^^^ big endian record length + EXPECT_EQUAL(TlsSnoopingResult::RecordSizeRfcViolation, do_snoop(buf1)); + + const unsigned char buf2[] = { 22, 3, 1, 72, 1, 1, 0, 10 }; // 18K+1 + EXPECT_EQUAL(TlsSnoopingResult::RecordSizeRfcViolation, do_snoop(buf2)); +} + +TEST("Undersized record returns RecordSizeRfcViolation") { + const unsigned char buf1[] = { 22, 3, 1, 0, 3, 1, 0, 0 }; + EXPECT_EQUAL(TlsSnoopingResult::RecordSizeRfcViolation, do_snoop(buf1)); +} + +TEST("Non-ClientHello handshake record returns RecordNotClientHello") { + const unsigned char buf[] = { 22, 3, 1, 10, 255, 2, 0, 10 }; + // ^ 1 == ClientHello + EXPECT_EQUAL(TlsSnoopingResult::RecordNotClientHello, do_snoop(buf)); +} + +TEST("Oversized or fragmented ClientHello record returns ClientHelloRecordTooBig") { + const unsigned char buf[] = { 22, 3, 1, 10, 255, 1, 1, 10 }; + // ^ MSB of 24-bit record length + EXPECT_EQUAL(TlsSnoopingResult::ClientHelloRecordTooBig, do_snoop(buf)); +} + +TEST("Expected ClientHello record size mismatch returns ExpectedRecordSizeMismatch") { + const unsigned char buf[] = { 22, 3, 1, 10, 2, 1, 0, 10 }; + // ^^ bits [8,16) of record length, should be 9 + EXPECT_EQUAL(TlsSnoopingResult::ExpectedRecordSizeMismatch, do_snoop(buf)); +} + +TEST("Valid ClientHello record size with LSB < 4 returns ProbablyTls") { + const unsigned char buf[] = { 22, 3, 1, 10, 3, 1, 0, 9 }; + EXPECT_EQUAL(TlsSnoopingResult::ProbablyTls, do_snoop(buf)); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt b/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt index 2d34a3e1c80..17abd82366d 100644 --- a/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/net/tls/CMakeLists.txt @@ -4,6 +4,7 @@ vespa_add_library(vespalib_vespalib_net_tls OBJECT crypto_codec.cpp crypto_codec_adapter.cpp crypto_exception.cpp + protocol_snooping.cpp tls_context.cpp tls_crypto_engine.cpp transport_security_options.cpp diff --git a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h index 4c253fdf24c..53693dd7f40 100644 --- a/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h +++ b/vespalib/src/vespa/vespalib/net/tls/impl/openssl_crypto_codec_impl.h @@ -11,10 +11,8 @@ namespace vespalib::net::tls { class TlsContext; } namespace vespalib::net::tls::impl { /* - * Frame-level OpenSSL-backed TLSv1.2 crypto codec implementation. - * - * Currently has sub-optimal buffer management, and is mostly intended - * as a starting point. + * Frame-level OpenSSL-backed TLSv1.2/TLSv1.3 (depending on OpenSSL version) + * crypto codec implementation. * * NOT thread safe per instance, but independent instances may be * used by different threads safely. diff --git a/vespalib/src/vespa/vespalib/net/tls/protocol_snooping.cpp b/vespalib/src/vespa/vespalib/net/tls/protocol_snooping.cpp new file mode 100644 index 00000000000..6ccfc7f27ac --- /dev/null +++ b/vespalib/src/vespa/vespalib/net/tls/protocol_snooping.cpp @@ -0,0 +1,127 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "protocol_snooping.h" +#include <iostream> +#include <cstdlib> +#include <stdint.h> + +namespace vespalib::net::tls::snooping { + +namespace { + +// Precondition for all helper functions: buffer is at least `min_header_bytes_to_observe()` bytes long + +// From RFC 5246: +// 0x16 - Handshake content type byte of TLSCiphertext record +inline bool is_tls_handshake_packet(const char* buf) { + return (buf[0] == 0x16); +} + +// First byte of 2-byte ProtocolVersion, always 3 on TLSv1.2 and v1.3 +// Next is the TLS minor version, either 1 or 3 depending on version (though the +// RFCs say it _should_ be 1 for backwards compatibility reasons). +// Yes, the TLS spec says that you should technically ignore the protocol version +// field here, but we want all the signals we can get. +inline bool is_expected_tls_protocol_version(const char* buf) { + return ((buf[1] == 0x03) && ((buf[2] == 0x01) || (buf[2] == 0x03))); +} + +// Length is big endian u16 in bytes 3, 4 +inline uint16_t tls_record_length(const char* buf) { + return (uint16_t(static_cast<unsigned char>(buf[3]) << 8) + + static_cast<unsigned char>(buf[4])); +} + +// First byte of Handshake record in byte 5, which shall be ClientHello (0x01) +inline bool is_client_hello_handshake_record(const char* buf) { + return (buf[5] == 0x01); +} + +// Last 2 bytes are the 2 first big-endian bytes of a 3-byte Handshake +// record length field. No support for records that are large enough that +// the MSB should ever be non-zero. +inline bool client_hello_record_size_within_expected_bounds(const char* buf) { + return (buf[6] == 0x00); +} + +// The byte after the MSB of the 24-bit handshake record size should be equal +// to the most significant byte of the record length value, minus the Handshake +// record header size. +// Again, we make the assumption that ClientHello messages are not fragmented, +// so their max size must be <= 16KiB. This also just happens to be a lower +// number than the minimum FS4/FRT packet type byte at the same location. +// Oooh yeah, leaky abstractions to the rescue! +inline bool handshake_record_size_matches_length(const char* buf, uint16_t length) { + return (static_cast<unsigned char>(buf[7]) == ((length - 4) >> 8)); +} + +} // anon ns + +TlsSnoopingResult snoop_client_hello_header(const char* buf) noexcept { + if (!is_tls_handshake_packet(buf)) { + return TlsSnoopingResult::HandshakeMismatch; + } + if (!is_expected_tls_protocol_version(buf)) { + return TlsSnoopingResult::ProtocolVersionMismatch; + } + // Length of TLS record follows. Must be <= 16KiB + 2048 (16KiB + 256 on v1.3). + // We expect that the first record contains _only_ a ClientHello with no coalescing + // and no fragmentation. This is technically a violation of the TLS spec, but this + // particular detection logic is only intended to be used against other Vespa nodes + // where we control frame sizes and where such fragmentation should not take place. + // We also do not support TLSv1.3 0-RTT which may trigger early data. + uint16_t length = tls_record_length(buf); + if ((length < 4) || (length > (16384 + 2048))) { + return TlsSnoopingResult::RecordSizeRfcViolation; + } + if (!is_client_hello_handshake_record(buf)) { + return TlsSnoopingResult::RecordNotClientHello; + } + if (!client_hello_record_size_within_expected_bounds(buf)) { + return TlsSnoopingResult::ClientHelloRecordTooBig; + } + if (!handshake_record_size_matches_length(buf, length)) { + return TlsSnoopingResult::ExpectedRecordSizeMismatch; + } + // Hooray! It very probably most likely is a TLS connection! :D + return TlsSnoopingResult::ProbablyTls; +} + +const char* to_string(TlsSnoopingResult result) noexcept { + switch (result) { + case TlsSnoopingResult::ProbablyTls: return "ProbablyTls"; + case TlsSnoopingResult::HandshakeMismatch: return "HandshakeMismatch"; + case TlsSnoopingResult::ProtocolVersionMismatch: return "ProtocolVersionMismatch"; + case TlsSnoopingResult::RecordSizeRfcViolation: return "RecordSizeRfcViolation"; + case TlsSnoopingResult::RecordNotClientHello: return "RecordNotClientHello"; + case TlsSnoopingResult::ClientHelloRecordTooBig: return "ClientHelloRecordTooBig"; + case TlsSnoopingResult::ExpectedRecordSizeMismatch: return "ExpectedRecordSizeMismatch"; + } + abort(); +} + +std::ostream& operator<<(std::ostream& os, TlsSnoopingResult result) { + os << to_string(result); + return os; +} + +const char* describe_result(TlsSnoopingResult result) noexcept { + switch (result) { + case TlsSnoopingResult::ProbablyTls: + return "client data matches TLS heuristics, very likely a TLS connection"; + case TlsSnoopingResult::HandshakeMismatch: + return "not a TLS handshake packet"; + case TlsSnoopingResult::ProtocolVersionMismatch: + return "ProtocolVersion mismatch"; + case TlsSnoopingResult::RecordSizeRfcViolation: + return "ClientHello record size is greater than RFC allows"; + case TlsSnoopingResult::RecordNotClientHello: + return "record is not ClientHello"; + case TlsSnoopingResult::ClientHelloRecordTooBig: + return "ClientHello record is too big (fragmented?)"; + case TlsSnoopingResult::ExpectedRecordSizeMismatch: + return "ClientHello vs Handshake header record size mismatch"; + } + abort(); +} + +} diff --git a/vespalib/src/vespa/vespalib/net/tls/protocol_snooping.h b/vespalib/src/vespa/vespalib/net/tls/protocol_snooping.h new file mode 100644 index 00000000000..f53e136e597 --- /dev/null +++ b/vespalib/src/vespa/vespalib/net/tls/protocol_snooping.h @@ -0,0 +1,32 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <iosfwd> +#include <stddef.h> + +namespace vespalib::net::tls::snooping { + +constexpr inline size_t min_header_bytes_to_observe() { return 8; } + +enum class TlsSnoopingResult { + ProbablyTls, // Very safe to assume TLSv1.x client + HandshakeMismatch, // Almost guaranteed to trigger for plaintext RPC + ProtocolVersionMismatch, + RecordSizeRfcViolation, + RecordNotClientHello, + ClientHelloRecordTooBig, + ExpectedRecordSizeMismatch +}; + +const char* to_string(TlsSnoopingResult) noexcept; +std::ostream& operator<<(std::ostream& os, TlsSnoopingResult); + +// Precondition: buf is at least `min_header_bytes_to_observe()` bytes long. This is the minimum amount +// of bytes always sent for a packet in our existing plaintext production protocols and +// therefore the maximum we can expect to always be present. +// Yes, this is a pragmatic and delightfully leaky abstraction. +TlsSnoopingResult snoop_client_hello_header(const char* buf) noexcept; + +const char* describe_result(TlsSnoopingResult result) noexcept; + +} diff --git a/vespalib/src/vespa/vespalib/util/slaveproc.h b/vespalib/src/vespa/vespalib/util/slaveproc.h index 72b20cfb069..c08a13f0b1d 100644 --- a/vespalib/src/vespa/vespalib/util/slaveproc.h +++ b/vespalib/src/vespa/vespalib/util/slaveproc.h @@ -8,14 +8,9 @@ #include <queue> #include "sync.h" -namespace vespalib { - -namespace slaveproc { - -class Timer; - -} // namespace slaveproc +namespace vespalib::slaveproc { class Timer; } +namespace vespalib { /** * @brief Slave Process utility class for running external programs * @@ -56,12 +51,12 @@ private: bool _failed; int _exitCode; - SlaveProc(const SlaveProc &); // no copy - SlaveProc &operator=(const SlaveProc &); // no assignment - void checkProc(); public: + SlaveProc(const SlaveProc &) = delete; + SlaveProc &operator=(const SlaveProc &) = delete; + /** * @brief Run a slave process * |