diff options
author | Jon Marius Venstad <venstad@gmail.com> | 2021-01-27 09:05:04 +0100 |
---|---|---|
committer | Jon Marius Venstad <venstad@gmail.com> | 2021-01-27 09:05:04 +0100 |
commit | 9c25a72c7f18808d8a1e0efdc96916ad9c598fe0 (patch) | |
tree | 7f64b3acaf5ae13d7853904524195f1f282b3b70 /documentapi | |
parent | 9c33b487fb942394ad945e854a877a9de0b7aff4 (diff) |
Revert "Revert "Jonmv/document protocol super config""
This reverts commit d7359f7c72ff06889af594431baf4075e2b4da78.
Diffstat (limited to 'documentapi')
12 files changed, 163 insertions, 104 deletions
diff --git a/documentapi/abi-spec.json b/documentapi/abi-spec.json index 39d6898215c..2e04487f93d 100644 --- a/documentapi/abi-spec.json +++ b/documentapi/abi-spec.json @@ -1128,6 +1128,7 @@ "public com.yahoo.documentapi.messagebus.MessageBusParams setRoutingConfigId(java.lang.String)", "public java.lang.String getProtocolConfigId()", "public com.yahoo.documentapi.messagebus.MessageBusParams setProtocolConfigId(java.lang.String)", + "public com.yahoo.documentapi.messagebus.MessageBusParams setDocumentProtocolPoliciesConfig(com.yahoo.messagebus.documentapi.DocumentProtocolPoliciesConfig, com.yahoo.vespa.config.content.DistributionConfig)", "public com.yahoo.documentapi.messagebus.MessageBusParams setRouteName(java.lang.String)", "public com.yahoo.documentapi.messagebus.MessageBusParams setRoute(java.lang.String)", "public com.yahoo.documentapi.messagebus.MessageBusParams setRouteNameForGet(java.lang.String)", @@ -1573,6 +1574,7 @@ "fields": [ "protected final java.lang.String clusterName", "protected final java.lang.String distributionConfigId", + "protected final com.yahoo.vespa.config.content.DistributionConfig distributionConfig", "protected final com.yahoo.documentapi.messagebus.protocol.ContentPolicy$SlobrokHostPatternGenerator slobrokHostPatternGenerator" ] }, @@ -1595,8 +1597,7 @@ "public" ], "methods": [ - "public void <init>(java.lang.String)", - "public void <init>(java.util.Map)", + "public void <init>(java.lang.String, com.yahoo.vespa.config.content.DistributionConfig)", "public void <init>(com.yahoo.documentapi.messagebus.protocol.ContentPolicy$Parameters)", "public void select(com.yahoo.messagebus.routing.RoutingContext)", "public void merge(com.yahoo.messagebus.routing.RoutingContext)", @@ -1806,6 +1807,7 @@ "public static com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority getPriorityByName(java.lang.String)", "public void <init>(com.yahoo.document.DocumentTypeManager)", "public void <init>(com.yahoo.document.DocumentTypeManager, java.lang.String)", + "public void <init>(com.yahoo.document.DocumentTypeManager, com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet, com.yahoo.messagebus.documentapi.DocumentProtocolPoliciesConfig, com.yahoo.vespa.config.content.DistributionConfig)", "public void <init>(com.yahoo.document.DocumentTypeManager, java.lang.String, com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet)", "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocol putRoutingPolicyFactory(java.lang.String, com.yahoo.documentapi.messagebus.protocol.RoutingPolicyFactory)", "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocol putRoutableFactory(int, com.yahoo.documentapi.messagebus.protocol.RoutableFactory, com.yahoo.component.VersionSpecification)", @@ -1928,6 +1930,7 @@ "public" ], "methods": [ + "public void <init>(com.yahoo.messagebus.documentapi.DocumentProtocolPoliciesConfig)", "public void <init>(java.lang.String)", "public synchronized java.lang.String getError()", "public void configure(com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig)", @@ -2952,18 +2955,6 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutingPolicyFactories": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "abstract" - ], - "methods": [ - "public void <init>()" - ], - "fields": [] - }, "com.yahoo.documentapi.messagebus.protocol.RoutingPolicyFactory": { "superClass": "java.lang.Object", "interfaces": [], @@ -2973,8 +2964,7 @@ "abstract" ], "methods": [ - "public abstract com.yahoo.documentapi.messagebus.protocol.DocumentProtocolRoutingPolicy createPolicy(java.lang.String)", - "public abstract void destroy()" + "public abstract com.yahoo.documentapi.messagebus.protocol.DocumentProtocolRoutingPolicy createPolicy(java.lang.String)" ], "fields": [] }, diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusParams.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusParams.java index a838f3b8723..824b1144a67 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusParams.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusParams.java @@ -3,8 +3,15 @@ package com.yahoo.documentapi.messagebus; import com.yahoo.documentapi.DocumentAccessParams; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.messagebus.SourceSessionParams; +import com.yahoo.messagebus.documentapi.DocumentProtocolPoliciesConfig; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; +import com.yahoo.vespa.config.content.DistributionConfig; + +import java.util.Objects; + +import static java.util.Objects.requireNonNull; /** * @author Einar M R Rosenvinge @@ -13,6 +20,8 @@ public class MessageBusParams extends DocumentAccessParams { private String routingConfigId = null; private String protocolConfigId = null; + private DocumentProtocolPoliciesConfig policiesConfig = null; + private DistributionConfig distributionConfig = null; private String route = "route:default"; private String routeForGet = "route:default-get"; private int traceLevel = 0; @@ -79,6 +88,14 @@ public class MessageBusParams extends DocumentAccessParams { return this; } + /** Sets the config used by the {@link DocumentProtocol} policies. */ + public MessageBusParams setDocumentProtocolPoliciesConfig(DocumentProtocolPoliciesConfig policiesConfig, + DistributionConfig distributionConfig) { + this.policiesConfig = requireNonNull(policiesConfig); + this.distributionConfig = requireNonNull(distributionConfig); + return this; + } + /** * Sets the name of the route to send appropriate requests to. This is a convenience method for prefixing a route * with "route:", and using {@link #setRoute} instead. diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ContentPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ContentPolicy.java index c03231543df..f8e6989bbfa 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ContentPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ContentPolicy.java @@ -6,7 +6,6 @@ import com.yahoo.document.BucketId; import com.yahoo.document.BucketIdFactory; import com.yahoo.jrt.slobrok.api.IMirror; import com.yahoo.jrt.slobrok.api.Mirror; -import java.util.logging.Level; import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; import com.yahoo.messagebus.ErrorCode; @@ -22,6 +21,7 @@ import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; +import com.yahoo.vespa.config.content.DistributionConfig; import java.util.ArrayList; import java.util.Collections; @@ -32,6 +32,7 @@ import java.util.Random; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -221,18 +222,28 @@ public class ContentPolicy extends SlobrokPolicy { /** Class parsing the semicolon separated parameter string and exposes the appropriate value to the policy. */ public static class Parameters { + protected final String clusterName; protected final String distributionConfigId; + protected final DistributionConfig distributionConfig; protected final SlobrokHostPatternGenerator slobrokHostPatternGenerator; public Parameters(Map<String, String> params) { + this(params, null); + } + + private Parameters(Map<String, String> params, DistributionConfig config) { clusterName = params.get("cluster"); + if (clusterName == null) + throw new IllegalArgumentException("Required parameter 'cluster', the name of the content cluster, not set"); + distributionConfig = config; + if (distributionConfig != null && distributionConfig.cluster(clusterName) == null) + throw new IllegalArgumentException("Distribution config for cluster '" + clusterName + "' not found"); distributionConfigId = params.get("clusterconfigid"); // TODO jonmv: remove slobrokHostPatternGenerator = createPatternGenerator(); - if (clusterName == null) throw new IllegalArgumentException("Required parameter 'cluster', the name of the content cluster, not set"); } - String getDistributionConfigId() { + private String getDistributionConfigId() { return distributionConfigId == null ? clusterName : distributionConfigId; } public String getClusterName() { @@ -245,7 +256,8 @@ public class ContentPolicy extends SlobrokPolicy { return new TargetCachingSlobrokHostFetcher(slobrokHostPatternGenerator, policy, percent); } public Distribution createDistribution(SlobrokPolicy policy) { - return new Distribution(getDistributionConfigId()); + return distributionConfig == null ? new Distribution(getDistributionConfigId()) + : new Distribution(distributionConfig.cluster(clusterName)); } /** @@ -548,12 +560,8 @@ public class ContentPolicy extends SlobrokPolicy { private final Parameters parameters; /** Constructor used in production. */ - public ContentPolicy(String param) { - this(parse(param)); - } - - public ContentPolicy(Map<String, String> params) { - this(new Parameters(params)); + public ContentPolicy(String param, DistributionConfig config) { + this(new Parameters(parse(param), config)); } /** Constructor specifying a bit more in detail, so we can override what needs to be overridden in tests */ diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java index 2680ed011af..0ce83a3c204 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java @@ -11,10 +11,12 @@ import com.yahoo.messagebus.ErrorCode; import com.yahoo.messagebus.Protocol; import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.Routable; +import com.yahoo.messagebus.documentapi.DocumentProtocolPoliciesConfig; import com.yahoo.messagebus.routing.RoutingContext; import com.yahoo.messagebus.routing.RoutingNodeIterator; import com.yahoo.messagebus.routing.RoutingPolicy; import com.yahoo.text.Utf8String; +import com.yahoo.vespa.config.content.DistributionConfig; import java.util.Collections; import java.util.HashSet; @@ -24,6 +26,8 @@ import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; +import static java.util.Objects.requireNonNull; + /** * Implements the message bus protocol that is used by all components of Vespa. * @@ -243,27 +247,36 @@ public class DocumentProtocol implements Protocol { this(docMan, configId, new LoadTypeSet()); } + public DocumentProtocol(DocumentTypeManager documentTypeManager, LoadTypeSet loadTypes, + DocumentProtocolPoliciesConfig policiesConfig, DistributionConfig distributionConfig) { + this(requireNonNull(documentTypeManager), null, requireNonNull(loadTypes), + requireNonNull(policiesConfig), requireNonNull(distributionConfig)); + } + public DocumentProtocol(DocumentTypeManager docMan, String configId, LoadTypeSet set) { - // Prepare config string for routing policy factories. - String cfg = (configId == null ? "client" : configId); - if (docMan != null) { + this(docMan, configId == null ? "client" : configId, set, null, null); + } + + private DocumentProtocol(DocumentTypeManager docMan, String configId, LoadTypeSet set, + DocumentProtocolPoliciesConfig policiesConfig, DistributionConfig distributionConfig) { + if (docMan != null) this.docMan = docMan; - } else { + else { this.docMan = new DocumentTypeManager(); - DocumentTypeManagerConfigurer.configure(this.docMan, cfg); + DocumentTypeManagerConfigurer.configure(this.docMan, configId); } - routableRepository = new RoutableRepository(set); + this.routableRepository = new RoutableRepository(set); // When adding factories to this list, please KEEP THEM ORDERED alphabetically like they are now. putRoutingPolicyFactory("AND", new RoutingPolicyFactories.AndPolicyFactory()); - putRoutingPolicyFactory("Content", new RoutingPolicyFactories.ContentPolicyFactory()); - putRoutingPolicyFactory("DocumentRouteSelector", new RoutingPolicyFactories.DocumentRouteSelectorPolicyFactory(cfg)); + putRoutingPolicyFactory("Content", new RoutingPolicyFactories.ContentPolicyFactory(distributionConfig)); + putRoutingPolicyFactory("DocumentRouteSelector", new RoutingPolicyFactories.DocumentRouteSelectorPolicyFactory(configId, policiesConfig)); putRoutingPolicyFactory("Extern", new RoutingPolicyFactories.ExternPolicyFactory()); putRoutingPolicyFactory("LocalService", new RoutingPolicyFactories.LocalServicePolicyFactory()); - putRoutingPolicyFactory("MessageType", new RoutingPolicyFactories.MessageTypePolicyFactory(cfg)); + putRoutingPolicyFactory("MessageType", new RoutingPolicyFactories.MessageTypePolicyFactory(configId, policiesConfig)); putRoutingPolicyFactory("RoundRobin", new RoutingPolicyFactories.RoundRobinPolicyFactory()); putRoutingPolicyFactory("LoadBalancer", new RoutingPolicyFactories.LoadBalancerPolicyFactory()); - putRoutingPolicyFactory("Storage", new RoutingPolicyFactories.ContentPolicyFactory()); + putRoutingPolicyFactory("Storage", new RoutingPolicyFactories.ContentPolicyFactory(distributionConfig)); putRoutingPolicyFactory("SubsetService", new RoutingPolicyFactories.SubsetServicePolicyFactory()); // Prepare version specifications to use when adding routable factories. diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java index 8fbd1548f68..6151daf043f 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java @@ -5,13 +5,15 @@ import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.document.DocumentGet; import com.yahoo.document.select.DocumentSelector; import com.yahoo.document.select.Result; -import java.util.logging.Level; +import com.yahoo.document.select.parser.ParseException; import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.documentapi.DocumentProtocolPoliciesConfig; import com.yahoo.messagebus.routing.Route; import com.yahoo.messagebus.routing.RoutingContext; import java.util.HashMap; import java.util.Map; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -30,6 +32,24 @@ public class DocumentRouteSelectorPolicy private String error = "Not configured."; private ConfigSubscriber subscriber; + /** This policy is constructed with the proper config at its time of creation. */ + public DocumentRouteSelectorPolicy(DocumentProtocolPoliciesConfig config) { + Map<String, DocumentSelector> selectors = new HashMap<>(); + config.cluster().forEach((name, cluster) -> { + try { + selectors.put(name, new DocumentSelector(cluster.selector())); + } + catch (ParseException e) { + throw new IllegalArgumentException("Error parsing selector '" + + cluster.selector() + + "' for route '" + name +"'", + e); + } + }); + this.config = Map.copyOf(selectors); + this.error = null; + } + /** * This policy is constructed with a configuration identifier that can be subscribed to for the document selector * config. If the string is either null or empty it will default to the proper one. diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MessageTypePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MessageTypePolicy.java index 4226c1e6cac..026a26cfc0c 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MessageTypePolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MessageTypePolicy.java @@ -2,13 +2,17 @@ package com.yahoo.documentapi.messagebus.protocol; import com.yahoo.config.subscription.ConfigSubscriber; +import com.yahoo.messagebus.documentapi.DocumentProtocolPoliciesConfig; import com.yahoo.messagebus.routing.Route; import com.yahoo.messagebus.routing.RoutingContext; import com.yahoo.vespa.config.content.MessagetyperouteselectorpolicyConfig; + import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; +import static java.util.stream.Collectors.toUnmodifiableMap; + /** * @author baldersheim */ @@ -18,6 +22,13 @@ public class MessageTypePolicy implements DocumentProtocolRoutingPolicy, ConfigS private ConfigSubscriber subscriber; private volatile Route defaultRoute; + MessageTypePolicy(DocumentProtocolPoliciesConfig.Cluster config) { + configRef.set(config.route().stream() + .collect(toUnmodifiableMap(route -> route.messageType(), + route -> Route.parse(route.name())))); + defaultRoute = Route.parse(config.defaultRoute()); + } + MessageTypePolicy(String configId) { subscriber = new ConfigSubscriber(); subscriber.subscribe(this, MessagetyperouteselectorpolicyConfig.class, configId); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactories.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactories.java index 7b44a1a4f0d..8535fa610dd 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactories.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactories.java @@ -1,54 +1,72 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi.messagebus.protocol; +import com.yahoo.messagebus.documentapi.DocumentProtocolPoliciesConfig; +import com.yahoo.vespa.config.content.DistributionConfig; + /** * @author Simon Thoresen Hult + * @author jonmv */ -public abstract class RoutingPolicyFactories { +class RoutingPolicyFactories { + + private RoutingPolicyFactories() { } static class AndPolicyFactory implements RoutingPolicyFactory { public DocumentProtocolRoutingPolicy createPolicy(String param) { return new ANDPolicy(param); } - - - public void destroy() { - } } static class ContentPolicyFactory implements RoutingPolicyFactory { + private final DistributionConfig distributionConfig; + public ContentPolicyFactory(DistributionConfig config) { this.distributionConfig = config; } public DocumentProtocolRoutingPolicy createPolicy(String param) { - return new ContentPolicy(param); - } - - public void destroy() { + return new ContentPolicy(param, distributionConfig); } } static class MessageTypePolicyFactory implements RoutingPolicyFactory { + private final String configId; + private final DocumentProtocolPoliciesConfig config; - public MessageTypePolicyFactory(String configId) { + public MessageTypePolicyFactory(String configId, DocumentProtocolPoliciesConfig config) { this.configId = configId; + this.config = config; } + public DocumentProtocolRoutingPolicy createPolicy(String param) { - return new MessageTypePolicy((param == null || param.isEmpty()) ? configId : param); - } + if (config != null) { + if (config.cluster(param) == null) + return new ErrorPolicy("No message type config for cluster '" + param + "'"); - public void destroy() { + return new MessageTypePolicy(config.cluster(param)); + } + return new MessageTypePolicy(param == null || param.isEmpty() ? configId : param); } } static class DocumentRouteSelectorPolicyFactory implements RoutingPolicyFactory { private final String configId; + private final DocumentProtocolPoliciesConfig config; - public DocumentRouteSelectorPolicyFactory(String configId) { + public DocumentRouteSelectorPolicyFactory(String configId, DocumentProtocolPoliciesConfig config) { this.configId = configId; + this.config = config; } public DocumentProtocolRoutingPolicy createPolicy(String param) { - DocumentRouteSelectorPolicy ret = new DocumentRouteSelectorPolicy((param == null || param.isEmpty()) ? + if (config != null) { + try { + return new DocumentRouteSelectorPolicy(config); + } + catch (IllegalArgumentException e) { + return new ErrorPolicy(e.getMessage()); + } + } + DocumentRouteSelectorPolicy ret = new DocumentRouteSelectorPolicy(param == null || param.isEmpty() ? configId : param); String error = ret.getError(); if (error != null) { @@ -56,10 +74,6 @@ public abstract class RoutingPolicyFactories { } return ret; } - - - public void destroy() { - } } static class ExternPolicyFactory implements RoutingPolicyFactory { @@ -71,49 +85,30 @@ public abstract class RoutingPolicyFactories { } return ret; } - - - public void destroy() { - } } static class LocalServicePolicyFactory implements RoutingPolicyFactory { public DocumentProtocolRoutingPolicy createPolicy(String param) { return new LocalServicePolicy(param); } - - - public void destroy() { - } } static class RoundRobinPolicyFactory implements RoutingPolicyFactory { public DocumentProtocolRoutingPolicy createPolicy(String param) { return new RoundRobinPolicy(); } - - - public void destroy() { - } } static class LoadBalancerPolicyFactory implements RoutingPolicyFactory { public DocumentProtocolRoutingPolicy createPolicy(String param) { return new LoadBalancerPolicy(param); } - - - public void destroy() { - } } static class SubsetServicePolicyFactory implements RoutingPolicyFactory { public DocumentProtocolRoutingPolicy createPolicy(String param) { return new SubsetServicePolicy(param); } - - - public void destroy() { - } } + } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactory.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactory.java index 6ea5020607e..3e368832c98 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactory.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactory.java @@ -24,11 +24,6 @@ public interface RoutingPolicyFactory { * @param param The parameter to use when creating the policy. * @return The created routing policy. */ - public DocumentProtocolRoutingPolicy createPolicy(String param); + DocumentProtocolRoutingPolicy createPolicy(String param); - /** - * Destroys this factory and frees up any resources it has held. Making further calls on a destroyed - * factory causes a runtime exception. - */ - public void destroy(); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SlobrokPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SlobrokPolicy.java index f4e1bb33dd1..1ffce622d78 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SlobrokPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SlobrokPolicy.java @@ -10,10 +10,10 @@ import java.util.Map; import java.util.TreeMap; /** - * Abstract class for policies that allow you to specify which slobrok to use for the - * routing. + * Abstract class for policies that allow you to specify which slobrok to use for the routing. */ public abstract class SlobrokPolicy implements DocumentProtocolRoutingPolicy { + private boolean firstTry = true; protected List<Mirror.Entry> lookup(RoutingContext context, String pattern) { diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTest.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTest.java index f324245b612..5aa3994a757 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTest.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTest.java @@ -5,6 +5,7 @@ import org.junit.Ignore; import org.junit.Test; public class ContentPolicyTest extends Simulator { + /** * Verify that a resent message with failures doesn't ruin overall performance. (By dumping the cached state too often * so other requests are sent to wrong target) @@ -17,6 +18,7 @@ public class ContentPolicyTest extends Simulator { + "Last correctnode 99, wrongnode 1, downnode 0, worked 92, failed 8", new PersistentFailureTestParameters().addBadNode(new BadNode(3, FailureType.TRANSIENT_ERROR))); } + /** * Verify that a resent message with failures doesn't ruin overall performance. (By dumping the cached state too often * so other requests are sent to wrong target) @@ -29,6 +31,7 @@ public class ContentPolicyTest extends Simulator { + "Last correctnode 99, wrongnode 1, downnode 0, worked 92, failed 8", new PersistentFailureTestParameters().addBadNode(new BadNode(3, FailureType.FATAL_ERROR))); } + /** * Verify that a node responding with old cluster state doesn't ruin overall performance (By dumping/switching cached * state too often) @@ -41,6 +44,7 @@ public class ContentPolicyTest extends Simulator { + "Last correctnode 100, wrongnode 0, downnode 0, worked 100, failed 0", new PersistentFailureTestParameters().addBadNode(new BadNode(3, FailureType.OLD_CLUSTER_STATE).setDownInCurrentState())); } + /** * Verify that a reset cluster state version doesn't keep sending requests to the wrong node. * We expect a few failures in first half. We should have detected the issue before second half, so there all should be fine. @@ -52,6 +56,7 @@ public class ContentPolicyTest extends Simulator { + "Last correctnode .*, wrongnode 0, downnode 0, worked .*, failed 0", new PersistentFailureTestParameters().addBadNode(new BadNode(3, FailureType.RESET_CLUSTER_STATE).setDownInCurrentState())); } + /** * Verify that a reset cluster state version doesn't keep sending requests to the wrong node. * We expect a few failures in first half. We should have detected the issue before second half, so there all should be fine. @@ -70,6 +75,7 @@ public class ContentPolicyTest extends Simulator { + "Last correctnode .*, wrongnode 100, downnode 100, worked 0, failed 100", new PersistentFailureTestParameters().addBadNode(new BadNode(3, FailureType.RESET_CLUSTER_STATE_NO_GOOD_NODES).setDownInCurrentState())); } + /** * Verify that a reset cluster state version doesn't keep sending requests to the wrong node. * We expect a few failures in first half. We should have detected the issue before second half, so there all should be fine. @@ -86,6 +92,7 @@ public class ContentPolicyTest extends Simulator { + "Last correctnode .*, wrongnode 91, downnode 0, worked 0, failed 100", new PersistentFailureTestParameters().addBadNode(new BadNode(3, FailureType.RESET_CLUSTER_STATE_NO_GOOD_NODES))); } + /** * Verify that a reset cluster state version doesn't keep sending requests to the wrong node. * Another scenario where we have a node coming up in correct state. @@ -98,6 +105,7 @@ public class ContentPolicyTest extends Simulator { + "Last correctnode .*, wrongnode 0, downnode 0, worked .*, failed 0", new PersistentFailureTestParameters().newNodeAdded().addBadNode(new BadNode(3, FailureType.RESET_CLUSTER_STATE).setDownInCurrentState())); } + /** Test node that is not in slobrok. Until fleetcontroller detects this, we expect 10% of the requests to go to wrong node. */ @Test @Ignore // FIXME test has been implicitly disabled for ages, figure out and fix diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTestEnvironment.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTestEnvironment.java index 479e0b0f422..6d2477e1871 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTestEnvironment.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTestEnvironment.java @@ -34,11 +34,13 @@ import java.util.TreeSet; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public abstract class ContentPolicyTestEnvironment { - protected StoragePolicyTestFactory policyFactory; + protected ContentPolicyTestFactory policyFactory; protected PolicyTestFrame frame; private Set<Integer> nodes; protected static int[] bucketOneNodePreference = new int[]{ 3, 5, 7, 6, 8, 0, 9, 2, 1, 4 }; @@ -51,7 +53,7 @@ public abstract class ContentPolicyTestEnvironment { frame = new PolicyTestFrame(manager); nodes = new TreeSet<>(); DocumentProtocol protocol = (DocumentProtocol) frame.getMessageBus().getProtocol((Utf8Array)DocumentProtocol.NAME); - policyFactory = new StoragePolicyTestFactory(nodes); + policyFactory = new ContentPolicyTestFactory(nodes); protocol.putRoutingPolicyFactory("storage", policyFactory); frame.setMessage(createMessage("id:ns:testdoc:n=1:foo")); frame.setHop(new HopSpec("test", "[storage:cluster=foo]")); @@ -104,7 +106,7 @@ public abstract class ContentPolicyTestEnvironment { public static class TestHostFetcher extends ContentPolicy.HostFetcher { private final String clusterName; - private RandomGen randomizer = new RandomGen(1234); + private final RandomGen randomizer = new RandomGen(1234); private final Set<Integer> nodes; private Integer avoidPickingAtRandom = null; @@ -121,13 +123,14 @@ public abstract class ContentPolicyTestEnvironment { try{ if (distributor == null) { if (nodes.size() == 1) { - assertTrue(avoidPickingAtRandom != nodes.iterator().next()); + assertNotSame(avoidPickingAtRandom, nodes.iterator().next()); distributor = nodes.iterator().next(); } else { Iterator<Integer> it = nodes.iterator(); for (int i = 0, n = randomizer.nextInt(nodes.size() - 1); i<n; ++i) it.next(); distributor = it.next(); - if (avoidPickingAtRandom != null && distributor == avoidPickingAtRandom) distributor = it.next(); + if (avoidPickingAtRandom != null && avoidPickingAtRandom.equals(distributor)) + distributor = it.next(); } } if (nodes.contains(distributor)) { @@ -137,8 +140,7 @@ public abstract class ContentPolicyTestEnvironment { } } catch (RuntimeException e) { e.printStackTrace(); - assertTrue(e.getMessage(), false); - throw e; + throw new AssertionError(e.getMessage()); } } } @@ -160,12 +162,12 @@ public abstract class ContentPolicyTestEnvironment { public Distribution createDistribution(SlobrokPolicy policy) { return distribution; } } - public static class StoragePolicyTestFactory implements RoutingPolicyFactory { + public static class ContentPolicyTestFactory implements RoutingPolicyFactory { private Set<Integer> nodes; - private final LinkedList<TestParameters> parameterInstances = new LinkedList<TestParameters>(); + private final LinkedList<TestParameters> parameterInstances = new LinkedList<>(); private Integer avoidPickingAtRandom = null; - public StoragePolicyTestFactory(Set<Integer> nodes) { + public ContentPolicyTestFactory(Set<Integer> nodes) { this.nodes = nodes; } public DocumentProtocolRoutingPolicy createPolicy(String parameters) { diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/Simulator.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/Simulator.java index d23dd9ea998..be880e69781 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/Simulator.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/Simulator.java @@ -31,7 +31,7 @@ public abstract class Simulator extends ContentPolicyTestEnvironment { RESET_CLUSTER_STATE_NO_GOOD_NODES, NODE_NOT_IN_SLOBROK }; - private Integer getIdealTarget(String idString, String clusterState) { + private int getIdealTarget(String idString, String clusterState) { DocumentId did = new DocumentId(idString); BucketIdFactory factory = new BucketIdFactory(); BucketId bid = factory.getBucketId(did); @@ -145,6 +145,7 @@ public abstract class Simulator extends ContentPolicyTestEnvironment { return currentClusterState; } } + public void runSimulation(String expected, PersistentFailureTestParameters params) { params.validate(); // Set nodes in slobrok @@ -157,16 +158,16 @@ public abstract class Simulator extends ContentPolicyTestEnvironment { replyWrongDistribution(target, "foo", null, params.getInitialClusterState().toString()); } RandomGen randomizer = new RandomGen(432121); - int correctnode[] = new int[2], - wrongnode[] = new int[2], - failed[] = new int[2], - worked[] = new int[2], - downnode[] = new int[2]; + int[] correctnode = new int[2], + wrongnode = new int[2], + failed = new int[2], + worked = new int[2], + downnode = new int[2]; for (int step = 0, steps = (params.getTotalRequests() / params.getParallellRequests()); step < steps; ++step) { int half = (step < steps / 2 ? 0 : 1); if (debug) System.err.println("Starting step " + step + " in half " + half); - String docId[] = new String[params.getParallellRequests()]; - RoutingNode targets[] = new RoutingNode[params.getParallellRequests()]; + String[] docId = new String[params.getParallellRequests()]; + RoutingNode[] targets = new RoutingNode[params.getParallellRequests()]; for (int i=0; i<params.getParallellRequests(); ++i) { docId[i] = "id:ns:testdoc::" + (step * params.getParallellRequests() + i); frame.setMessage(createMessage(docId[i])); @@ -206,7 +207,6 @@ public abstract class Simulator extends ContentPolicyTestEnvironment { } } StringBuilder actual = new StringBuilder(); - String result[][] = new String[2][]; for (int i=0; i<2; ++i) { actual.append(i == 0 ? "First " : " Last ") .append("correctnode ").append(correctnode[i]) @@ -215,7 +215,7 @@ public abstract class Simulator extends ContentPolicyTestEnvironment { .append(", worked ").append(worked[i]) .append(", failed ").append(failed[i]); } - if (!Pattern.matches(expected, actual.toString())) { + if ( ! Pattern.matches(expected, actual.toString())) { assertEquals(expected, actual.toString()); } } |