diff options
36 files changed, 313 insertions, 199 deletions
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java index 498037a4984..bb3d7e049d1 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java @@ -15,7 +15,12 @@ import com.yahoo.vespa.clustercontroller.core.Timer; import com.yahoo.vespa.clustercontroller.core.ContentCluster; import com.yahoo.vespa.clustercontroller.core.listeners.NodeAddedOrRemovedListener; -import java.util.*; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.StringTokenizer; +import java.util.TreeMap; import java.util.logging.Logger; public class SlobrokClient implements NodeLookup { @@ -190,8 +195,8 @@ public class SlobrokClient implements NodeLookup { private Map<Node, SlobrokData> getSlobrokData(String pattern) { Map<Node, SlobrokData> result = new TreeMap<>(); - Mirror.Entry[] entries = mirror.lookup(pattern); - log.log(LogLevel.SPAM, "Looking for slobrok entries with pattern '" + pattern + "'. Found " + entries.length + " entries."); + List<Mirror.Entry> entries = mirror.lookup(pattern); + log.log(LogLevel.SPAM, "Looking for slobrok entries with pattern '" + pattern + "'. Found " + entries.size() + " entries."); for (Mirror.Entry entry : entries) { StringTokenizer st = new StringTokenizer(entry.getName(), "/"); String addressType = st.nextToken(); @@ -209,7 +214,7 @@ public class SlobrokClient implements NodeLookup { private static class SlobrokData { public Node node; - public String rpcAddress; + String rpcAddress; SlobrokData(Node node, String rpcAddress) { this.node = node; diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java index db072afd201..65d94807d7f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java @@ -3,7 +3,6 @@ package com.yahoo.config.model.producer; import com.google.common.annotations.Beta; import com.yahoo.config.ConfigInstance; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.subscription.ConfigInstanceUtil; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java index 97236222338..ecfa9a9e53e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java @@ -34,9 +34,9 @@ import java.util.TreeMap; public final class DocumentProtocol implements Protocol, DocumentrouteselectorpolicyConfig.Producer { private static final String NAME = "document"; - private ApplicationSpec application; - private RoutingTableSpec routingTable; - ConfigModelRepo repo; + private final ApplicationSpec application; + private final RoutingTableSpec routingTable; + private final ConfigModelRepo repo; public static String getIndexedRouteName(String configId) { return configId + "-index"; @@ -51,7 +51,7 @@ public final class DocumentProtocol implements Protocol, Documentrouteselectorpo * * @param plugins The plugins to reflect on. */ - public DocumentProtocol(ConfigModelRepo plugins) { + DocumentProtocol(ConfigModelRepo plugins) { application = createApplicationSpec(plugins); routingTable = createRoutingTable(plugins); this.repo = plugins; @@ -249,7 +249,13 @@ public final class DocumentProtocol implements Protocol, Documentrouteselectorpo route.addHop("indexing"); table.addRoute(route); - table.addRoute(new RouteSpec("default-get").addHop("indexing")); + if (content.size() == 1) { + table.addRoute(new RouteSpec("default-get").addHop("[Content:cluster=" + content.get(0).getConfigId() + "]")); + } else { + //TODO This should ideally skip indexing and go directly to correct cluster. + // But will handle the single cluster for now. + table.addRoute(new RouteSpec("default-get").addHop("indexing")); + } } private static boolean indexingHopExists(RoutingTableSpec table) { diff --git a/config-model/src/test/cfg/routing/replacehop/messagebus.cfg b/config-model/src/test/cfg/routing/replacehop/messagebus.cfg index 12701af6db3..8717deb50a5 100755 --- a/config-model/src/test/cfg/routing/replacehop/messagebus.cfg +++ b/config-model/src/test/cfg/routing/replacehop/messagebus.cfg @@ -12,7 +12,7 @@ routingtable[0].hop[2].ignoreresult false routingtable[0].route[0].name "default" routingtable[0].route[0].hop[0] "indexing" routingtable[0].route[1].name "default-get" -routingtable[0].route[1].hop[0] "indexing" +routingtable[0].route[1].hop[0] "[Content:cluster=music]" routingtable[0].route[2].name "music" routingtable[0].route[2].hop[0] "[MessageType:music]" routingtable[0].route[3].name "music-direct" diff --git a/config-model/src/test/cfg/routing/replaceroute/messagebus.cfg b/config-model/src/test/cfg/routing/replaceroute/messagebus.cfg index d3af5b36c26..7eae6e8ebc5 100755 --- a/config-model/src/test/cfg/routing/replaceroute/messagebus.cfg +++ b/config-model/src/test/cfg/routing/replaceroute/messagebus.cfg @@ -9,7 +9,7 @@ routingtable[0].hop[1].ignoreresult false routingtable[0].route[0].name "default" routingtable[0].route[0].hop[0] "foo" routingtable[0].route[1].name "default-get" -routingtable[0].route[1].hop[0] "indexing" +routingtable[0].route[1].hop[0] "[Content:cluster=music]" routingtable[0].route[2].name "music" routingtable[0].route[2].hop[0] "[MessageType:music]" routingtable[0].route[3].name "music-direct" diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentBaseTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentBaseTest.java index 0432b181c2c..f5642922d5b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentBaseTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentBaseTest.java @@ -1,6 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; +import com.yahoo.messagebus.routing.RouteSpec; + +import static org.junit.Assert.assertEquals; + public class ContentBaseTest { public static String getHosts() { return "<?xml version='1.0' encoding='utf-8' ?>" + @@ -10,4 +14,12 @@ public class ContentBaseTest { " </host>" + "</hosts>"; } + + static void assertRoute(RouteSpec r, String name, String... hops) { + assertEquals(name, r.getName()); + assertEquals(hops.length, r.getNumHops()); + for(int i = 0; i < hops.length; i++) { + assertEquals(hops[i], r.getHop(i)); + } + } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 0e200efd688..7c365859862 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -10,6 +10,7 @@ import com.yahoo.config.model.test.TestRoot; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; +import com.yahoo.messagebus.routing.RoutingTableSpec; import com.yahoo.metrics.MetricsmanagerConfig; import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; import com.yahoo.vespa.config.content.FleetcontrollerConfig; @@ -26,6 +27,8 @@ import com.yahoo.vespa.model.content.engines.ProtonEngine; import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder; +import com.yahoo.vespa.model.routing.DocumentProtocol; +import com.yahoo.vespa.model.routing.Routing; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.Rule; @@ -853,44 +856,73 @@ public class ContentClusterTest extends ContentBaseTest { } } - @Test - public void all_clusters_bucket_spaces_config_contains_mappings_across_all_clusters() { + private VespaModel createDualContentCluster() { String xml = "<services>" + - "<admin version=\"2.0\">" + - " <adminserver hostalias=\"node0\"/>" + - "</admin>" + - "<content version=\"1.0\" id=\"foocluster\">" + - " <redundancy>1</redundancy>" + - " <documents>" + - " <document type=\"bunnies\" mode=\"index\"/>" + - " <document type=\"hares\" mode=\"index\"/>" + - " </documents>" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"node0\"/>" + - " </group>" + - "</content>" + - "<content version=\"1.0\" id=\"barcluster\">" + - " <redundancy>1</redundancy>" + - " <documents>" + - " <document type=\"rabbits\" mode=\"index\" global=\"true\"/>" + - " </documents>" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"node0\"/>" + - " </group>" + - "</content>" + - "</services>"; + "<admin version=\"2.0\">" + + " <adminserver hostalias=\"node0\"/>" + + "</admin>" + + "<content version=\"1.0\" id=\"foo_c\">" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type=\"bunnies\" mode=\"index\"/>" + + " <document type=\"hares\" mode=\"index\"/>" + + " </documents>" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"node0\"/>" + + " </group>" + + "</content>" + + "<content version=\"1.0\" id=\"bar_c\">" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type=\"rabbits\" mode=\"index\" global=\"true\"/>" + + " </documents>" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"node0\"/>" + + " </group>" + + "</content>" + + "</services>"; List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("bunnies", "hares", "rabbits"); - VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); + return new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); + } + @Test + public void all_clusters_bucket_spaces_config_contains_mappings_across_all_clusters() { + VespaModel model = createDualContentCluster(); AllClustersBucketSpacesConfig.Builder builder = new AllClustersBucketSpacesConfig.Builder(); model.getConfig(builder, "client"); AllClustersBucketSpacesConfig config = builder.build(); assertEquals(2, config.cluster().size()); - assertClusterHasBucketSpaceMappings(config, "foocluster", Arrays.asList("bunnies", "hares"), Collections.emptyList()); - assertClusterHasBucketSpaceMappings(config, "barcluster", Collections.emptyList(), Collections.singletonList("rabbits")); + assertClusterHasBucketSpaceMappings(config, "foo_c", Arrays.asList("bunnies", "hares"), Collections.emptyList()); + assertClusterHasBucketSpaceMappings(config, "bar_c", Collections.emptyList(), Collections.singletonList("rabbits")); + } + @Test + public void test_routing_with_multiple_clusters() { + VespaModel model = createDualContentCluster(); + Routing routing = model.getRouting(); + assertNotNull(routing); + assertEquals("[]", routing.getErrors().toString()); + assertEquals(1, routing.getProtocols().size()); + DocumentProtocol protocol = (DocumentProtocol) routing.getProtocols().get(0); + RoutingTableSpec spec = protocol.getRoutingTableSpec(); + assertEquals(3, spec.getNumHops()); + assertEquals("docproc/cluster.bar_c.indexing/chain.indexing", spec.getHop(0).getName()); + assertEquals("docproc/cluster.foo_c.indexing/chain.indexing", spec.getHop(1).getName()); + assertEquals("indexing", spec.getHop(2).getName()); + + assertEquals(10, spec.getNumRoutes()); + assertRoute(spec.getRoute(0), "bar_c", "[MessageType:bar_c]"); + assertRoute(spec.getRoute(1), "bar_c-direct", "[Content:cluster=bar_c]"); + assertRoute(spec.getRoute(2), "bar_c-index", "docproc/cluster.bar_c.indexing/chain.indexing", "[Content:cluster=bar_c]"); + assertRoute(spec.getRoute(3), "default", "indexing"); + assertRoute(spec.getRoute(4), "default-get", "indexing"); + assertRoute(spec.getRoute(5), "foo_c", "[MessageType:foo_c]"); + assertRoute(spec.getRoute(6), "foo_c-direct", "[Content:cluster=foo_c]"); + assertRoute(spec.getRoute(7), "foo_c-index", "docproc/cluster.foo_c.indexing/chain.indexing", "[Content:cluster=foo_c]"); + assertRoute(spec.getRoute(8), "storage/cluster.bar_c", "route:bar_c"); + assertRoute(spec.getRoute(9), "storage/cluster.foo_c", "route:foo_c"); } private ContentCluster createWithZone(String clusterXml, Zone zone) throws Exception { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java index bf8e2a353cf..919fec5be2f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.content; import com.yahoo.cloud.config.ClusterListConfig; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.messagebus.routing.RouteSpec; import com.yahoo.messagebus.routing.RoutingTableSpec; import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; @@ -158,21 +157,13 @@ public class IndexedTest extends ContentBaseTest { assertEquals("jdisc/chain.indexing", spec.getHop(1).getName()); assertRoute(spec.getRoute(0), "default", "indexing"); - assertRoute(spec.getRoute(1), "default-get", "indexing"); + assertRoute(spec.getRoute(1), "default-get", "[Content:cluster=test]"); assertRoute(spec.getRoute(2), "storage/cluster.test", "route:test"); assertRoute(spec.getRoute(3), "test", "[MessageType:test]"); assertRoute(spec.getRoute(4), "test-direct", "[Content:cluster=test]"); assertRoute(spec.getRoute(5), "test-index", "jdisc/chain.indexing", "[Content:cluster=test]"); } - private static void assertRoute(RouteSpec r, String name, String... hops) { - assertEquals(name, r.getName()); - assertEquals(hops.length, r.getNumHops()); - for(int i = 0; i < hops.length; i++) { - assertEquals(hops[i], r.getHop(i)); - } - } - @Test public void requireProtonStreamingOnly() { diff --git a/container-disc/src/main/java/com/yahoo/container/FilterConfigProvider.java b/container-disc/src/main/java/com/yahoo/container/FilterConfigProvider.java index c17b9d445a2..a127cb03a01 100644 --- a/container-disc/src/main/java/com/yahoo/container/FilterConfigProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/FilterConfigProvider.java @@ -57,7 +57,11 @@ public final class FilterConfigProvider implements Provider<FilterConfig> { private final FilterConfig filterConfig; public FilterConfigProvider(HttpFilterConfig vespaConfig) { - filterConfig = new MapFilterConfig(toMap(vespaConfig), vespaConfig.filterName(), vespaConfig.filterClass()); + this(new MapFilterConfig(toMap(vespaConfig), vespaConfig.filterName(), vespaConfig.filterClass())); + } + + private FilterConfigProvider(FilterConfig filterConfig) { + this.filterConfig = filterConfig; } private static Map<String, String> toMap(HttpFilterConfig vespaConfig) { @@ -67,6 +71,10 @@ public final class FilterConfigProvider implements Provider<FilterConfig> { return parameters; } + public static FilterConfigProvider from(String filterName, String filterClass, Map<String, String> initParameters) { + return new FilterConfigProvider(new MapFilterConfig(initParameters, filterName, filterClass)); + } + @Override public FilterConfig get() { return filterConfig; diff --git a/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java index f895ad21b0a..287af5c34b9 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java @@ -5,14 +5,17 @@ import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.Field; import com.yahoo.document.FieldPath; -import com.yahoo.document.serialization.*; +import com.yahoo.document.serialization.DocumentSerializer; +import com.yahoo.document.serialization.DocumentSerializerFactory; +import com.yahoo.document.serialization.FieldReader; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.document.serialization.XmlStream; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.vespa.objects.BufferSerializer; import com.yahoo.vespa.objects.Deserializer; import com.yahoo.vespa.objects.Identifiable; import com.yahoo.vespa.objects.Ids; import com.yahoo.vespa.objects.Serializer; -import com.yahoo.document.config.DocumentmanagerConfig.Datatype.Structtype.Compresstype; /** * @author Einar M R Rosenvinge diff --git a/documentapi/abi-spec.json b/documentapi/abi-spec.json index cbfe07eb93d..e1e5f778423 100644 --- a/documentapi/abi-spec.json +++ b/documentapi/abi-spec.json @@ -2035,10 +2035,9 @@ "abstract" ], "methods": [ - "public void <init>(java.util.Map)", "public void init()", "public com.yahoo.jrt.slobrok.api.IMirror getMirror()", - "public com.yahoo.jrt.slobrok.api.Mirror$Entry[] lookup(com.yahoo.messagebus.routing.RoutingContext, java.lang.String)", + "public java.util.List lookup(com.yahoo.messagebus.routing.RoutingContext, java.lang.String)", "public synchronized void configure(com.yahoo.cloud.config.SlobroksConfig)", "public void destroy()", "public bridge synthetic void configure(com.yahoo.config.ConfigInstance)" @@ -2207,7 +2206,7 @@ "public void <init>(java.lang.String)", "public java.util.List getNodeWeights()", "public int getIndex(java.lang.String)", - "public com.yahoo.documentapi.messagebus.protocol.LoadBalancer$Node getRecipient(com.yahoo.jrt.slobrok.api.Mirror$Entry[])", + "public com.yahoo.documentapi.messagebus.protocol.LoadBalancer$Node getRecipient(java.util.List)", "public void received(com.yahoo.documentapi.messagebus.protocol.LoadBalancer$Node, boolean)" ], "fields": [] diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java index 312fd6e5964..e81ac4ae05e 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java @@ -103,7 +103,7 @@ public class ExternPolicy implements DocumentProtocolRoutingPolicy { if (generation != upd) { generation = upd; recipients.clear(); - Mirror.Entry[] arr = mirror.lookup(pattern); + List<Mirror.Entry> arr = mirror.lookup(pattern); for (Mirror.Entry entry : arr) { recipients.add(Hop.parse(entry.getSpec() + session)); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternalSlobrokPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternalSlobrokPolicy.java index 3ebe5b7281e..39242bb6cab 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternalSlobrokPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternalSlobrokPolicy.java @@ -11,6 +11,7 @@ import com.yahoo.jrt.slobrok.api.SlobrokList; import com.yahoo.messagebus.routing.RoutingContext; import com.yahoo.cloud.config.SlobroksConfig; +import java.util.List; import java.util.Map; /** @@ -19,16 +20,16 @@ import java.util.Map; */ public abstract class ExternalSlobrokPolicy extends AsyncInitializationPolicy implements ConfigSubscriber.SingleSubscriber<SlobroksConfig> { String error; - Supervisor orb = null; - Mirror mirror = null; - SlobrokList slobroks = null; - boolean firstTry = true; + private Supervisor orb = null; + private Mirror mirror = null; + private SlobrokList slobroks = null; + private boolean firstTry = true; private ConfigSubscriber subscriber; String[] configSources = null; - String slobrokConfigId = "admin/slobrok.0"; + private final static String slobrokConfigId = "admin/slobrok.0"; - public ExternalSlobrokPolicy(Map<String, String> param) { + ExternalSlobrokPolicy(Map<String, String> param) { super(); String conf = param.get("config"); @@ -72,16 +73,16 @@ public abstract class ExternalSlobrokPolicy extends AsyncInitializationPolicy im return mirror; } - public Mirror.Entry[] lookup(RoutingContext context, String pattern) { + public List<Mirror.Entry> lookup(RoutingContext context, String pattern) { IMirror mirror1 = (mirror != null ? mirror : context.getMirror()); - Mirror.Entry[] arr = mirror1.lookup(pattern); + List<Mirror.Entry> arr = mirror1.lookup(pattern); - if ((arr.length == 0) && firstTry) { + if ((arr.isEmpty()) && firstTry) { synchronized(this) { try { int count = 0; - while (arr.length == 0 && count < 100) { + while (arr.isEmpty() && count < 100) { Thread.sleep(50); arr = mirror1.lookup(pattern); count++; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancer.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancer.java index 167a480e1aa..a2875f14ab5 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancer.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancer.java @@ -62,8 +62,8 @@ public class LoadBalancer { * @param choices the node choices, represented as Slobrok entries * @return the chosen node, or null only if the given choices were zero */ - public Node getRecipient(Mirror.Entry[] choices) { - if (choices.length == 0) return null; + public Node getRecipient(List<Mirror.Entry> choices) { + if (choices.isEmpty()) return null; double weightSum = 0.0; Node selectedNode = null; @@ -79,7 +79,7 @@ public class LoadBalancer { } if (selectedNode == null) { // Position>sum of all weights: Wrap around (but keep the remainder for some reason) position -= weightSum; - selectedNode = new Node(choices[0], getNodeMetrics(choices[0])); + selectedNode = new Node(choices.get(0), getNodeMetrics(choices.get(0))); } position += 1.0; selectedNode.metrics.sent.incrementAndGet(); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerPolicy.java index 7c5345351a8..9cf82144e71 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerPolicy.java @@ -9,6 +9,7 @@ import com.yahoo.messagebus.routing.Route; import com.yahoo.messagebus.routing.RoutingContext; import com.yahoo.messagebus.routing.RoutingNodeIterator; +import java.util.List; import java.util.Map; /** @@ -75,7 +76,7 @@ public class LoadBalancerPolicy extends ExternalSlobrokPolicy { @return Returns a hop representing the TCP address of the target, or null if none could be found. */ private LoadBalancer.Node getRecipient(RoutingContext context) { - Mirror.Entry [] lastLookup = lookup(context, pattern); + List<Mirror.Entry> lastLookup = lookup(context, pattern); return loadBalancer.getRecipient(lastLookup); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LocalServicePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LocalServicePolicy.java index eb56a85e378..d4ebd4ecd81 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LocalServicePolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LocalServicePolicy.java @@ -83,7 +83,7 @@ public class LocalServicePolicy implements DocumentProtocolRoutingPolicy { entry.generation = upd; entry.recipients.clear(); - Mirror.Entry[] arr = ctx.getMirror().lookup(ctx.getHopPrefix() + "*" + ctx.getHopSuffix()); + List<Mirror.Entry> arr = ctx.getMirror().lookup(ctx.getHopPrefix() + "*" + ctx.getHopSuffix()); String self = localAddress != null ? localAddress : toAddress(ctx.getMessageBus().getConnectionSpec()); for (Mirror.Entry item : arr) { if (self.equals(toAddress(item.getSpec()))) { diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoundRobinPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoundRobinPolicy.java index 244d101b36f..3a1599ab71b 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoundRobinPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoundRobinPolicy.java @@ -82,7 +82,7 @@ public class RoundRobinPolicy implements DocumentProtocolRoutingPolicy { entry.generation = upd; entry.recipients.clear(); for (int i = 0; i < ctx.getNumRecipients(); ++i) { - Mirror.Entry[] arr = ctx.getMirror().lookup(ctx.getRecipient(i).getHop(0).toString()); + List<Mirror.Entry> arr = ctx.getMirror().lookup(ctx.getRecipient(i).getHop(0).toString()); for (Mirror.Entry item : arr) { entry.recipients.add(Hop.parse(item.getName())); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java index 341589643d2..048149e86ab 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java @@ -107,7 +107,7 @@ public class StoragePolicy extends ExternalSlobrokPolicy { this.policy = policy; } - private Mirror.Entry[] getEntries(String hostPattern, RoutingContext context) { + private List<Mirror.Entry> getEntries(String hostPattern, RoutingContext context) { return policy.lookup(context, hostPattern); } @@ -116,16 +116,16 @@ public class StoragePolicy extends ExternalSlobrokPolicy { public IMirror getMirror(RoutingContext context) { return context.getMirror(); } public String getTargetSpec(Integer distributor, RoutingContext context) { - Mirror.Entry[] arr = getEntries(patternGenerator.getDistributorHostPattern(distributor), context); - if (arr.length == 0) return null; + List<Mirror.Entry> arr = getEntries(patternGenerator.getDistributorHostPattern(distributor), context); + if (arr.isEmpty()) return null; if (distributor != null) { - if (arr.length == 1) { - return convertSlobrokNameToSessionName(arr[0].getSpec()); + if (arr.size() == 1) { + return convertSlobrokNameToSessionName(arr.get(0).getSpec()); } else { - log.log(LogLevel.WARNING, "Got " + arr.length + " matches for a distributor."); + log.log(LogLevel.WARNING, "Got " + arr.size() + " matches for a distributor."); } } else { - return convertSlobrokNameToSessionName(arr[randomizer.nextInt(arr.length)].getSpec()); + return convertSlobrokNameToSessionName(arr.get(randomizer.nextInt(arr.size())).getSpec()); } return null; } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java index 31802f2872f..3251c038fc7 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java @@ -109,10 +109,10 @@ public class SubsetServicePolicy implements DocumentProtocolRoutingPolicy { entry.generation = upd; entry.recipients.clear(); - Mirror.Entry[] arr = ctx.getMirror().lookup(ctx.getHopPrefix() + "*" + ctx.getHopSuffix()); + List<Mirror.Entry> arr = ctx.getMirror().lookup(ctx.getHopPrefix() + "*" + ctx.getHopSuffix()); int pos = ctx.getMessageBus().getConnectionSpec().hashCode(); - for (int i = 0; i < subsetSize && i < arr.length; ++i) { - entry.recipients.add(Hop.parse(arr[((pos + i) & Integer.MAX_VALUE) % arr.length].getName())); + for (int i = 0; i < subsetSize && i < arr.size(); ++i) { + entry.recipients.add(Hop.parse(arr.get(((pos + i) & Integer.MAX_VALUE) % arr.size()).getName())); } } return entry; diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/TargetCachingSlobrokHostFetcherTest.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/TargetCachingSlobrokHostFetcherTest.java index 7e6c7bc468a..4413b657739 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/TargetCachingSlobrokHostFetcherTest.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/TargetCachingSlobrokHostFetcherTest.java @@ -7,6 +7,8 @@ import com.yahoo.messagebus.routing.RoutingContext; import org.junit.Test; import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -23,26 +25,25 @@ import static org.mockito.Mockito.when; */ public class TargetCachingSlobrokHostFetcherTest { - static String idOfIndex(int index) { + private static String idOfIndex(int index) { return String.format("storage/cluster.foo/distributor/%d/default", index); } - static String idOfWildcardLookup() { + private static String idOfWildcardLookup() { return "storage/cluster.foo/distributor/*/default"; } - static String lookupSpecOfIndex(int index) { + private static String lookupSpecOfIndex(int index) { return String.format("tcp/localhost:%d", index); } - static String resolvedSpecOfIndex(int index) { + private static String resolvedSpecOfIndex(int index) { return String.format("tcp/localhost:%d/default", index); } - static Mirror.Entry[] dummyEntries(int... indices) { + private static List<Mirror.Entry> dummyEntries(int... indices) { return Arrays.stream(indices) - .mapToObj(index -> new Mirror.Entry(idOfIndex(index), lookupSpecOfIndex(index))) - .toArray(Mirror.Entry[]::new); + .mapToObj(index -> new Mirror.Entry(idOfIndex(index), lookupSpecOfIndex(index))).collect(Collectors.toList()); } static class Fixture { diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/LoadBalancerTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/LoadBalancerTestCase.java index 698b778c312..51dd1ac12b8 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/LoadBalancerTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/LoadBalancerTestCase.java @@ -5,6 +5,7 @@ import com.yahoo.documentapi.messagebus.protocol.LoadBalancer; import com.yahoo.jrt.slobrok.api.Mirror; import org.junit.Test; +import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; @@ -42,9 +43,9 @@ public class LoadBalancerTestCase { public void testLoadBalancer() { LoadBalancer lb = new LoadBalancer("foo"); - Mirror.Entry[] entries = new Mirror.Entry[]{ new Mirror.Entry("foo/0/default", "tcp/bar:1"), - new Mirror.Entry("foo/1/default", "tcp/bar:2"), - new Mirror.Entry("foo/2/default", "tcp/bar:3") }; + List<Mirror.Entry> entries = Arrays.asList(new Mirror.Entry("foo/0/default", "tcp/bar:1"), + new Mirror.Entry("foo/1/default", "tcp/bar:2"), + new Mirror.Entry("foo/2/default", "tcp/bar:3")); List<LoadBalancer.NodeMetrics> weights = lb.getNodeWeights(); { @@ -100,7 +101,7 @@ public class LoadBalancerTestCase { public void testLoadBalancerOneItemOnly() { LoadBalancer lb = new LoadBalancer("foo"); - Mirror.Entry[] entries = new Mirror.Entry[]{ new Mirror.Entry("foo/0/default", "tcp/bar:1") }; + List<Mirror.Entry> entries = Arrays.asList(new Mirror.Entry("foo/0/default", "tcp/bar:1") ); List<LoadBalancer.NodeMetrics> weights = lb.getNodeWeights(); assertEquals("foo/0/default" , lb.getRecipient(entries).entry.getName()); diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java index e2f1c9cd937..fd9d3f78ca8 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java @@ -1,21 +1,61 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi.messagebus.protocol.test; -import com.yahoo.document.*; -import com.yahoo.documentapi.messagebus.protocol.*; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentTypeManagerConfigurer; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.documentapi.messagebus.protocol.ANDPolicy; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.documentapi.messagebus.protocol.DocumentRouteSelectorPolicy; +import com.yahoo.documentapi.messagebus.protocol.ErrorPolicy; +import com.yahoo.documentapi.messagebus.protocol.ExternPolicy; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; +import com.yahoo.documentapi.messagebus.protocol.LoadBalancerPolicy; +import com.yahoo.documentapi.messagebus.protocol.LocalServicePolicy; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.RoundRobinPolicy; +import com.yahoo.documentapi.messagebus.protocol.SubsetServicePolicy; +import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; import com.yahoo.jrt.ListenFailedException; import com.yahoo.jrt.slobrok.api.IMirror; import com.yahoo.jrt.slobrok.api.Mirror; import com.yahoo.jrt.slobrok.server.Slobrok; -import com.yahoo.messagebus.*; +import com.yahoo.messagebus.DestinationSession; +import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.IntermediateSession; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.MessageBus; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.SourceSession; +import com.yahoo.messagebus.SourceSessionParams; import com.yahoo.messagebus.network.rpc.test.TestServer; -import com.yahoo.messagebus.routing.*; +import com.yahoo.messagebus.routing.HopBlueprint; +import com.yahoo.messagebus.routing.HopSpec; +import com.yahoo.messagebus.routing.PolicyDirective; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.messagebus.routing.RouteSpec; +import com.yahoo.messagebus.routing.RoutingNode; +import com.yahoo.messagebus.routing.RoutingPolicy; +import com.yahoo.messagebus.routing.RoutingSpec; +import com.yahoo.messagebus.routing.RoutingTableSpec; import com.yahoo.messagebus.test.Receptor; import org.junit.Before; import org.junit.Test; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -721,7 +761,7 @@ public class PolicyTestCase { throws InterruptedException, TimeoutException { for (int i = 0; i < TIMEOUT_MILLIS / 10; ++i) { - if (slobrok.lookup(pattern).length == numEntries) { + if (slobrok.lookup(pattern).size() == numEntries) { return; } Thread.sleep(10); diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java index 875ef276b01..89d5db62899 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java @@ -6,19 +6,32 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.jrt.ListenFailedException; import com.yahoo.jrt.slobrok.api.Mirror; import com.yahoo.jrt.slobrok.server.Slobrok; -import com.yahoo.messagebus.*; +import com.yahoo.messagebus.EmptyReply; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.MessageBus; +import com.yahoo.messagebus.MessageBusParams; +import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.network.Identity; import com.yahoo.messagebus.network.Network; import com.yahoo.messagebus.network.ServiceAddress; import com.yahoo.messagebus.network.rpc.RPCNetwork; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; import com.yahoo.messagebus.network.rpc.test.TestServer; -import com.yahoo.messagebus.routing.*; +import com.yahoo.messagebus.routing.HopSpec; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.messagebus.routing.RoutingNode; +import com.yahoo.messagebus.routing.RoutingSpec; +import com.yahoo.messagebus.routing.RoutingTableSpec; import com.yahoo.messagebus.test.Receptor; import com.yahoo.messagebus.test.SimpleProtocol; import com.yahoo.messagebus.test.SimpleReply; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import static org.junit.Assert.assertEquals; @@ -272,8 +285,8 @@ public class PolicyTestFrame { */ public boolean waitSlobrok(String pattern, int cnt) { for (int i = 0; i < 1000 && !Thread.currentThread().isInterrupted(); ++i) { - Mirror.Entry[] res = net.getMirror().lookup(pattern); - if (res.length == cnt) { + List<Mirror.Entry> res = net.getMirror().lookup(pattern); + if (res.size() == cnt) { return true; } try { Thread.sleep(10); } catch (InterruptedException e) { /* ignore */ } diff --git a/jrt/src/com/yahoo/jrt/slobrok/api/IMirror.java b/jrt/src/com/yahoo/jrt/slobrok/api/IMirror.java index 9b3bd81fc3c..0079e2c9d67 100644 --- a/jrt/src/com/yahoo/jrt/slobrok/api/IMirror.java +++ b/jrt/src/com/yahoo/jrt/slobrok/api/IMirror.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jrt.slobrok.api; +import java.util.List; + /** * Defines an interface for the name server lookup. * @@ -21,7 +23,7 @@ public interface IMirror { * @return a list of all matching services, with corresponding connect specs * @param pattern The pattern used for matching **/ - Mirror.Entry[] lookup(String pattern); + List<Mirror.Entry> lookup(String pattern); /** * Obtain the number of updates seen by this mirror. The value may wrap, but will never become 0 again. This can be diff --git a/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java b/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java index 127aa3d0d4f..c632191e31d 100644 --- a/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java +++ b/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java @@ -14,6 +14,7 @@ import com.yahoo.jrt.Values; import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.logging.Logger; import java.util.logging.Level; @@ -86,7 +87,7 @@ public class Mirror implements IMirror { } @Override - public Entry[] lookup(String pattern) { + public List<Entry> lookup(String pattern) { ArrayList<Entry> found = new ArrayList<>(); char[] p = pattern.toCharArray(); for (Entry specEntry : specs) { @@ -94,7 +95,7 @@ public class Mirror implements IMirror { found.add(specEntry); } } - return found.toArray(new Entry[found.size()]); + return found; } @Override diff --git a/jrt/tests/com/yahoo/jrt/SlobrokTest.java b/jrt/tests/com/yahoo/jrt/SlobrokTest.java index ee15c7cd1de..20266b0826a 100644 --- a/jrt/tests/com/yahoo/jrt/SlobrokTest.java +++ b/jrt/tests/com/yahoo/jrt/SlobrokTest.java @@ -1,10 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jrt; - import java.util.ArrayList; -import java.util.Arrays; import java.util.Comparator; +import java.util.List; import com.yahoo.jrt.slobrok.api.SlobrokList; import com.yahoo.jrt.slobrok.api.Mirror; @@ -17,7 +16,6 @@ import org.junit.Before; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; - public class SlobrokTest { private static class SpecList extends ArrayList<Mirror.Entry> { @@ -71,14 +69,13 @@ public class SlobrokTest { return a.compareTo(b); } }; - Mirror.Entry[] expect = - result.toArray(new Mirror.Entry[result.size()]); - Arrays.sort(expect, cmp); - Mirror.Entry[] actual = new Mirror.Entry[0]; + List<Entry> expect = result; + expect.sort(cmp); + List<Entry> actual = new ArrayList<>(); for (int i = 0; i < 1000; i++) { actual = mirror.lookup(pattern); - Arrays.sort(actual, cmp); - if (Arrays.equals(actual, expect)) { + actual.sort(cmp); + if (actual.equals(expect)) { // err("lookup successful for pattern: " + pattern); return; } @@ -87,18 +84,18 @@ public class SlobrokTest { error = true; err("lookup failed for pattern: " + pattern); err("actual values:"); - if (actual.length == 0) { + if (actual.isEmpty()) { err(" { EMPTY }"); } - for (int i = 0; i < actual.length; i++) { - err(" {" + actual[i].getName() + ", " + actual[i].getSpec() + "}"); + for (Entry e : actual) { + err(" {" + e.getName() + ", " + e.getSpec() + "}"); } err("expected values:"); - if (expect.length == 0) { + if (expect.isEmpty()) { err(" { EMPTY }"); } - for (int i = 0; i < expect.length; i++) { - err(" {" + expect[i].getName() + ", " + expect[i].getSpec() + "}"); + for (Entry e : expect) { + err(" {" + e.getName() + ", " + e.getSpec() + "}"); } } @@ -113,9 +110,9 @@ public class SlobrokTest { assertTrue(mirror.ready()); assertTrue(mirror.updates() > 0); - Mirror.Entry[] oneArr = mirror.lookup("*/*/*"); - assertTrue(oneArr.length == 1); - Mirror.Entry one = oneArr[0]; + List<Entry> oneArr = mirror.lookup("*/*/*"); + assertTrue(oneArr.size() == 1); + Mirror.Entry one = oneArr.get(0); assertTrue(one.equals(new Mirror.Entry(wantName, mySpec))); assertFalse(one.equals(new Mirror.Entry("B/x/w", mySpec))); assertFalse(one.equals(new Mirror.Entry(wantName, "foo:99"))); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/local/LocalWire.java b/messagebus/src/main/java/com/yahoo/messagebus/network/local/LocalWire.java index 4f5fd6ab30a..9f6295b1ad2 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/local/LocalWire.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/local/LocalWire.java @@ -41,7 +41,7 @@ public class LocalWire implements IMirror { } @Override - public Mirror.Entry[] lookup(String pattern) { + public List<Mirror.Entry> lookup(String pattern) { List<Mirror.Entry> out = new ArrayList<>(); Pattern regex = Pattern.compile(pattern.replace("*", "[a-zA-Z0-9_-]+")); for (String key : services.keySet()) { @@ -49,7 +49,7 @@ public class LocalWire implements IMirror { out.add(new Mirror.Entry(key, key)); } } - return out.toArray(new Mirror.Entry[out.size()]); + return out; } @Override diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCService.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCService.java index abe1b7b4db3..7c404207737 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCService.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCService.java @@ -4,6 +4,7 @@ package com.yahoo.messagebus.network.rpc; import com.yahoo.jrt.slobrok.api.IMirror; import com.yahoo.jrt.slobrok.api.Mirror; +import java.util.List; import java.util.concurrent.ThreadLocalRandom; /** @@ -18,7 +19,7 @@ public class RPCService { private final String pattern; private int addressIdx = ThreadLocalRandom.current().nextInt(Integer.MAX_VALUE); private int addressGen = 0; - private Mirror.Entry[] addressList = null; + private List<Mirror.Entry> addressList = null; /** * Create a new RPCService backed by the given network and using the given service pattern. @@ -51,9 +52,9 @@ public class RPCService { addressGen = mirror.updates(); addressList = mirror.lookup(pattern); } - if (addressList != null && addressList.length > 0) { - addressIdx = ++addressIdx % addressList.length; - Mirror.Entry entry = addressList[addressIdx]; + if (addressList != null && !addressList.isEmpty()) { + addressIdx = ++addressIdx % addressList.size(); + Mirror.Entry entry = addressList.get(addressIdx); return new RPCServiceAddress(entry.getName(), entry.getSpec()); } } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/test/TestServer.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/test/TestServer.java index d7ce31a5223..b17bb892f7d 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/test/TestServer.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/test/TestServer.java @@ -17,6 +17,7 @@ import com.yahoo.messagebus.routing.RoutingSpec; import com.yahoo.messagebus.routing.RoutingTableSpec; import com.yahoo.messagebus.test.SimpleProtocol; +import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Logger; @@ -121,8 +122,8 @@ public class TestServer { for (int i = 0; i < 6000 && !Thread.currentThread().isInterrupted(); ++i) { boolean done = true; for (String pattern : slobrokState.getPatterns()) { - Mirror.Entry[] res = net.getMirror().lookup(pattern); - if (res.length != slobrokState.getCount(pattern)) { + List<Mirror.Entry> res = net.getMirror().lookup(pattern); + if (res.size() != slobrokState.getCount(pattern)) { done = false; } } diff --git a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/ServiceAddressTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/ServiceAddressTestCase.java index 476d85f59e9..1dbb30de585 100755 --- a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/ServiceAddressTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/ServiceAddressTestCase.java @@ -11,6 +11,7 @@ import org.junit.Before; import org.junit.Test; import java.net.UnknownHostException; +import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -67,8 +68,8 @@ public class ServiceAddressTestCase { private boolean waitSlobrok(String pattern, int num) { for (int i = 0; i < 1000 && !Thread.currentThread().isInterrupted(); ++i) { - Mirror.Entry[] res = network.getMirror().lookup(pattern); - if (res.length == num) { + List<Mirror.Entry> res = network.getMirror().lookup(pattern); + if (res.size() == num) { return true; } try { diff --git a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/SlobrokTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/SlobrokTestCase.java index c6737385f4c..dd779fd84c0 100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/SlobrokTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/SlobrokTestCase.java @@ -31,8 +31,8 @@ public class SlobrokTestCase { lst.add(new Mirror.Entry(fullName, spec)); return this; } - public Mirror.Entry[] toArray() { - return lst.toArray(new Mirror.Entry[lst.size()]); + public List<Mirror.Entry> toArray() { + return lst; } } @@ -44,18 +44,18 @@ public class SlobrokTestCase { int port2; int port3; - void check(RPCNetwork net, String pattern, Mirror.Entry[] expect) { + void check(RPCNetwork net, String pattern, List<Mirror.Entry> expect) { Comparator<Mirror.Entry> cmp = new Comparator<Mirror.Entry>() { public int compare(Mirror.Entry a, Mirror.Entry b) { return a.compareTo(b); } }; - Arrays.sort(expect, cmp); - Mirror.Entry[] actual = null; + expect.sort(cmp); + List<Mirror.Entry> actual = null; for (int i = 0; i < 1000; i++) { actual = net.getMirror().lookup(pattern); - Arrays.sort(actual, cmp); - if (Arrays.equals(actual, expect)) { + actual.sort(cmp); + if (actual.equals(expect)) { System.out.printf("lookup successful for pattern: %s\n", pattern); return; } @@ -65,7 +65,7 @@ public class SlobrokTestCase { } System.out.printf("lookup failed for pattern: %s\n", pattern); System.out.printf("actual values:\n"); - if (actual == null || actual.length == 0) { + if (actual == null || actual.isEmpty()) { System.out.printf(" { EMPTY }\n"); } else { for (Mirror.Entry entry : actual) { @@ -73,7 +73,7 @@ public class SlobrokTestCase { } } System.out.printf("expected values:\n"); - if (expect.length == 0) { + if (expect.isEmpty()) { System.out.printf(" { EMPTY }\n"); } else { for (Mirror.Entry entry : expect) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilter.java index 28ead318cc0..d15cd288f39 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilter.java @@ -2,10 +2,11 @@ package com.yahoo.vespa.hosted.provision.restapi.v2.filter; import com.google.inject.Inject; -import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.http.filter.DiscFilterRequest; +import com.yahoo.jdisc.http.filter.FilterConfig; import com.yahoo.jdisc.http.filter.SecurityRequestFilter; +import com.yahoo.vespa.athenz.utils.AthenzIdentities; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.restapi.v2.ErrorResponse; import com.yahoo.yolean.chain.After; @@ -31,8 +32,12 @@ public class AuthorizationFilter implements SecurityRequestFilter { private final BiConsumer<ErrorResponse, ResponseHandler> rejectAction; @Inject - public AuthorizationFilter(Zone zone, NodeRepository nodeRepository) { - this.authorizer = new Authorizer(zone.system(), nodeRepository); + public AuthorizationFilter(NodeRepository nodeRepository, FilterConfig filterConfig) { + this.authorizer = new Authorizer(nodeRepository, + AthenzIdentities.from(filterConfig.getInitParameter("controller.identity")), + AthenzIdentities.from(filterConfig.getInitParameter("configserver.identity")), + AthenzIdentities.from(filterConfig.getInitParameter("proxy.identity")), + AthenzIdentities.from(filterConfig.getInitParameter("tenant-host.identity"))); this.rejectAction = AuthorizationFilter::logAndReject; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/Authorizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/Authorizer.java index afcde0949e3..44636727531 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/Authorizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/Authorizer.java @@ -2,9 +2,7 @@ package com.yahoo.vespa.hosted.provision.restapi.v2.filter; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import org.apache.http.NameValuePair; @@ -15,7 +13,6 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; @@ -32,19 +29,15 @@ import java.util.stream.Collectors; public class Authorizer implements BiPredicate<NodePrincipal, URI> { private final NodeRepository nodeRepository; private final AthenzIdentity controllerIdentity; - private final AthenzIdentity configServerIdentity = new AthenzService("vespa.vespa", "configserver"); - private final AthenzIdentity proxyIdentity = new AthenzService("vespa.vespa", "proxy"); - private final AthenzIdentity tenantIdentity = new AthenzService("vespa.vespa", "tenant-host"); private final Set<AthenzIdentity> trustedIdentities; private final Set<AthenzIdentity> hostAdminIdentities; - public Authorizer(SystemName system, NodeRepository nodeRepository) { + Authorizer(NodeRepository nodeRepository, AthenzIdentity controllerIdentity, AthenzIdentity configServerIdentity, + AthenzIdentity proxyIdentity, AthenzIdentity tenantIdentity) { this.nodeRepository = nodeRepository; - controllerIdentity = system == SystemName.main - ? new AthenzService("vespa.vespa", "hosting") - : new AthenzService("vespa.vespa.cd", "hosting"); - this.trustedIdentities = new HashSet<>(Arrays.asList(controllerIdentity, configServerIdentity)); - this.hostAdminIdentities = new HashSet<>(Arrays.asList(controllerIdentity, configServerIdentity, proxyIdentity, tenantIdentity)); + this.controllerIdentity = controllerIdentity; + this.trustedIdentities = Set.of(controllerIdentity, configServerIdentity); + this.hostAdminIdentities = Set.of(controllerIdentity, configServerIdentity, proxyIdentity, tenantIdentity); } /** Returns whether principal is authorized to access given URI */ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilterTest.java index 2969b608d3a..2f1755dd106 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilterTest.java @@ -2,28 +2,23 @@ package com.yahoo.vespa.hosted.provision.restapi.v2.filter; import com.yahoo.application.container.handler.Request.Method; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Zone; +import com.yahoo.container.FilterConfigProvider; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.hosted.provision.restapi.v2.filter.FilterTester.Request; import com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors; import com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository; -import org.junit.Before; import org.junit.Test; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + /** * @author mpolden */ public class AuthorizationFilterTest { - private FilterTester tester; - - @Before - public void before() { - tester = filterTester(SystemName.main); - } + private final FilterTester tester = filterTester(); @Test public void filter() { @@ -43,11 +38,12 @@ public class AuthorizationFilterTest { tester.assertSuccess(new Request(Method.GET, "/nodes/v2/node/foo").commonName("foo")); } - private static FilterTester filterTester(SystemName system) { - Zone zone = new Zone(system, Environment.prod, RegionName.defaultName()); + private static FilterTester filterTester() { + Map<String, String> params = Stream.of("controller", "configserver", "proxy", "tenant-host") + .collect(Collectors.toMap(e -> e + ".identity", e -> "vespa." + e)); return new FilterTester(new AuthorizationFilter( - zone, - new MockNodeRepository(new MockCurator(), new MockNodeFlavors()))); + new MockNodeRepository(new MockCurator(), new MockNodeFlavors()), + FilterConfigProvider.from("my-filter", AuthorizationFilter.class.getName(), params).get())); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizerTest.java index d696328cd7f..b539420a55a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizerTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.provision.restapi.v2.filter; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.athenz.utils.AthenzIdentities; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors; @@ -17,7 +17,6 @@ import java.util.List; import java.util.Optional; import java.util.Set; -import static java.util.Collections.emptyList; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -26,14 +25,22 @@ import static org.junit.Assert.assertTrue; */ public class AuthorizerTest { + private static final String CONTROLLER_IDENTITY = "vespa.controller"; + private static final String CONFIG_SERVER_IDENTITY = "vespa.configserver"; + private static final String PROXY_IDENTITY = "vespa.proxy"; + private static final String TENANT_HOST_IDENTITY = "vespa.tenant-host"; + private Authorizer authorizer; - private MockNodeRepository nodeRepository; @Before public void before() { NodeFlavors flavors = new MockNodeFlavors(); - nodeRepository = new MockNodeRepository(new MockCurator(), flavors); - authorizer = new Authorizer(SystemName.main, nodeRepository); + MockNodeRepository nodeRepository = new MockNodeRepository(new MockCurator(), flavors); + authorizer = new Authorizer(nodeRepository, + AthenzIdentities.from(CONTROLLER_IDENTITY), + AthenzIdentities.from(CONFIG_SERVER_IDENTITY), + AthenzIdentities.from(PROXY_IDENTITY), + AthenzIdentities.from(TENANT_HOST_IDENTITY)); Set<String> ipAddresses = Set.of("127.0.0.1", "::1"); Flavor flavor = flavors.getFlavorOrThrow("default"); @@ -98,13 +105,11 @@ public class AuthorizerTest { assertTrue(authorizedTenantHostNode("host1", "/athenz/v1/provider/identity-document/node/child1-1")); // Trusted services can access everything in their own system - assertFalse(authorizedController("vespa.vespa.cd.hosting", "/")); // Wrong system - assertTrue(new Authorizer(SystemName.cd, nodeRepository).test(NodePrincipal.withAthenzIdentity("vespa.vespa.cd.hosting", emptyList()), uri("/"))); - assertTrue(authorizedController("vespa.vespa.hosting", "/")); - assertTrue(authorizedController("vespa.vespa.configserver", "/")); - assertTrue(authorizedController("vespa.vespa.hosting", "/nodes/v2/node/")); - assertTrue(authorizedController("vespa.vespa.hosting", "/nodes/v2/node/node1")); - assertTrue(authorizedController("vespa.vespa.configserver", "/nodes/v2/node/node1")); + assertTrue(authorizedController(CONTROLLER_IDENTITY, "/")); + assertTrue(authorizedController(CONFIG_SERVER_IDENTITY, "/")); + assertTrue(authorizedController(CONTROLLER_IDENTITY, "/nodes/v2/node/")); + assertTrue(authorizedController(CONTROLLER_IDENTITY, "/nodes/v2/node/node1")); + assertTrue(authorizedController(CONFIG_SERVER_IDENTITY, "/nodes/v2/node/node1")); } @Test @@ -145,14 +150,14 @@ public class AuthorizerTest { assertTrue(authorizedTenantHostNode("proxy1-host", "/flags/v1/data")); assertFalse(authorizedTenantHostNode("proxy1-host", "/flags/v1/data/flagid")); assertFalse(authorizedTenantHostNode("proxy1-host", "/flags/v1/foo")); - assertTrue(authorizedController("vespa.vespa.configserver", "/flags/v1/data")); - assertFalse(authorizedController("vespa.vespa.configserver", "/flags/v1/data/flagid")); - assertFalse(authorizedController("vespa.vespa.configserver", "/flags/v1/foo")); + assertTrue(authorizedController(CONFIG_SERVER_IDENTITY, "/flags/v1/data")); + assertFalse(authorizedController(CONFIG_SERVER_IDENTITY, "/flags/v1/data/flagid")); + assertFalse(authorizedController(CONFIG_SERVER_IDENTITY, "/flags/v1/foo")); // Controller can access everything - assertTrue(authorizedController("vespa.vespa.hosting", "/flags/v1/data")); - assertTrue(authorizedController("vespa.vespa.hosting", "/flags/v1/data/flagid")); - assertTrue(authorizedController("vespa.vespa.hosting", "/flags/v1/foo")); + assertTrue(authorizedController(CONTROLLER_IDENTITY, "/flags/v1/data")); + assertTrue(authorizedController(CONTROLLER_IDENTITY, "/flags/v1/data/flagid")); + assertTrue(authorizedController(CONTROLLER_IDENTITY, "/flags/v1/foo")); } @Test @@ -170,19 +175,19 @@ public class AuthorizerTest { } private boolean authorizedTenantNode(String hostname, String path) { - return authorized(NodePrincipal.withAthenzIdentity("vespa.vespa.tenant", hostname, emptyList()), path); + return authorized(NodePrincipal.withAthenzIdentity("vespa.vespa.tenant", hostname, List.of()), path); } private boolean authorizedTenantHostNode(String hostname, String path) { - return authorized(NodePrincipal.withAthenzIdentity("vespa.vespa.tenant-host", hostname, emptyList()), path); + return authorized(NodePrincipal.withAthenzIdentity(TENANT_HOST_IDENTITY, hostname, List.of()), path); } private boolean authorizedLegacyNode(String hostname, String path) { - return authorized(NodePrincipal.withLegacyIdentity(hostname, emptyList()), path); + return authorized(NodePrincipal.withLegacyIdentity(hostname, List.of()), path); } private boolean authorizedController(String controllerIdentity, String path) { - return authorized(NodePrincipal.withAthenzIdentity(controllerIdentity, emptyList()), path); + return authorized(NodePrincipal.withAthenzIdentity(controllerIdentity, List.of()), path); } private boolean authorized(NodePrincipal principal, String path) { diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitor.java index 482969c6625..1a402419ac0 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitor.java @@ -67,7 +67,7 @@ public class SlobrokMonitor implements AutoCloseable { } List<Mirror.Entry> lookup(String pattern) { - return Arrays.asList(mirror.lookup(pattern)); + return mirror.lookup(pattern); } @Override @@ -76,6 +76,6 @@ public class SlobrokMonitor implements AutoCloseable { } boolean registeredInSlobrok(String slobrokServiceName) { - return mirror.lookup(slobrokServiceName).length > 0; + return !mirror.lookup(slobrokServiceName).isEmpty(); } } |