diff options
252 files changed, 5564 insertions, 7996 deletions
diff --git a/chain/pom.xml b/chain/pom.xml index 8f76598c51e..6b4da4f8c72 100755 --- a/chain/pom.xml +++ b/chain/pom.xml @@ -39,11 +39,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-core</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>provided-dependencies</artifactId> <version>${project.version}</version> diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java index 498037a4984..bb3d7e049d1 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java @@ -15,7 +15,12 @@ import com.yahoo.vespa.clustercontroller.core.Timer; import com.yahoo.vespa.clustercontroller.core.ContentCluster; import com.yahoo.vespa.clustercontroller.core.listeners.NodeAddedOrRemovedListener; -import java.util.*; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.StringTokenizer; +import java.util.TreeMap; import java.util.logging.Logger; public class SlobrokClient implements NodeLookup { @@ -190,8 +195,8 @@ public class SlobrokClient implements NodeLookup { private Map<Node, SlobrokData> getSlobrokData(String pattern) { Map<Node, SlobrokData> result = new TreeMap<>(); - Mirror.Entry[] entries = mirror.lookup(pattern); - log.log(LogLevel.SPAM, "Looking for slobrok entries with pattern '" + pattern + "'. Found " + entries.length + " entries."); + List<Mirror.Entry> entries = mirror.lookup(pattern); + log.log(LogLevel.SPAM, "Looking for slobrok entries with pattern '" + pattern + "'. Found " + entries.size() + " entries."); for (Mirror.Entry entry : entries) { StringTokenizer st = new StringTokenizer(entry.getName(), "/"); String addressType = st.nextToken(); @@ -209,7 +214,7 @@ public class SlobrokClient implements NodeLookup { private static class SlobrokData { public Node node; - public String rpcAddress; + String rpcAddress; SlobrokData(Node node, String rpcAddress) { this.node = node; diff --git a/config-model/pom.xml b/config-model/pom.xml index 965d97bb605..715718f8bb5 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -293,11 +293,6 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-core</artifactId> - <scope>test</scope> - </dependency> </dependencies> <build> diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java index db072afd201..65d94807d7f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java @@ -3,7 +3,6 @@ package com.yahoo.config.model.producer; import com.google.common.annotations.Beta; import com.yahoo.config.ConfigInstance; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.subscription.ConfigInstanceUtil; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java index 97236222338..ecfa9a9e53e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java @@ -34,9 +34,9 @@ import java.util.TreeMap; public final class DocumentProtocol implements Protocol, DocumentrouteselectorpolicyConfig.Producer { private static final String NAME = "document"; - private ApplicationSpec application; - private RoutingTableSpec routingTable; - ConfigModelRepo repo; + private final ApplicationSpec application; + private final RoutingTableSpec routingTable; + private final ConfigModelRepo repo; public static String getIndexedRouteName(String configId) { return configId + "-index"; @@ -51,7 +51,7 @@ public final class DocumentProtocol implements Protocol, Documentrouteselectorpo * * @param plugins The plugins to reflect on. */ - public DocumentProtocol(ConfigModelRepo plugins) { + DocumentProtocol(ConfigModelRepo plugins) { application = createApplicationSpec(plugins); routingTable = createRoutingTable(plugins); this.repo = plugins; @@ -249,7 +249,13 @@ public final class DocumentProtocol implements Protocol, Documentrouteselectorpo route.addHop("indexing"); table.addRoute(route); - table.addRoute(new RouteSpec("default-get").addHop("indexing")); + if (content.size() == 1) { + table.addRoute(new RouteSpec("default-get").addHop("[Content:cluster=" + content.get(0).getConfigId() + "]")); + } else { + //TODO This should ideally skip indexing and go directly to correct cluster. + // But will handle the single cluster for now. + table.addRoute(new RouteSpec("default-get").addHop("indexing")); + } } private static boolean indexingHopExists(RoutingTableSpec table) { diff --git a/config-model/src/test/cfg/routing/replacehop/messagebus.cfg b/config-model/src/test/cfg/routing/replacehop/messagebus.cfg index 12701af6db3..8717deb50a5 100755 --- a/config-model/src/test/cfg/routing/replacehop/messagebus.cfg +++ b/config-model/src/test/cfg/routing/replacehop/messagebus.cfg @@ -12,7 +12,7 @@ routingtable[0].hop[2].ignoreresult false routingtable[0].route[0].name "default" routingtable[0].route[0].hop[0] "indexing" routingtable[0].route[1].name "default-get" -routingtable[0].route[1].hop[0] "indexing" +routingtable[0].route[1].hop[0] "[Content:cluster=music]" routingtable[0].route[2].name "music" routingtable[0].route[2].hop[0] "[MessageType:music]" routingtable[0].route[3].name "music-direct" diff --git a/config-model/src/test/cfg/routing/replaceroute/messagebus.cfg b/config-model/src/test/cfg/routing/replaceroute/messagebus.cfg index d3af5b36c26..7eae6e8ebc5 100755 --- a/config-model/src/test/cfg/routing/replaceroute/messagebus.cfg +++ b/config-model/src/test/cfg/routing/replaceroute/messagebus.cfg @@ -9,7 +9,7 @@ routingtable[0].hop[1].ignoreresult false routingtable[0].route[0].name "default" routingtable[0].route[0].hop[0] "foo" routingtable[0].route[1].name "default-get" -routingtable[0].route[1].hop[0] "indexing" +routingtable[0].route[1].hop[0] "[Content:cluster=music]" routingtable[0].route[2].name "music" routingtable[0].route[2].hop[0] "[MessageType:music]" routingtable[0].route[3].name "music-direct" diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentBaseTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentBaseTest.java index 0432b181c2c..f5642922d5b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentBaseTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentBaseTest.java @@ -1,6 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; +import com.yahoo.messagebus.routing.RouteSpec; + +import static org.junit.Assert.assertEquals; + public class ContentBaseTest { public static String getHosts() { return "<?xml version='1.0' encoding='utf-8' ?>" + @@ -10,4 +14,12 @@ public class ContentBaseTest { " </host>" + "</hosts>"; } + + static void assertRoute(RouteSpec r, String name, String... hops) { + assertEquals(name, r.getName()); + assertEquals(hops.length, r.getNumHops()); + for(int i = 0; i < hops.length; i++) { + assertEquals(hops[i], r.getHop(i)); + } + } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 0e200efd688..7c365859862 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -10,6 +10,7 @@ import com.yahoo.config.model.test.TestRoot; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; +import com.yahoo.messagebus.routing.RoutingTableSpec; import com.yahoo.metrics.MetricsmanagerConfig; import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; import com.yahoo.vespa.config.content.FleetcontrollerConfig; @@ -26,6 +27,8 @@ import com.yahoo.vespa.model.content.engines.ProtonEngine; import com.yahoo.vespa.model.content.utils.ContentClusterBuilder; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import com.yahoo.vespa.model.content.utils.SearchDefinitionBuilder; +import com.yahoo.vespa.model.routing.DocumentProtocol; +import com.yahoo.vespa.model.routing.Routing; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.Rule; @@ -853,44 +856,73 @@ public class ContentClusterTest extends ContentBaseTest { } } - @Test - public void all_clusters_bucket_spaces_config_contains_mappings_across_all_clusters() { + private VespaModel createDualContentCluster() { String xml = "<services>" + - "<admin version=\"2.0\">" + - " <adminserver hostalias=\"node0\"/>" + - "</admin>" + - "<content version=\"1.0\" id=\"foocluster\">" + - " <redundancy>1</redundancy>" + - " <documents>" + - " <document type=\"bunnies\" mode=\"index\"/>" + - " <document type=\"hares\" mode=\"index\"/>" + - " </documents>" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"node0\"/>" + - " </group>" + - "</content>" + - "<content version=\"1.0\" id=\"barcluster\">" + - " <redundancy>1</redundancy>" + - " <documents>" + - " <document type=\"rabbits\" mode=\"index\" global=\"true\"/>" + - " </documents>" + - " <group>" + - " <node distribution-key=\"0\" hostalias=\"node0\"/>" + - " </group>" + - "</content>" + - "</services>"; + "<admin version=\"2.0\">" + + " <adminserver hostalias=\"node0\"/>" + + "</admin>" + + "<content version=\"1.0\" id=\"foo_c\">" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type=\"bunnies\" mode=\"index\"/>" + + " <document type=\"hares\" mode=\"index\"/>" + + " </documents>" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"node0\"/>" + + " </group>" + + "</content>" + + "<content version=\"1.0\" id=\"bar_c\">" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type=\"rabbits\" mode=\"index\" global=\"true\"/>" + + " </documents>" + + " <group>" + + " <node distribution-key=\"0\" hostalias=\"node0\"/>" + + " </group>" + + "</content>" + + "</services>"; List<String> sds = ApplicationPackageUtils.generateSearchDefinitions("bunnies", "hares", "rabbits"); - VespaModel model = new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); + return new VespaModelCreatorWithMockPkg(getHosts(), xml, sds).create(); + } + @Test + public void all_clusters_bucket_spaces_config_contains_mappings_across_all_clusters() { + VespaModel model = createDualContentCluster(); AllClustersBucketSpacesConfig.Builder builder = new AllClustersBucketSpacesConfig.Builder(); model.getConfig(builder, "client"); AllClustersBucketSpacesConfig config = builder.build(); assertEquals(2, config.cluster().size()); - assertClusterHasBucketSpaceMappings(config, "foocluster", Arrays.asList("bunnies", "hares"), Collections.emptyList()); - assertClusterHasBucketSpaceMappings(config, "barcluster", Collections.emptyList(), Collections.singletonList("rabbits")); + assertClusterHasBucketSpaceMappings(config, "foo_c", Arrays.asList("bunnies", "hares"), Collections.emptyList()); + assertClusterHasBucketSpaceMappings(config, "bar_c", Collections.emptyList(), Collections.singletonList("rabbits")); + } + @Test + public void test_routing_with_multiple_clusters() { + VespaModel model = createDualContentCluster(); + Routing routing = model.getRouting(); + assertNotNull(routing); + assertEquals("[]", routing.getErrors().toString()); + assertEquals(1, routing.getProtocols().size()); + DocumentProtocol protocol = (DocumentProtocol) routing.getProtocols().get(0); + RoutingTableSpec spec = protocol.getRoutingTableSpec(); + assertEquals(3, spec.getNumHops()); + assertEquals("docproc/cluster.bar_c.indexing/chain.indexing", spec.getHop(0).getName()); + assertEquals("docproc/cluster.foo_c.indexing/chain.indexing", spec.getHop(1).getName()); + assertEquals("indexing", spec.getHop(2).getName()); + + assertEquals(10, spec.getNumRoutes()); + assertRoute(spec.getRoute(0), "bar_c", "[MessageType:bar_c]"); + assertRoute(spec.getRoute(1), "bar_c-direct", "[Content:cluster=bar_c]"); + assertRoute(spec.getRoute(2), "bar_c-index", "docproc/cluster.bar_c.indexing/chain.indexing", "[Content:cluster=bar_c]"); + assertRoute(spec.getRoute(3), "default", "indexing"); + assertRoute(spec.getRoute(4), "default-get", "indexing"); + assertRoute(spec.getRoute(5), "foo_c", "[MessageType:foo_c]"); + assertRoute(spec.getRoute(6), "foo_c-direct", "[Content:cluster=foo_c]"); + assertRoute(spec.getRoute(7), "foo_c-index", "docproc/cluster.foo_c.indexing/chain.indexing", "[Content:cluster=foo_c]"); + assertRoute(spec.getRoute(8), "storage/cluster.bar_c", "route:bar_c"); + assertRoute(spec.getRoute(9), "storage/cluster.foo_c", "route:foo_c"); } private ContentCluster createWithZone(String clusterXml, Zone zone) throws Exception { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java index bf8e2a353cf..919fec5be2f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.content; import com.yahoo.cloud.config.ClusterListConfig; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.messagebus.routing.RouteSpec; import com.yahoo.messagebus.routing.RoutingTableSpec; import com.yahoo.vespa.config.content.core.StorServerConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; @@ -158,21 +157,13 @@ public class IndexedTest extends ContentBaseTest { assertEquals("jdisc/chain.indexing", spec.getHop(1).getName()); assertRoute(spec.getRoute(0), "default", "indexing"); - assertRoute(spec.getRoute(1), "default-get", "indexing"); + assertRoute(spec.getRoute(1), "default-get", "[Content:cluster=test]"); assertRoute(spec.getRoute(2), "storage/cluster.test", "route:test"); assertRoute(spec.getRoute(3), "test", "[MessageType:test]"); assertRoute(spec.getRoute(4), "test-direct", "[Content:cluster=test]"); assertRoute(spec.getRoute(5), "test-index", "jdisc/chain.indexing", "[Content:cluster=test]"); } - private static void assertRoute(RouteSpec r, String name, String... hops) { - assertEquals(name, r.getName()); - assertEquals(hops.length, r.getNumHops()); - for(int i = 0; i < hops.length; i++) { - assertEquals(hops[i], r.getHop(i)); - } - } - @Test public void requireProtonStreamingOnly() { diff --git a/config-provisioning/src/main/resources/configdefinitions/config-server-security.def b/config-provisioning/src/main/resources/configdefinitions/config-server-security.def new file mode 100644 index 00000000000..cfd5c9c04e5 --- /dev/null +++ b/config-provisioning/src/main/resources/configdefinitions/config-server-security.def @@ -0,0 +1,9 @@ +# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=config.provisioning + +athenzProviderHostname string +controllerHostIdentity string +configServerHostIdentity string +proxyHostIdentity string +tenantHostIdentity string +tenantIdentity string diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UrlDownloadRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UrlDownloadRpcServer.java index 3c24cb58cff..8c25593dde0 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UrlDownloadRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UrlDownloadRpcServer.java @@ -114,7 +114,7 @@ public class UrlDownloadRpcServer { } private static String urlToDirName(String uri) { - return String.valueOf(XXHashFactory.nativeInstance().hash64().hash(ByteBuffer.wrap(Utf8.toBytes(uri)), 0)); + return String.valueOf(XXHashFactory.fastestJavaInstance().hash64().hash(ByteBuffer.wrap(Utf8.toBytes(uri)), 0)); } private static void setIfModifiedSince(HttpURLConnection connection, File downloadDir) throws IOException { diff --git a/config/src/apps/vespa-get-config/getconfig.cpp b/config/src/apps/vespa-get-config/getconfig.cpp index d12a278cc38..290beacf5a4 100644 --- a/config/src/apps/vespa-get-config/getconfig.cpp +++ b/config/src/apps/vespa-get-config/getconfig.cpp @@ -51,6 +51,7 @@ GetConfig::usage() fprintf(stderr, "-i configId (config id, optional)\n"); fprintf(stderr, "-j (output config as json, optional)\n"); fprintf(stderr, "-l (output config in legacy cfg format, optional)\n"); + fprintf(stderr, "-g generation (config generation, optional)\n"); fprintf(stderr, "-a schema (config def schema file, optional)\n"); fprintf(stderr, "-v defVersion (config definition version, optional, deprecated)\n"); fprintf(stderr, "-m defMd5 (definition md5sum, optional)\n"); @@ -107,6 +108,7 @@ GetConfig::Main() bool printAsJson = false; int traceLevel = config::protocol::readTraceLevel(); const char *vespaVersionString = nullptr; + int64_t generation = 0; if (configId == NULL) { configId = ""; @@ -119,7 +121,7 @@ GetConfig::Main() const char *optArg = NULL; int optInd = 0; - while ((c = GetOpt("a:n:v:i:jlm:c:t:V:w:r:s:p:dh", optArg, optInd)) != -1) { + while ((c = GetOpt("a:n:v:g:i:jlm:c:t:V:w:r:s:p:dh", optArg, optInd)) != -1) { int retval = 1; switch (c) { case 'a': @@ -130,6 +132,9 @@ GetConfig::Main() break; case 'v': break; + case 'g': + generation = atoll(optArg); + break; case 'i': configId = optArg; break; @@ -220,7 +225,7 @@ GetConfig::Main() FRTConfigRequestFactory requestFactory(protocolVersion, traceLevel, vespaVersion, config::protocol::readProtocolCompressionType()); FRTConnection connection(spec, *_supervisor, TimingValues()); ConfigKey key(configId, defName, defNamespace, defMD5, defSchema); - ConfigState state(configMD5, 0, false); + ConfigState state(configMD5, generation, false); FRTConfigRequest::UP request = requestFactory.createConfigRequest(key, &connection, state, serverTimeout * 1000); _target->InvokeSync(request->getRequest(), clientTimeout); // seconds diff --git a/configdefinitions/src/vespa/routing-provider.def b/configdefinitions/src/vespa/routing-provider.def index 10d7be11e4d..eedf4f3c86f 100755 --- a/configdefinitions/src/vespa/routing-provider.def +++ b/configdefinitions/src/vespa/routing-provider.def @@ -1,6 +1,8 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # Configuration for the routing config provider +# TODO Remove once YCA filter is removed + namespace=cloud.config # disabled by default, automatically enabled for hosted vespa diff --git a/configdefinitions/src/vespa/routing.def b/configdefinitions/src/vespa/routing.def index eeb37abb4e9..043ef1fff68 100755 --- a/configdefinitions/src/vespa/routing.def +++ b/configdefinitions/src/vespa/routing.def @@ -1,6 +1,8 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. # List of routing hosts populated from the zone-application in Hosted mode +# TODO Remove once YCA filter is removed + namespace=cloud.config # Host name(s) of routing/proxy nodes diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java index 877b2acb86f..4fbda42fdc7 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java @@ -32,6 +32,7 @@ import java.util.Set; import java.util.stream.Collectors; import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; +import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER; /** @@ -43,12 +44,12 @@ import static com.yahoo.config.model.api.container.ContainerServiceType.QRSERVER public class ConfigConvergenceChecker extends AbstractComponent { private static final ApplicationId routingApplicationId = ApplicationId.from("hosted-vespa", "routing", "default"); - private static final String nodeAdminName = "node-admin"; private static final String statePath = "/state/v1/"; private static final String configSubPath = "config"; private final static Set<String> serviceTypesToCheck = new HashSet<>(Arrays.asList( CONTAINER.serviceName, QRSERVER.serviceName, + LOGSERVER_CONTAINER.serviceName, "searchnode", "storagenode", "distributor" diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java index fc3f55cc6aa..a10b8d9c6fb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java @@ -61,7 +61,7 @@ public class FileDBRegistry implements FileRegistry { } private static String uriToRelativeFile(String uri) { - String relative = "uri/" + String.valueOf(XXHashFactory.nativeInstance().hash64().hash(ByteBuffer.wrap(Utf8.toBytes(uri)), 0)); + String relative = "uri/" + XXHashFactory.fastestJavaInstance().hash64().hash(ByteBuffer.wrap(Utf8.toBytes(uri)), 0); if (uri.endsWith(".json")) { relative += ".json"; } else if (uri.endsWith(".json.lz4")) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/RoutingProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/RoutingProducer.java index 7c85b51c920..07e57270dbd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/RoutingProducer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/RoutingProducer.java @@ -17,6 +17,8 @@ import java.util.Set; * * @author Christian Andersen */ +// TODO Delete once YCA filter is removed +@Deprecated(forRemoval = true) public class RoutingProducer implements RoutingConfig.Producer { static final ApplicationName ROUTING_APPLICATION = ApplicationName.from("routing"); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModelConfigProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModelConfigProvider.java index 75f53667c4a..9631713c870 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModelConfigProvider.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/SuperModelConfigProvider.java @@ -25,8 +25,9 @@ public class SuperModelConfigProvider implements LbServicesConfig.Producer, Rout private final SuperModel superModel; private final LbServicesProducer lbProd; - private final RoutingProducer zoneProd; + @SuppressWarnings("removal") private final RoutingProducer zoneProd; + @SuppressWarnings("removal") // For RoutingProducer public SuperModelConfigProvider(SuperModel superModel, Zone zone, FlagSource flagSource) { this.superModel = superModel; this.lbProd = new LbServicesProducer(Collections.unmodifiableMap(superModel.getModelsPerTenant()), zone, flagSource); diff --git a/configserver/src/main/sh/start-logd b/configserver/src/main/sh/start-logd index 2b6ed01f09c..df5f6b00209 100644 --- a/configserver/src/main/sh/start-logd +++ b/configserver/src/main/sh/start-logd @@ -74,7 +74,7 @@ ROOT=${VESPA_HOME%/} export VESPA_CONFIG_ID="file:$VESPA_HOME/conf/logd/logd.cfg" -if [ "$multitenant" = "true" ]; then +if [ "$cloudconfig_server__multitenant" = "true" ]; then PIDFILE_LOGD=var/run/logd.pid VESPA_SERVICE_NAME=logd export VESPA_SERVICE_NAME diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/model/RoutingProducerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/model/RoutingProducerTest.java index 4d968a58bb6..c6a607f81b1 100755 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/model/RoutingProducerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/model/RoutingProducerTest.java @@ -27,6 +27,7 @@ import static org.junit.Assert.assertThat; /** * @author Christian Andersen */ +@SuppressWarnings("removal") // For RoutingProducer public class RoutingProducerTest { @Test public void testNodesFromRoutingAppOnly() throws Exception { diff --git a/container-di/pom.xml b/container-di/pom.xml index 216f9e9628b..d4f97dced40 100644 --- a/container-di/pom.xml +++ b/container-di/pom.xml @@ -39,11 +39,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-core</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>provided-dependencies</artifactId> <version>${project.version}</version> diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Highlight.java b/container-search/src/main/java/com/yahoo/prelude/query/Highlight.java index 44691b04b84..ac2e5f70fe8 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Highlight.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Highlight.java @@ -28,12 +28,6 @@ public class Highlight implements Cloneable { public Highlight() {} private void addHighlightItem(String key, Item value) { - /*List<IndexedItem> l = highlightItems.get(key); - if (l == null) { - l = new ArrayList<IndexedItem>(); - highlightItems.put(key, l); - } - l.addField(value);*/ AndItem item = highlightItems.get(key); if (item == null) { item = new AndItem(); @@ -54,6 +48,7 @@ public class Highlight implements Cloneable { /** * Add custom highlight phrase + * * @param field Field name * @param phrase List of terms to be highlighted as a phrase */ diff --git a/container/pom.xml b/container/pom.xml index 6007c066770..518eb9d984a 100644 --- a/container/pom.xml +++ b/container/pom.xml @@ -39,10 +39,6 @@ </exclusion> <exclusion> <groupId>com.yahoo.vespa</groupId> - <artifactId>metrics</artifactId> - </exclusion> - <exclusion> - <groupId>com.yahoo.vespa</groupId> <artifactId>vespaclient-core</artifactId> </exclusion> <exclusion> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java index c4b69ce5588..2fc852d79d5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java @@ -23,28 +23,23 @@ import java.util.Set; public class RoutingPolicy { private final ApplicationId owner; + private final ClusterSpec.Id cluster; private final ZoneId zone; - private final HostName alias; private final HostName canonicalName; private final Optional<String> dnsZone; private final Set<RotationName> rotations; /** DO NOT USE. Public for serialization purposes */ - public RoutingPolicy(ApplicationId owner, ZoneId zone, HostName alias, HostName canonicalName, + public RoutingPolicy(ApplicationId owner, ClusterSpec.Id cluster, ZoneId zone, HostName canonicalName, Optional<String> dnsZone, Set<RotationName> rotations) { this.owner = Objects.requireNonNull(owner, "owner must be non-null"); + this.cluster = Objects.requireNonNull(cluster, "cluster must be non-null"); this.zone = Objects.requireNonNull(zone, "zone must be non-null"); - this.alias = Objects.requireNonNull(alias, "alias must be non-null"); this.canonicalName = Objects.requireNonNull(canonicalName, "canonicalName must be non-null"); this.dnsZone = Objects.requireNonNull(dnsZone, "dnsZone must be non-null"); this.rotations = ImmutableSortedSet.copyOf(Objects.requireNonNull(rotations, "rotations must be non-null")); } - public RoutingPolicy(ApplicationId owner, ZoneId zone, ClusterSpec.Id cluster, SystemName system, HostName canonicalName, - Optional<String> dnsZone, Set<RotationName> rotations) { - this(owner, zone, HostName.from(endpointOf(cluster, owner, zone, system).dnsName()), canonicalName, dnsZone, rotations); - } - /** The application owning this */ public ApplicationId owner() { return owner; @@ -55,9 +50,9 @@ public class RoutingPolicy { return zone; } - /** This alias (lhs of a CNAME or ALIAS record) */ - public HostName alias() { - return alias; + /** The cluster this applies to */ + public ClusterSpec.Id cluster() { + return cluster; } /** The canonical name for this (rhs of a CNAME or ALIAS record) */ @@ -75,8 +70,13 @@ public class RoutingPolicy { return rotations; } - /** Endpoints for this routing policy */ - public EndpointList endpointsIn(SystemName system) { + /** Returns the endpoint of this */ + public Endpoint endpointIn(SystemName system) { + return Endpoint.of(owner).target(cluster, zone).on(Port.tls()).directRouting().in(system); + } + + /** Returns rotation endpoints of this */ + public EndpointList rotationEndpointsIn(SystemName system) { return EndpointList.of(rotations.stream().map(rotation -> endpointOf(owner, rotation, system))); } @@ -86,19 +86,21 @@ public class RoutingPolicy { if (o == null || getClass() != o.getClass()) return false; RoutingPolicy policy = (RoutingPolicy) o; return owner.equals(policy.owner) && + cluster.equals(policy.cluster) && zone.equals(policy.zone) && - canonicalName.equals(policy.canonicalName); + canonicalName.equals(policy.canonicalName) && + dnsZone.equals(policy.dnsZone); } @Override public int hashCode() { - return Objects.hash(owner, zone, canonicalName); + return Objects.hash(owner, cluster, zone, canonicalName, dnsZone); } @Override public String toString() { - return String.format("%s -> %s [rotations: %s%s], owned by %s, in %s", alias, canonicalName, rotations, - dnsZone.map(z -> ", DNS zone: " + z).orElse(""), owner.toShortString(), + return String.format("%s [rotations: %s%s], %s owned by %s, in %s", canonicalName, rotations, + dnsZone.map(z -> ", DNS zone: " + z).orElse(""), cluster, owner.toShortString(), zone.value()); } @@ -107,9 +109,4 @@ public class RoutingPolicy { return Endpoint.of(application).target(rotation).on(Port.tls()).directRouting().in(system); } - /** Returns the endpoint of given cluster */ - public static Endpoint endpointOf(ClusterSpec.Id cluster, ApplicationId application, ZoneId zone, SystemName system) { - return Endpoint.of(application).target(cluster, zone).on(Port.tls()).directRouting().in(system); - } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java index bfd432a0677..708133d2a07 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java @@ -1,22 +1,20 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; -import com.yahoo.config.provision.zone.ZoneId; -import java.util.Collections; import java.util.List; import java.util.Set; import java.util.stream.Collectors; /** - * This represents a system-level application in hosted Vespa. E.g. the zone-application. + * This represents a system-level application in hosted Vespa. All infrastructure nodes in a hosted Vespa zones are + * allocated to a system application. * * @author mpolden */ @@ -24,8 +22,8 @@ public enum SystemApplication { configServerHost(ApplicationId.from("hosted-vespa", "configserver-host", "default"), NodeType.confighost), proxyHost(ApplicationId.from("hosted-vespa", "proxy-host", "default"), NodeType.proxyhost), - configServer(ApplicationId.from("hosted-vespa", "zone-config-servers", "default"), NodeType.config), - zone(ApplicationId.from("hosted-vespa", "routing", "default"), ImmutableSet.of(NodeType.proxy, NodeType.host), + configServer(ApplicationId.from("hosted-vespa", "zone-config-servers", "default"), NodeType.config, configServerHost), + zone(ApplicationId.from("hosted-vespa", "routing", "default"), Set.of(NodeType.proxy, NodeType.host), configServerHost, proxyHost, configServer); private final ApplicationId id; @@ -33,7 +31,7 @@ public enum SystemApplication { private final List<SystemApplication> dependencies; SystemApplication(ApplicationId id, NodeType nodeType, SystemApplication... dependencies) { - this(id, Collections.singleton(nodeType), dependencies); + this(id, Set.of(nodeType), dependencies); } SystemApplication(ApplicationId id, Set<NodeType> nodeTypes, SystemApplication... dependencies) { @@ -41,8 +39,8 @@ public enum SystemApplication { throw new IllegalArgumentException("Node types must be non-empty"); } this.id = id; - this.nodeTypes = ImmutableSet.copyOf(nodeTypes); - this.dependencies = ImmutableList.copyOf(dependencies); + this.nodeTypes = Set.copyOf(nodeTypes); + this.dependencies = List.of(dependencies); } public ApplicationId id() { @@ -79,7 +77,7 @@ public enum SystemApplication { /** All known system applications */ public static List<SystemApplication> all() { - return ImmutableList.copyOf(values()); + return List.of(values()); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java index 417a1944ad3..03d894f9a17 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java @@ -137,11 +137,10 @@ public class RoutingPolicyMaintainer extends Maintainer { /** Register DNS alias for given load balancer */ private RoutingPolicy registerCname(ApplicationId application, ZoneId zone, LoadBalancer loadBalancer) { - RoutingPolicy routingPolicy = new RoutingPolicy(application, zone, - loadBalancer.cluster(), controller().system(), + RoutingPolicy routingPolicy = new RoutingPolicy(application, loadBalancer.cluster(), zone, loadBalancer.hostname(), loadBalancer.dnsZone(), loadBalancer.rotations()); - RecordName name = RecordName.from(routingPolicy.alias().value()); + RecordName name = RecordName.from(routingPolicy.endpointIn(controller().system()).dnsName()); RecordData data = RecordData.fqdn(loadBalancer.hostname().value()); List<Record> existingRecords = nameService.findRecords(Record.Type.CNAME, name); if (existingRecords.size() > 1) { @@ -170,11 +169,12 @@ public class RoutingPolicyMaintainer extends Maintainer { // Remove any active load balancers removalCandidates.removeIf(policy -> activeLoadBalancers.contains(policy.canonicalName())); for (RoutingPolicy policy : removalCandidates) { + String dnsName = policy.endpointIn(controller().system()).dnsName(); try { - List<Record> records = nameService.findRecords(Record.Type.CNAME, RecordName.from(policy.alias().value())); + List<Record> records = nameService.findRecords(Record.Type.CNAME, RecordName.from(dnsName)); nameService.removeRecords(records); } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to remove record '" + policy.alias() + + log.log(LogLevel.WARNING, "Failed to remove record '" + dnsName + "'. Retrying in " + maintenanceInterval()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index 25c9c7f6607..142907ea6c3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -1,15 +1,15 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import java.time.Duration; +import java.util.EnumSet; import java.util.Optional; import java.util.Set; import java.util.logging.Logger; @@ -23,7 +23,7 @@ public class SystemUpgrader extends InfrastructureUpgrader { private static final Logger log = Logger.getLogger(SystemUpgrader.class.getName()); - private static final Set<Node.State> upgradableNodeStates = ImmutableSet.of(Node.State.active, Node.State.reserved); + private static final Set<Node.State> upgradableNodeStates = EnumSet.of(Node.State.active, Node.State.reserved); public SystemUpgrader(Controller controller, Duration interval, JobControl jobControl) { super(controller, interval, jobControl, controller.zoneRegistry().upgradePolicy(), null); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java index 722cde68c65..7c4f9a66fd3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RotationName; import com.yahoo.slime.ArrayTraverser; @@ -21,11 +22,12 @@ import java.util.function.Function; * Serializer and deserializer for a {@link RoutingPolicy}. * * @author mortent + * @author mpolden */ public class RoutingPolicySerializer { private static final String routingPoliciesField = "routingPolicies"; - private static final String aliasField = "alias"; + private static final String clusterField = "cluster"; private static final String canonicalNameField = "canonicalName"; private static final String zoneField = "zone"; private static final String dnsZoneField = "dnsZone"; @@ -37,7 +39,7 @@ public class RoutingPolicySerializer { Cursor policyArray = root.setArray(routingPoliciesField); routingPolicies.forEach(policy -> { Cursor policyObject = policyArray.addObject(); - policyObject.setString(aliasField, policy.alias().value()); + policyObject.setString(clusterField, policy.cluster().value()); policyObject.setString(zoneField, policy.zone().value()); policyObject.setString(canonicalNameField, policy.canonicalName().value()); policy.dnsZone().ifPresent(dnsZone -> policyObject.setString(dnsZoneField, dnsZone)); @@ -57,8 +59,8 @@ public class RoutingPolicySerializer { Set<RotationName> rotations = new LinkedHashSet<>(); inspect.field(rotationsField).traverse((ArrayTraverser) (j, rotation) -> rotations.add(RotationName.from(rotation.asString()))); policies.add(new RoutingPolicy(owner, + clusterId(inspect.field(clusterField)), ZoneId.from(inspect.field(zoneField).asString()), - HostName.from(inspect.field(aliasField).asString()), HostName.from(inspect.field(canonicalNameField).asString()), optionalField(inspect.field(dnsZoneField), Function.identity()), rotations)); @@ -66,6 +68,11 @@ public class RoutingPolicySerializer { return Collections.unmodifiableSet(policies); } + // TODO: Remove and inline after Vespa 7.43 + private static ClusterSpec.Id clusterId(Inspector field) { + return optionalField(field, ClusterSpec.Id::from).orElseGet(() -> new ClusterSpec.Id("default")); + } + private static <T> Optional<T> optionalField(Inspector field, Function<String, T> fieldMapper) { return Optional.of(field).filter(Inspector::valid).map(Inspector::asString).map(fieldMapper); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 74206d05009..f1e15e9367e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -508,7 +508,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // Per-cluster rotations Set<RoutingPolicy> routingPolicies = controller.applications().routingPolicies(application.id()); for (RoutingPolicy policy : routingPolicies) { - policy.endpointsIn(controller.system()).asList().stream() + policy.rotationEndpointsIn(controller.system()).asList().stream() .map(Endpoint::url) .map(URI::toString) .forEach(globalRotationsArray::addString); @@ -584,7 +584,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private void toSlime(Cursor response, DeploymentId deploymentId, Deployment deployment, HttpRequest request) { - response.setString("tenant", deploymentId.applicationId().tenant().value()); response.setString("application", deploymentId.applicationId().application().value()); response.setString("instance", deploymentId.applicationId().instance().value()); // pointless @@ -598,6 +597,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler { controller.applications().getDeploymentEndpoints(deploymentId) .ifPresent(endpoints -> endpoints.forEach(endpoint -> serviceUrlArray.addString(endpoint.toString()))); + // Add endpoint(s) for routing policies + for (RoutingPolicy policy : controller.applications().routingPolicies(deploymentId.applicationId())) { + serviceUrlArray.addString(policy.endpointIn(controller.system()).url().toString()); + } + response.setString("nodes", withPath("/zone/v2/" + deploymentId.zoneId().environment() + "/" + deploymentId.zoneId().region() + "/nodes/v2/node/?&recursive=true&application=" + deploymentId.applicationId().tenant() + "." + deploymentId.applicationId().application() + "." + deploymentId.applicationId().instance(), request.getUri()).toString()); response.setString("yamasUrl", monitoringSystemUri(deploymentId).toString()); response.setString("version", deployment.version().toFullString()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 56d8694400c..962289274c7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -102,11 +102,17 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer public void addNodes(List<ZoneId> zones, List<SystemApplication> applications, Optional<NodeType> type) { for (ZoneId zone : zones) { for (SystemApplication application : applications) { + NodeType nodeType = type.orElseGet(() -> { + // Zone application has two node types. Use proxy + if (application == SystemApplication.zone) return NodeType.proxy; + if (application.nodeTypes().size() != 1) throw new IllegalArgumentException(application + " has several node types. Unable to detect type automatically"); + return application.nodeTypes().iterator().next(); + }); List<Node> nodes = IntStream.rangeClosed(1, 3) .mapToObj(i -> new Node( HostName.from("node-" + i + "-" + application.id().application() .value()), - Node.State.active, type.orElseGet(() -> application.nodeTypes().iterator().next()), + Node.State.active, nodeType, Optional.of(application.id()), initialVersion, initialVersion diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java index 0541a0b05f5..b0f64eee532 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java @@ -72,7 +72,7 @@ public class RoutingPolicyMaintainerTest { assertEquals("lb-0--tenant1:app1:default--prod.us-central-1.", records2.get().get(0).data().asString()); assertEquals("lb-0--tenant1:app1:default--prod.us-west-1.", records2.get().get(1).data().asString()); assertEquals(2, tester.controller().applications().routingPolicies(app1.id()).iterator().next() - .endpointsIn(SystemName.main).asList().size()); + .rotationEndpointsIn(SystemName.main).asList().size()); // Applications gains a new deployment ApplicationPackage updatedApplicationPackage = new ApplicationPackageBuilder() diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index ccc175402cd..001e7b736bb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -2,9 +2,9 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; @@ -195,10 +195,11 @@ public class SystemUpgraderTest { // System upgrades in zone 1: systemUpgrader.maintain(); - List<SystemApplication> allExceptZone = List.of(SystemApplication.configServerHost, - SystemApplication.proxyHost, - SystemApplication.configServer); - completeUpgrade(allExceptZone, version2, zone1); + List<SystemApplication> allExceptZoneAndConfig = List.of(SystemApplication.configServerHost, + SystemApplication.proxyHost); + completeUpgrade(allExceptZoneAndConfig, version2, zone1); + systemUpgrader.maintain(); + completeUpgrade(SystemApplication.configServer, version2, zone1); systemUpgrader.maintain(); completeUpgrade(SystemApplication.zone, version2, zone1); convergeServices(SystemApplication.zone, zone1); @@ -206,7 +207,9 @@ public class SystemUpgraderTest { // zone 2 and 3: systemUpgrader.maintain(); - completeUpgrade(allExceptZone, version2, zone2, zone3); + completeUpgrade(allExceptZoneAndConfig, version2, zone2, zone3); + systemUpgrader.maintain(); + completeUpgrade(SystemApplication.configServer, version2, zone2, zone3); systemUpgrader.maintain(); completeUpgrade(SystemApplication.zone, version2, zone2, zone3); convergeServices(SystemApplication.zone, zone2, zone3); @@ -214,7 +217,9 @@ public class SystemUpgraderTest { // zone 4: systemUpgrader.maintain(); - completeUpgrade(allExceptZone, version2, zone4); + completeUpgrade(allExceptZoneAndConfig, version2, zone4); + systemUpgrader.maintain(); + completeUpgrade(SystemApplication.configServer, version2, zone4); systemUpgrader.maintain(); completeUpgrade(SystemApplication.zone, version2, zone4); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java index 4fe465ce01e..4a4fd39ccb7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java @@ -3,9 +3,11 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RotationName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import org.junit.Test; @@ -19,20 +21,21 @@ import static org.junit.Assert.assertEquals; */ public class RoutingPolicySerializerTest { + private final RoutingPolicySerializer serializer = new RoutingPolicySerializer(); + @Test public void test_serialization() { - RoutingPolicySerializer serializer = new RoutingPolicySerializer(); ApplicationId owner = ApplicationId.defaultId(); Set<RotationName> rotations = Set.of(RotationName.from("r1"), RotationName.from("r2")); Set<RoutingPolicy> loadBalancers = ImmutableSet.of(new RoutingPolicy(owner, + ClusterSpec.Id.from("my-cluster1"), ZoneId.from("prod", "us-north-1"), - HostName.from("my-pretty-alias"), HostName.from("long-and-ugly-name"), Optional.of("zone1"), rotations), new RoutingPolicy(owner, + ClusterSpec.Id.from("my-cluster2"), ZoneId.from("prod", "us-north-2"), - HostName.from("my-pretty-alias-2"), HostName.from("long-and-ugly-name-2"), Optional.empty(), rotations)); @@ -40,4 +43,30 @@ public class RoutingPolicySerializerTest { assertEquals(loadBalancers, serialized); } + @Test + public void test_legacy_serialization() { // TODO: Remove after 7.43 has been released + String json = "{\n" + + " \"routingPolicies\": [\n" + + " {\n" + + " \"alias\": \"my-pretty-alias\",\n" + + " \"zone\": \"prod.us-north-1\",\n" + + " \"canonicalName\": \"long-and-ugly-name\",\n" + + " \"dnsZone\": \"zone1\",\n" + + " \"rotations\": [\n" + + " \"r1\",\n" + + " \"r2\"\n" + + " ]\n" + + " }\n" + + " ]\n" + + "}"; + ApplicationId owner = ApplicationId.defaultId(); + Set<RoutingPolicy> expected = Set.of(new RoutingPolicy(owner, + ClusterSpec.Id.from("default"), + ZoneId.from("prod", "us-north-1"), + HostName.from("long-and-ugly-name"), + Optional.of("zone1"), + Set.of(RotationName.from("r1"), RotationName.from("r2")))); + assertEquals(expected, serializer.fromSlime(owner, SlimeUtils.jsonToSlime(json))); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java index d9cdea2ea7b..c851cb18e8c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -4,7 +4,9 @@ package com.yahoo.vespa.hosted.controller.restapi; import com.yahoo.application.container.JDisc; import com.yahoo.application.container.handler.Request; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzUser; @@ -15,23 +17,23 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.identifiers.ScrewdriverId; -import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; -import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; import com.yahoo.vespa.hosted.controller.deployment.BuildJob; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps; import com.yahoo.vespa.hosted.controller.integration.ArtifactRepositoryMock; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; -import com.yahoo.vespa.hosted.controller.security.AthenzCredentials; -import com.yahoo.vespa.hosted.controller.security.AthenzTenantSpec; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import com.yahoo.vespa.hosted.controller.security.AthenzCredentials; +import com.yahoo.vespa.hosted.controller.security.AthenzTenantSpec; import java.io.File; import java.time.Duration; @@ -94,6 +96,28 @@ public class ContainerControllerTester { return application; } + public void deployCompletely(Application application, ApplicationPackage applicationPackage, long projectId, + boolean failStaging) { + jobCompletion(JobType.component).application(application) + .projectId(projectId) + .uploadArtifact(applicationPackage) + .submit(); + DeploymentSteps steps = controller().applications().deploymentTrigger().steps(applicationPackage.deploymentSpec()); + boolean succeeding = true; + for (var job : steps.jobs()) { + if (!succeeding) return; + var zone = job.zone(controller().system()); + deploy(application, applicationPackage, zone); + if (failStaging && zone.environment() == Environment.staging) { + succeeding = false; + } + if (zone.environment().isTest()) { + controller().applications().deactivate(application.id(), zone); + } + jobCompletion(job).application(application).success(succeeding).projectId(projectId).submit(); + } + } + /** Notify the controller about a job completing */ public BuildJob jobCompletion(JobType job) { return new BuildJob(this::notifyJobCompletion, artifactRepository()).type(job); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java index 11ac250d4e0..ef86ffa125f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java @@ -6,11 +6,11 @@ import com.yahoo.application.container.handler.Request; import com.yahoo.application.container.handler.Response; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.Version; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.http.filter.FilterChainRepository; import com.yahoo.jdisc.http.filter.SecurityRequestFilter; import com.yahoo.jdisc.http.filter.SecurityRequestFilterChain; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index d3f0f423089..a7a28591d62 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -11,6 +11,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.RotationName; import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.athenz.api.AthenzDomain; @@ -34,7 +35,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; @@ -1344,10 +1344,16 @@ public class ApplicationApiTest extends ControllerContainerTest { } @Test - public void applicationWithPerClusterGlobalRotation() { + public void applicationWithRoutingPolicy() { Application app = controllerTester.createApplication(); - RoutingPolicy policy = new RoutingPolicy(app.id(), ZoneId.from(Environment.prod, RegionName.from("us-west-1")), - ClusterSpec.Id.from("default"), controllerTester.controller().system(), + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("us-west-1") + .build(); + controllerTester.deployCompletely(app, applicationPackage, 1, false); + RoutingPolicy policy = new RoutingPolicy(app.id(), + ClusterSpec.Id.from("default"), + ZoneId.from(Environment.prod, RegionName.from("us-west-1")), HostName.from("lb-0-canonical-name"), Optional.of("dns-zone-1"), Set.of(RotationName.from("c0"))); tester.controller().curator().writeRoutingPolicies(app.id(), Set.of(policy)); @@ -1355,7 +1361,12 @@ public class ApplicationApiTest extends ControllerContainerTest { // GET application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", GET) .userIdentity(USER_ID), - new File("application-cluster-global-rotation.json")); + new File("application-with-routing-policy.json")); + + // GET deployment + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default", GET) + .userIdentity(USER_ID), + new File("deployment-with-routing-policy.json")); } private void notifyCompletion(DeploymentJobs.JobReport report, ContainerControllerTester tester) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-cluster-global-rotation.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-cluster-global-rotation.json deleted file mode 100644 index baaf0cd038d..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-cluster-global-rotation.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "tenant": "tenant1", - "application": "application1", - "instance": "default", - "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/", - "deployedInternally": false, - "deploymentJobs": [], - "changeBlockers": [], - "compileVersion": "(ignore)", - "globalRotations": [ - "https://c0.application1.tenant1.global.vespa.oath.cloud/" - ], - "instances": [], - "metrics": { - "queryServiceQuality": 0.0, - "writeServiceQuality": 0.0 - }, - "activity": {} -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-with-routing-policy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-with-routing-policy.json new file mode 100644 index 00000000000..33b2c7bd1df --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-with-routing-policy.json @@ -0,0 +1,203 @@ +{ + "tenant": "tenant1", + "application": "application1", + "instance": "default", + "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + }, + "projectId": 1, + "deployedInternally": false, + "deploymentJobs": [ + { + "type": "component", + "success": true, + "lastCompleted": { + "id": 42, + "version": "(ignore)", + "revision": { + "hash": "1.0.42-commit1", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "Application commit", + "at": "(ignore)" + }, + "lastSuccess": { + "id": 42, + "version": "(ignore)", + "revision": { + "hash": "1.0.42-commit1", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "Application commit", + "at": "(ignore)" + } + }, + { + "type": "system-test", + "success": true, + "lastTriggered": { + "id": -1, + "version": "(ignore)", + "revision": { + "hash": "1.0.42-commit1", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "Testing last changes outside prod", + "at": "(ignore)" + }, + "lastCompleted": { + "id": 42, + "version": "(ignore)", + "revision": { + "hash": "1.0.42-commit1", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "Testing last changes outside prod", + "at": "(ignore)" + }, + "lastSuccess": { + "id": 42, + "version": "(ignore)", + "revision": { + "hash": "1.0.42-commit1", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "Testing last changes outside prod", + "at": "(ignore)" + } + }, + { + "type": "staging-test", + "success": true, + "lastTriggered": { + "id": -1, + "version": "(ignore)", + "revision": { + "hash": "1.0.42-commit1", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "Testing deployment for production-us-west-1 (platform 7, application 1.0.42-commit1)", + "at": "(ignore)" + }, + "lastCompleted": { + "id": 42, + "version": "(ignore)", + "revision": { + "hash": "1.0.42-commit1", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "Testing deployment for production-us-west-1 (platform 7, application 1.0.42-commit1)", + "at": "(ignore)" + }, + "lastSuccess": { + "id": 42, + "version": "(ignore)", + "revision": { + "hash": "1.0.42-commit1", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "Testing deployment for production-us-west-1 (platform 7, application 1.0.42-commit1)", + "at": "(ignore)" + } + }, + { + "type": "production-us-west-1", + "success": true, + "lastTriggered": { + "id": -1, + "version": "(ignore)", + "revision": { + "hash": "1.0.42-commit1", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "New change available", + "at": "(ignore)" + }, + "lastCompleted": { + "id": 42, + "version": "(ignore)", + "revision": { + "hash": "1.0.42-commit1", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "New change available", + "at": "(ignore)" + }, + "lastSuccess": { + "id": 42, + "version": "(ignore)", + "revision": { + "hash": "1.0.42-commit1", + "source": { + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1" + } + }, + "reason": "New change available", + "at": "(ignore)" + } + } + ], + "changeBlockers": [], + "compileVersion": "(ignore)", + "globalRotations": [ + "https://c0.application1.tenant1.global.vespa.oath.cloud/" + ], + "instances": [ + { + "environment": "prod", + "region": "us-west-1", + "instance": "default", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default" + } + ], + "metrics": { + "queryServiceQuality": 0.0, + "writeServiceQuality": 0.0 + }, + "activity": {} +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json new file mode 100644 index 00000000000..b1a4c24d6ed --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy.json @@ -0,0 +1,38 @@ +{ + "tenant": "tenant1", + "application": "application1", + "instance": "default", + "environment": "prod", + "region": "us-west-1", + "serviceUrls": [ + "http://old-endpoint.vespa.yahooapis.com:4080", + "http://qrs-endpoint.vespa.yahooapis.com:4080", + "http://feeding-endpoint.vespa.yahooapis.com:4080", + "http://global-endpoint.vespa.yahooapis.com:4080", + "http://alias-endpoint.vespa.yahooapis.com:4080", + "https://application1.tenant1.us-west-1.vespa.oath.cloud/" + ], + "nodes": "http://localhost:8080/zone/v2/prod/us-west-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.default", + "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-west-1&application=tenant1.application1", + "version": "(ignore)", + "revision": "1.0.42-commit1", + "deployTimeEpochMs": "(ignore)", + "screwdriverId": "1", + "gitRepository": "repository1", + "gitBranch": "master", + "gitCommit": "commit1", + "activity": {}, + "cost": { + "tco": 0, + "waste": 0, + "utilization": 0.0, + "cluster": {} + }, + "metrics": { + "queriesPerSecond": 0.0, + "writesPerSecond": 0.0, + "documentCount": 0.0, + "queryLatencyMillis": 0.0, + "writeLatencyMillis": 0.0 + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index 575427c9222..73977d7c2fa 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -5,18 +5,15 @@ import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import org.junit.Before; import org.junit.Test; import java.io.File; @@ -31,13 +28,6 @@ public class DeploymentApiTest extends ControllerContainerTest { private final static String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/"; - private ContainerControllerTester tester; - - @Before - public void before() { - tester = new ContainerControllerTester(container, responseFiles); - } - @Test public void testDeploymentApi() { ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles); @@ -55,11 +45,11 @@ public class DeploymentApiTest extends ControllerContainerTest { "application2"); Application applicationWithoutDeployment = tester.createApplication("domain3", "tenant3", "application3"); - deployCompletely(failingApplication, applicationPackage, 1L, true); - deployCompletely(productionApplication, applicationPackage, 2L, true); + tester.deployCompletely(failingApplication, applicationPackage, 1L, false); + tester.deployCompletely(productionApplication, applicationPackage, 2L, false); // Deploy once so that job information is stored, then remove the deployment - deployCompletely(applicationWithoutDeployment, applicationPackage, 3L, true); + tester.deployCompletely(applicationWithoutDeployment, applicationPackage, 3L, false); tester.controller().applications().deactivate(applicationWithoutDeployment.id(), ZoneId.from("prod", "us-west-1")); // New version released @@ -70,8 +60,8 @@ public class DeploymentApiTest extends ControllerContainerTest { tester.upgrader().maintain(); tester.controller().applications().deploymentTrigger().triggerReadyJobs(); tester.controller().applications().deploymentTrigger().triggerReadyJobs(); - deployCompletely(failingApplication, applicationPackage, 1L, false); - deployCompletely(productionApplication, applicationPackage, 2L, true); + tester.deployCompletely(failingApplication, applicationPackage, 1L, true); + tester.deployCompletely(productionApplication, applicationPackage, 2L, false); tester.controller().updateVersionStatus(censorConfigServers(VersionStatus.compute(tester.controller()), tester.controller())); @@ -98,34 +88,4 @@ public class DeploymentApiTest extends ControllerContainerTest { return new VersionStatus(censored); } - private void deployCompletely(Application application, ApplicationPackage applicationPackage, long projectId, - boolean success) { - tester.jobCompletion(JobType.component) - .application(application) - .projectId(projectId) - .uploadArtifact(applicationPackage) - .submit(); - tester.deploy(application, applicationPackage, ZoneId.from(Environment.test, RegionName.from("us-east-1")) - ); - tester.jobCompletion(JobType.systemTest) - .application(application) - .projectId(projectId) - .submit(); - tester.deploy(application, applicationPackage, ZoneId.from(Environment.staging, RegionName.from("us-east-3")) - ); - tester.jobCompletion(JobType.stagingTest) - .application(application) - .projectId(projectId) - .success(success) - .submit(); - if (success) { - tester.deploy(application, applicationPackage, ZoneId.from(Environment.prod, - RegionName.from("us-west-1"))); - tester.jobCompletion(JobType.productionUsWest1) - .application(application) - .projectId(projectId) - .submit(); - } - } - } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java index bc8dd8d4479..e4b0f526519 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java @@ -1,16 +1,14 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.os; - -import com.google.common.collect.ImmutableList; import com.yahoo.application.container.handler.Request; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.UpgradePolicy; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.config.provision.zone.UpgradePolicy; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; @@ -55,7 +53,7 @@ public class OsApiTest extends ControllerContainerTest { .setZones(zone1, zone2, zone3) .setOsUpgradePolicy(cloud1, UpgradePolicy.create().upgrade(zone1).upgrade(zone2)) .setOsUpgradePolicy(cloud2, UpgradePolicy.create().upgrade(zone3)); - osUpgraders = ImmutableList.of( + osUpgraders = List.of( new OsUpgrader(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), cloud1), diff --git a/docker-api/pom.xml b/docker-api/pom.xml index 4c1cf41bb12..1cb03f1819d 100644 --- a/docker-api/pom.xml +++ b/docker-api/pom.xml @@ -36,7 +36,7 @@ <dependency> <groupId>com.github.docker-java</groupId> <artifactId>docker-java</artifactId> - <version>3.0.13</version> + <version>3.1.2</version> <scope>compile</scope> <exclusions> <exclusion> diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerStats.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerStats.java index 738a65bc08b..d33ddadb52c 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerStats.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerStats.java @@ -1,9 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.dockerapi; -import java.util.Collections; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.dockerjava.api.model.CpuStatsConfig; +import com.github.dockerjava.api.model.MemoryStatsConfig; +import com.github.dockerjava.api.model.StatisticNetworksConfig; +import com.github.dockerjava.api.model.Statistics; + +import java.io.IOException; +import java.io.UncheckedIOException; import java.util.Map; import java.util.Optional; +import java.util.TreeMap; +import java.util.stream.Collectors; /** * Wrapper class for {@link com.github.dockerjava.api.model.Statistics} to prevent leaking from docker-java library. @@ -11,33 +20,103 @@ import java.util.Optional; * @author freva */ public class ContainerStats { - private final Map<String, Object> networks; - private final Map<String, Object> cpuStats; - private final Map<String, Object> memoryStats; - private final Map<String, Object> blkioStats; + private final Map<String, NetworkStats> networkStatsByInterface; + private final MemoryStats memoryStats; + private final CpuStats cpuStats; - public ContainerStats(Map<String, Object> networks, Map<String, Object> cpuStats, - Map<String, Object> memoryStats, Map<String, Object> blkioStats) { + ContainerStats(Statistics statistics) { // Network stats are null when container uses host network - this.networks = Optional.ofNullable(networks).orElse(Collections.emptyMap()); - this.cpuStats = cpuStats; - this.memoryStats = memoryStats; - this.blkioStats = blkioStats; + this.networkStatsByInterface = Optional.ofNullable(statistics.getNetworks()).orElseGet(Map::of) + .entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> new NetworkStats(e.getValue()), + (u, v) -> { throw new IllegalStateException(); }, + TreeMap::new)); + this.memoryStats = new MemoryStats(statistics.getMemoryStats()); + this.cpuStats = new CpuStats(statistics.getCpuStats()); + } + + public Map<String, NetworkStats> getNetworks() { + return networkStatsByInterface; } - public Map<String, Object> getNetworks() { - return networks; + public MemoryStats getMemoryStats() { + return memoryStats; } - public Map<String, Object> getCpuStats() { + public CpuStats getCpuStats() { return cpuStats; } - public Map<String, Object> getMemoryStats() { - return memoryStats; + public static class NetworkStats { + private final long rxBytes; + private final long rxDropped; + private final long rxErrors; + private final long txBytes; + private final long txDropped; + private final long txErrors; + + private NetworkStats(StatisticNetworksConfig statisticNetworksConfig) { + this.rxBytes = statisticNetworksConfig.getRxBytes(); + this.rxDropped = statisticNetworksConfig.getRxDropped(); + this.rxErrors = statisticNetworksConfig.getRxErrors(); + this.txBytes = statisticNetworksConfig.getTxBytes(); + this.txDropped = statisticNetworksConfig.getTxDropped(); + this.txErrors = statisticNetworksConfig.getTxErrors(); + } + + public long getRxBytes() { return this.rxBytes; } + public long getRxDropped() { return this.rxDropped; } + public long getRxErrors() { return this.rxErrors; } + public long getTxBytes() { return this.txBytes; } + public long getTxDropped() { return this.txDropped; } + public long getTxErrors() { return this.txErrors; } + } + + public class MemoryStats { + private final long cache; + private final long usage; + private final long limit; + + private MemoryStats(MemoryStatsConfig memoryStats) { + this.cache = memoryStats.getStats().getCache(); + this.usage = memoryStats.getUsage(); + this.limit = memoryStats.getLimit(); + } + + public long getCache() { return this.cache; } + public long getUsage() { return this.usage; } + public long getLimit() { return this.limit; } + } + + public class CpuStats { + private final int onlineCpus; + private final long systemCpuUsage; + private final long totalUsage; + private final long usageInKernelMode; + + public CpuStats(CpuStatsConfig cpuStats) { + // Added in 1.27 + this.onlineCpus = cpuStats.getCpuUsage().getPercpuUsage().size(); + this.systemCpuUsage = cpuStats.getSystemCpuUsage(); + this.totalUsage = cpuStats.getCpuUsage().getTotalUsage(); + this.usageInKernelMode = cpuStats.getCpuUsage().getUsageInKernelmode(); + } + + public int getOnlineCpus() { return this.onlineCpus; } + public long getSystemCpuUsage() { return this.systemCpuUsage; } + public long getTotalUsage() { return totalUsage; } + public long getUsageInKernelMode() { return usageInKernelMode; } } - public Map<String, Object> getBlkioStats() { - return blkioStats; + // For testing only, create ContainerStats from JSON returned by docker daemon stats API + public static ContainerStats fromJson(String json) { + try { + Statistics statistics = new ObjectMapper().readValue(json, Statistics.class); + return new ContainerStats(statistics); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } } diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java index db4dc303ab9..2e5cfab36cc 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java @@ -167,34 +167,34 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { List<Bind> volumeBinds = volumeBindSpecs.stream().map(Bind::parse).collect(Collectors.toList()); final HostConfig hostConfig = new HostConfig() - .withSecurityOpts(new ArrayList<>(securityOpts)); + .withSecurityOpts(new ArrayList<>(securityOpts)) + .withBinds(volumeBinds) + .withUlimits(ulimits) + .withCapAdd(addCapabilities.toArray(new Capability[0])) + .withCapDrop(dropCapabilities.toArray(new Capability[0])) + .withPrivileged(privileged); containerResources.ifPresent(cr -> hostConfig .withCpuShares(cr.cpuShares()) .withMemory(cr.memoryBytes()) // MemorySwap is the total amount of memory and swap, if MemorySwap == Memory, then container has no access swap .withMemorySwap(cr.memoryBytes()) - .withCpuPeriod(cr.cpuQuota() > 0 ? cr.cpuPeriod() : null) - .withCpuQuota(cr.cpuQuota() > 0 ? cr.cpuQuota() : null)); + .withCpuPeriod(cr.cpuQuota() > 0 ? (long) cr.cpuPeriod() : null) + .withCpuQuota(cr.cpuQuota() > 0 ? (long) cr.cpuQuota() : null)); final CreateContainerCmd containerCmd = docker .createContainerCmd(dockerImage.asString()) - .withHostConfig(hostConfig) // MUST BE FIRST (some of the later setters are simply proxied to HostConfig) + .withHostConfig(hostConfig) .withName(containerName.asString()) .withLabels(labels) - .withEnv(environmentAssignments) - .withBinds(volumeBinds) - .withUlimits(ulimits) - .withCapAdd(new ArrayList<>(addCapabilities)) - .withCapDrop(new ArrayList<>(dropCapabilities)) - .withPrivileged(privileged); + .withEnv(environmentAssignments); networkMode .filter(mode -> ! mode.toLowerCase().equals("host")) .ifPresent(mode -> containerCmd.withMacAddress(generateMACAddress(hostName, ipv4Address, ipv6Address))); hostName.ifPresent(containerCmd::withHostName); - networkMode.ifPresent(containerCmd::withNetworkMode); + networkMode.ifPresent(hostConfig::withNetworkMode); ipv4Address.ifPresent(containerCmd::withIpv4Address); ipv6Address.ifPresent(containerCmd::withIpv6Address); entrypoint.ifPresent(containerCmd::withEntrypoint); diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java index e57f61ce5f4..a8e08a19d3d 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java @@ -183,8 +183,7 @@ public class DockerImpl implements Docker { DockerStatsCallback statsCallback = dockerClient.statsCmd(containerName.asString()).exec(new DockerStatsCallback()); statsCallback.awaitCompletion(5, TimeUnit.SECONDS); - return statsCallback.stats.map(stats -> new ContainerStats( - stats.getNetworks(), stats.getCpuStats(), stats.getMemoryStats(), stats.getBlkioStats())); + return statsCallback.stats.map(ContainerStats::new); } catch (NotFoundException ignored) { return Optional.empty(); } catch (RuntimeException | InterruptedException e) { diff --git a/document/abi-spec.json b/document/abi-spec.json index d4db3026b27..81cf5509a57 100644 --- a/document/abi-spec.json +++ b/document/abi-spec.json @@ -4436,8 +4436,7 @@ "public static com.yahoo.document.serialization.DocumentSerializer createHead(com.yahoo.io.GrowableByteBuffer)", "public static com.yahoo.document.serialization.DocumentSerializer create6(com.yahoo.io.GrowableByteBuffer)", "public static com.yahoo.document.serialization.DocumentSerializer create6()", - "public static com.yahoo.document.serialization.DocumentSerializer create42(com.yahoo.io.GrowableByteBuffer)", - "public static com.yahoo.document.serialization.DocumentSerializer create42()" + "public static com.yahoo.document.serialization.DocumentSerializer create42(com.yahoo.io.GrowableByteBuffer)" ], "fields": [] }, diff --git a/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java index f895ad21b0a..287af5c34b9 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/FieldValue.java @@ -5,14 +5,17 @@ import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.Field; import com.yahoo.document.FieldPath; -import com.yahoo.document.serialization.*; +import com.yahoo.document.serialization.DocumentSerializer; +import com.yahoo.document.serialization.DocumentSerializerFactory; +import com.yahoo.document.serialization.FieldReader; +import com.yahoo.document.serialization.FieldWriter; +import com.yahoo.document.serialization.XmlStream; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.vespa.objects.BufferSerializer; import com.yahoo.vespa.objects.Deserializer; import com.yahoo.vespa.objects.Identifiable; import com.yahoo.vespa.objects.Ids; import com.yahoo.vespa.objects.Serializer; -import com.yahoo.document.config.DocumentmanagerConfig.Datatype.Structtype.Compresstype; /** * @author Einar M R Rosenvinge diff --git a/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java b/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java index 1b5681e7146..be4aa6cfb41 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java +++ b/document/src/main/java/com/yahoo/document/json/JsonFeedReader.java @@ -9,8 +9,11 @@ import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentRemove; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentUpdate; +import com.yahoo.vespaxmlparser.DocumentFeedOperation; +import com.yahoo.vespaxmlparser.DocumentUpdateFeedOperation; +import com.yahoo.vespaxmlparser.FeedOperation; import com.yahoo.vespaxmlparser.FeedReader; -import com.yahoo.vespaxmlparser.VespaXMLFeedReader.Operation; +import com.yahoo.vespaxmlparser.RemoveFeedOperation; /** @@ -34,26 +37,23 @@ public class JsonFeedReader implements FeedReader { } @Override - public void read(Operation operation) throws Exception { + public FeedOperation read() throws Exception { DocumentOperation documentOperation = reader.next(); if (documentOperation == null) { stream.close(); - operation.setInvalid(); - return; + return FeedOperation.INVALID; } if (documentOperation instanceof DocumentUpdate) { - operation.setDocumentUpdate((DocumentUpdate) documentOperation); + return new DocumentUpdateFeedOperation((DocumentUpdate) documentOperation, documentOperation.getCondition()); } else if (documentOperation instanceof DocumentRemove) { - operation.setRemove(documentOperation.getId()); + return new RemoveFeedOperation(documentOperation.getId(), documentOperation.getCondition()); } else if (documentOperation instanceof DocumentPut) { - operation.setDocument(((DocumentPut) documentOperation).getDocument()); + return new DocumentFeedOperation(((DocumentPut) documentOperation).getDocument(), documentOperation.getCondition()); } else { throw new IllegalStateException("Got unknown class from JSON reader: " + documentOperation.getClass().getName()); } - - operation.setCondition(documentOperation.getCondition()); } } diff --git a/document/src/main/java/com/yahoo/document/json/JsonReader.java b/document/src/main/java/com/yahoo/document/json/JsonReader.java index b7818b06b03..d512fd3a6d1 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonReader.java +++ b/document/src/main/java/com/yahoo/document/json/JsonReader.java @@ -4,7 +4,6 @@ package com.yahoo.document.json; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; -import com.google.common.annotations.Beta; import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentOperation; import com.yahoo.document.DocumentType; diff --git a/document/src/main/java/com/yahoo/document/json/JsonWriter.java b/document/src/main/java/com/yahoo/document/json/JsonWriter.java index ab0884a54a3..d7944246ff2 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonWriter.java +++ b/document/src/main/java/com/yahoo/document/json/JsonWriter.java @@ -38,7 +38,33 @@ import java.nio.ByteBuffer; import java.util.Iterator; import java.util.Map; -import static com.yahoo.document.json.JsonSerializationHelper.*; +import static com.yahoo.document.json.JsonSerializationHelper.fieldNameIfNotNull; +import static com.yahoo.document.json.JsonSerializationHelper.serializeArrayField; +import static com.yahoo.document.json.JsonSerializationHelper.serializeBoolField; +import static com.yahoo.document.json.JsonSerializationHelper.serializeByte; +import static com.yahoo.document.json.JsonSerializationHelper.serializeByteArray; +import static com.yahoo.document.json.JsonSerializationHelper.serializeByteBuffer; +import static com.yahoo.document.json.JsonSerializationHelper.serializeByteField; +import static com.yahoo.document.json.JsonSerializationHelper.serializeCollectionField; +import static com.yahoo.document.json.JsonSerializationHelper.serializeDouble; +import static com.yahoo.document.json.JsonSerializationHelper.serializeDoubleField; +import static com.yahoo.document.json.JsonSerializationHelper.serializeFloat; +import static com.yahoo.document.json.JsonSerializationHelper.serializeFloatField; +import static com.yahoo.document.json.JsonSerializationHelper.serializeInt; +import static com.yahoo.document.json.JsonSerializationHelper.serializeIntField; +import static com.yahoo.document.json.JsonSerializationHelper.serializeLong; +import static com.yahoo.document.json.JsonSerializationHelper.serializeLongField; +import static com.yahoo.document.json.JsonSerializationHelper.serializeMapField; +import static com.yahoo.document.json.JsonSerializationHelper.serializePredicateField; +import static com.yahoo.document.json.JsonSerializationHelper.serializeRawField; +import static com.yahoo.document.json.JsonSerializationHelper.serializeReferenceField; +import static com.yahoo.document.json.JsonSerializationHelper.serializeShort; +import static com.yahoo.document.json.JsonSerializationHelper.serializeString; +import static com.yahoo.document.json.JsonSerializationHelper.serializeStringField; +import static com.yahoo.document.json.JsonSerializationHelper.serializeStructField; +import static com.yahoo.document.json.JsonSerializationHelper.serializeStructuredField; +import static com.yahoo.document.json.JsonSerializationHelper.serializeTensorField; +import static com.yahoo.document.json.JsonSerializationHelper.serializeWeightedSet; import static com.yahoo.document.json.document.DocumentParser.FIELDS; import static com.yahoo.document.json.document.DocumentParser.REMOVE; diff --git a/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java b/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java index 8012ebafb13..28aa9ed1d8d 100644 --- a/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java +++ b/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java @@ -7,7 +7,9 @@ import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentUpdate; import com.yahoo.document.json.document.DocumentParser; -import com.yahoo.vespaxmlparser.VespaXMLFeedReader; +import com.yahoo.vespaxmlparser.DocumentFeedOperation; +import com.yahoo.vespaxmlparser.DocumentUpdateFeedOperation; +import com.yahoo.vespaxmlparser.FeedOperation; import java.io.IOException; import java.io.InputStream; @@ -25,32 +27,26 @@ public class SingleDocumentParser { this.docMan = docMan; } - public VespaXMLFeedReader.Operation parsePut(InputStream inputStream, String docId) { + public FeedOperation parsePut(InputStream inputStream, String docId) { return parse(inputStream, docId, DocumentParser.SupportedOperation.PUT); } - public VespaXMLFeedReader.Operation parseUpdate(InputStream inputStream, String docId) { + public FeedOperation parseUpdate(InputStream inputStream, String docId) { return parse(inputStream, docId, DocumentParser.SupportedOperation.UPDATE); } - private VespaXMLFeedReader.Operation parse(InputStream inputStream, String docId, DocumentParser.SupportedOperation supportedOperation) { + private FeedOperation parse(InputStream inputStream, String docId, DocumentParser.SupportedOperation supportedOperation) { final JsonReader reader = new JsonReader(docMan, inputStream, jsonFactory); final DocumentOperation documentOperation = reader.readSingleDocument(supportedOperation, docId); - VespaXMLFeedReader.Operation operation = new VespaXMLFeedReader.Operation(); try { inputStream.close(); } catch (IOException e) { throw new RuntimeException(e); } if (supportedOperation == DocumentParser.SupportedOperation.PUT) { - operation.setDocument(((DocumentPut) documentOperation).getDocument()); + return new DocumentFeedOperation(((DocumentPut) documentOperation).getDocument(), documentOperation.getCondition()); } else { - operation.setDocumentUpdate((DocumentUpdate) documentOperation); + return new DocumentUpdateFeedOperation((DocumentUpdate) documentOperation, documentOperation.getCondition()); } - - // (A potentially empty) test-and-set condition is always set by JsonReader - operation.setCondition(documentOperation.getCondition()); - - return operation; } } diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentDeserializer.java b/document/src/main/java/com/yahoo/document/serialization/DocumentDeserializer.java index 582a6b0bd92..afdce012b0f 100644 --- a/document/src/main/java/com/yahoo/document/serialization/DocumentDeserializer.java +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentDeserializer.java @@ -15,7 +15,7 @@ public interface DocumentDeserializer extends DocumentReader, DocumentUpdateRead /** * Returns the underlying buffer used for de-serialization. */ - public GrowableByteBuffer getBuf(); + GrowableByteBuffer getBuf(); } diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentSerializerFactory.java b/document/src/main/java/com/yahoo/document/serialization/DocumentSerializerFactory.java index 54ec4e2fcca..23266566e1c 100644 --- a/document/src/main/java/com/yahoo/document/serialization/DocumentSerializerFactory.java +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentSerializerFactory.java @@ -42,12 +42,4 @@ public class DocumentSerializerFactory { return new VespaDocumentSerializer42(buf); } - /** - * Creates a serializer for the document format that was created on Vespa 4.2. - */ - @SuppressWarnings("deprecation") - public static DocumentSerializer create42() { - return new VespaDocumentSerializer42(); - } - } diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateWriter.java b/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateWriter.java index 4b87edeeded..d95a344be77 100644 --- a/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateWriter.java +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateWriter.java @@ -21,15 +21,15 @@ import com.yahoo.document.update.TensorRemoveUpdate; * @since 5.1.27 */ public interface DocumentUpdateWriter { - public void write(DocumentUpdate update); - public void write(FieldUpdate update); - public void write(AddValueUpdate update, DataType superType); - public void write(MapValueUpdate update, DataType superType); - public void write(ArithmeticValueUpdate update); - public void write(AssignValueUpdate update, DataType superType); - public void write(RemoveValueUpdate update, DataType superType); - public void write(ClearValueUpdate clearValueUpdate, DataType superType); - public void write(TensorModifyUpdate update); - public void write(TensorAddUpdate update); - public void write(TensorRemoveUpdate update); + void write(DocumentUpdate update); + void write(FieldUpdate update); + void write(AddValueUpdate update, DataType superType); + void write(MapValueUpdate update, DataType superType); + void write(ArithmeticValueUpdate update); + void write(AssignValueUpdate update, DataType superType); + void write(RemoveValueUpdate update, DataType superType); + void write(ClearValueUpdate clearValueUpdate, DataType superType); + void write(TensorModifyUpdate update); + void write(TensorAddUpdate update); + void write(TensorRemoveUpdate update); } diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java index a763db33e7a..58c50f047f9 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializerHead.java @@ -5,7 +5,6 @@ import com.yahoo.document.DataType; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.TensorDataType; import com.yahoo.document.datatypes.TensorFieldValue; -import com.yahoo.document.json.readers.TensorRemoveUpdateReader; import com.yahoo.document.update.TensorAddUpdate; import com.yahoo.document.update.TensorModifyUpdate; import com.yahoo.document.update.TensorRemoveUpdate; @@ -18,7 +17,6 @@ import com.yahoo.tensor.TensorType; * * @author baldersheim */ -@SuppressWarnings("deprecation") public class VespaDocumentDeserializerHead extends VespaDocumentDeserializer6 { public VespaDocumentDeserializerHead(DocumentTypeManager manager, GrowableByteBuffer buffer) { diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/ConditionalFeedOperation.java b/document/src/main/java/com/yahoo/vespaxmlparser/ConditionalFeedOperation.java new file mode 100644 index 00000000000..e7a06560532 --- /dev/null +++ b/document/src/main/java/com/yahoo/vespaxmlparser/ConditionalFeedOperation.java @@ -0,0 +1,21 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaxmlparser; + +import com.yahoo.document.TestAndSetCondition; + +public class ConditionalFeedOperation extends FeedOperation { + private final TestAndSetCondition condition; + protected ConditionalFeedOperation(Type type) { + super(type); + this.condition = TestAndSetCondition.NOT_PRESENT_CONDITION; + } + protected ConditionalFeedOperation(Type type, TestAndSetCondition condition) { + super(type); + this.condition = condition; + } + + @Override + public TestAndSetCondition getCondition() { + return condition; + } +} diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/DocumentFeedOperation.java b/document/src/main/java/com/yahoo/vespaxmlparser/DocumentFeedOperation.java new file mode 100644 index 00000000000..f3ddbc9196c --- /dev/null +++ b/document/src/main/java/com/yahoo/vespaxmlparser/DocumentFeedOperation.java @@ -0,0 +1,23 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaxmlparser; + +import com.yahoo.document.Document; +import com.yahoo.document.TestAndSetCondition; + +public class DocumentFeedOperation extends ConditionalFeedOperation { + private final Document document; + public DocumentFeedOperation(Document document) { + super(Type.DOCUMENT); + this.document = document; + } + + public DocumentFeedOperation(Document document, TestAndSetCondition condition) { + super(Type.DOCUMENT, condition); + this.document = document; + } + + @Override + public Document getDocument() { + return document; + } +} diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/DocumentUpdateFeedOperation.java b/document/src/main/java/com/yahoo/vespaxmlparser/DocumentUpdateFeedOperation.java new file mode 100644 index 00000000000..af20d72a4e2 --- /dev/null +++ b/document/src/main/java/com/yahoo/vespaxmlparser/DocumentUpdateFeedOperation.java @@ -0,0 +1,22 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaxmlparser; + +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.TestAndSetCondition; + +public class DocumentUpdateFeedOperation extends ConditionalFeedOperation { + private final DocumentUpdate update; + public DocumentUpdateFeedOperation(DocumentUpdate update) { + super(Type.UPDATE); + this.update = update; + } + public DocumentUpdateFeedOperation(DocumentUpdate update, TestAndSetCondition condition) { + super(Type.UPDATE, condition); + this.update = update; + } + + @Override + public DocumentUpdate getDocumentUpdate() { + return update; + } +} diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/FeedOperation.java b/document/src/main/java/com/yahoo/vespaxmlparser/FeedOperation.java new file mode 100644 index 00000000000..9da3408da61 --- /dev/null +++ b/document/src/main/java/com/yahoo/vespaxmlparser/FeedOperation.java @@ -0,0 +1,39 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaxmlparser; + +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.TestAndSetCondition; + +public class FeedOperation { + public enum Type {DOCUMENT, REMOVE, UPDATE, INVALID} + public static final FeedOperation INVALID = new FeedOperation(Type.INVALID); + + private Type type; + protected FeedOperation(Type type) { + this.type = type; + } + public final Type getType() { return type; } + protected final void setType(Type type) { + this.type = type; + } + + public Document getDocument() { return null; } + public DocumentUpdate getDocumentUpdate() { return null; } + public DocumentId getRemove() { return null; } + + public TestAndSetCondition getCondition() { + return TestAndSetCondition.NOT_PRESENT_CONDITION; + } + @Override + public String toString() { + return "Operation{" + + "type=" + getType() + + ", doc=" + getDocument() + + ", remove=" + getRemove() + + ", docUpdate=" + getDocumentUpdate() + + " testandset=" + getCondition() + + '}'; + } +}
\ No newline at end of file diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/FeedReader.java b/document/src/main/java/com/yahoo/vespaxmlparser/FeedReader.java index 2c130cae782..c993d5a5153 100644 --- a/document/src/main/java/com/yahoo/vespaxmlparser/FeedReader.java +++ b/document/src/main/java/com/yahoo/vespaxmlparser/FeedReader.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespaxmlparser; -import com.yahoo.vespaxmlparser.VespaXMLFeedReader.Operation; - /** * Minimal interface for reading operations from a stream for a feeder. * @@ -14,8 +12,7 @@ public interface FeedReader { /** * Reads the next operation from the stream. - * @param operation The operation to fill in. Operation is unchanged if none was found. + * @return operation, possibly invalid if none was found. */ - void read(Operation operation) throws Exception; - + FeedOperation read() throws Exception; } diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/RemoveFeedOperation.java b/document/src/main/java/com/yahoo/vespaxmlparser/RemoveFeedOperation.java new file mode 100644 index 00000000000..782a6295ee1 --- /dev/null +++ b/document/src/main/java/com/yahoo/vespaxmlparser/RemoveFeedOperation.java @@ -0,0 +1,22 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespaxmlparser; + +import com.yahoo.document.DocumentId; +import com.yahoo.document.TestAndSetCondition; + +public class RemoveFeedOperation extends ConditionalFeedOperation { + private final DocumentId documentId; + public RemoveFeedOperation(DocumentId documentId) { + super(Type.REMOVE); + this.documentId = documentId; + } + public RemoveFeedOperation(DocumentId documentId, TestAndSetCondition condition) { + super(Type.REMOVE, condition); + this.documentId = documentId; + } + + @Override + public DocumentId getRemove() { + return documentId; + } +} diff --git a/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFeedReader.java b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFeedReader.java index a24f1abd22b..7bc0cc871ca 100644 --- a/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFeedReader.java +++ b/document/src/main/java/com/yahoo/vespaxmlparser/VespaXMLFeedReader.java @@ -56,7 +56,7 @@ public class VespaXMLFeedReader extends VespaXMLReader implements FeedReader { /** * Skips the initial "vespafeed" tag. */ - void readInitial() throws Exception { + private void readInitial() throws Exception { boolean found = false; while (reader.hasNext()) { @@ -74,127 +74,6 @@ public class VespaXMLFeedReader extends VespaXMLReader implements FeedReader { } } - public enum OperationType { - DOCUMENT, - REMOVE, - UPDATE, - INVALID - } - - /** - * Represents a feed operation found by the parser. Can be one of the following types: - * - getType() == DOCUMENT: getDocument() is valid. - * - getType() == REMOVE: getRemove() is valid. - * - getType() == UPDATE: getUpdate() is valid. - */ - public static class Operation { - - private OperationType type; - private Document doc; - private DocumentId remove; - private DocumentUpdate docUpdate; - private FeedOperation feedOperation; - private TestAndSetCondition condition; - - public Operation() { - setInvalid(); - } - - public void setInvalid() { - type = OperationType.INVALID; - doc = null; - remove = null; - docUpdate = null; - feedOperation = null; - condition = null; - } - - public OperationType getType() { - return type; - } - - public Document getDocument() { - return doc; - } - - public void setDocument(Document doc) { - this.type = OperationType.DOCUMENT; - this.doc = doc; - } - - public DocumentId getRemove() { - return remove; - } - - public void setRemove(DocumentId remove) { - this.type = OperationType.REMOVE; - this.remove = remove; - } - - public DocumentUpdate getDocumentUpdate() { - return docUpdate; - } - - public void setDocumentUpdate(DocumentUpdate docUpdate) { - this.type = OperationType.UPDATE; - this.docUpdate = docUpdate; - } - - public FeedOperation getFeedOperation() { - return feedOperation; - } - - public void setCondition(TestAndSetCondition condition) { - this.condition = condition; - } - - public TestAndSetCondition getCondition() { - return condition; - } - - @Override - public String toString() { - return "Operation{" + - "type=" + type + - ", doc=" + doc + - ", remove=" + remove + - ", docUpdate=" + docUpdate + - ", feedOperation=" + feedOperation + - '}'; - } - } - - public static class FeedOperation { - - private String name; - private Integer generation; - private Integer increment; - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public Integer getGeneration() { - return generation; - } - - public void setGeneration(int generation) { - this.generation = generation; - } - - public Integer getIncrement() { - return increment; - } - - public void setIncrement(int increment) { - this.increment = increment; - } - } - /** * <p>Reads all operations from the XML stream and puts into a list. Note * that if the XML stream is large, this may cause out of memory errors, so @@ -202,12 +81,11 @@ public class VespaXMLFeedReader extends VespaXMLReader implements FeedReader { * * @return The list of all read operations. */ - public List<Operation> readAll() throws Exception { - List<Operation> list = new ArrayList<Operation>(); + public List<FeedOperation> readAll() throws Exception { + List<FeedOperation> list = new ArrayList<>(); while (true) { - Operation op = new Operation(); - read(op); - if (op.getType() == OperationType.INVALID) { + FeedOperation op = read(); + if (op.getType() == FeedOperation.Type.INVALID) { return list; } else { list.add(op); @@ -219,10 +97,8 @@ public class VespaXMLFeedReader extends VespaXMLReader implements FeedReader { * @see com.yahoo.vespaxmlparser.FeedReader#read(com.yahoo.vespaxmlparser.VespaXMLFeedReader.Operation) */ @Override - public void read(Operation operation) throws Exception { + public FeedOperation read() throws Exception { String startTag = null; - operation.setInvalid(); - try { while (reader.hasNext()) { int type = reader.next(); @@ -233,36 +109,28 @@ public class VespaXMLFeedReader extends VespaXMLReader implements FeedReader { if ("document".equals(startTag)) { VespaXMLDocumentReader documentReader = new VespaXMLDocumentReader(reader, docTypeManager); Document document = new Document(documentReader); - operation.setDocument(document); - operation.setCondition(TestAndSetCondition.fromConditionString(documentReader.getCondition())); - return; + return new DocumentFeedOperation(document, TestAndSetCondition.fromConditionString(documentReader.getCondition())); } else if ("update".equals(startTag)) { VespaXMLUpdateReader updateReader = new VespaXMLUpdateReader(reader, docTypeManager); DocumentUpdate update = new DocumentUpdate(updateReader); - operation.setDocumentUpdate(update); - operation.setCondition(TestAndSetCondition.fromConditionString(updateReader.getCondition())); - return; + return new DocumentUpdateFeedOperation(update, TestAndSetCondition.fromConditionString(updateReader.getCondition())); } else if ("remove".equals(startTag)) { - boolean documentIdFound = false; + DocumentId documentId = null; Optional<String> condition = Optional.empty(); for (int i = 0; i < reader.getAttributeCount(); i++) { final String attributeName = reader.getAttributeName(i).toString(); if ("documentid".equals(attributeName) || "id".equals(attributeName)) { - operation.setRemove(new DocumentId(reader.getAttributeValue(i))); - documentIdFound = true; + documentId = new DocumentId(reader.getAttributeValue(i)); } else if ("condition".equals(attributeName)) { condition = Optional.of(reader.getAttributeValue(i)); } } - if (!documentIdFound) { + if (documentId == null) { throw newDeserializeException("Missing \"documentid\" attribute for remove operation"); } - - operation.setCondition(TestAndSetCondition.fromConditionString(condition)); - - return; + return new RemoveFeedOperation(documentId, TestAndSetCondition.fromConditionString(condition)); } else { throw newDeserializeException("Element \"" + startTag + "\" not allowed in this context"); } @@ -281,32 +149,7 @@ public class VespaXMLFeedReader extends VespaXMLReader implements FeedReader { throw(e); } - } - - public void read(FeedOperation fo) throws XMLStreamException { - while (reader.hasNext()) { - int type = reader.next(); - - if (type == XMLStreamReader.START_ELEMENT) { - if ("name".equals(reader.getName().toString())) { - fo.setName(reader.getElementText().toString()); - skipToEnd("name"); - } else if ("generation".equals(reader.getName().toString())) { - fo.setGeneration(Integer.parseInt(reader.getElementText().toString())); - skipToEnd("generation"); - } else if ("increment".equals(reader.getName().toString())) { - String text = reader.getElementText(); - if ("autodetect".equals(text)) { - fo.setIncrement(-1); - } else { - fo.setIncrement(Integer.parseInt(text)); - } - skipToEnd("increment"); - } - } else if (type == XMLStreamReader.END_ELEMENT) { - return; - } - } + return FeedOperation.INVALID; } } diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/PositionParserTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/PositionParserTestCase.java index a7fd782484e..e2aafcb4fdc 100644 --- a/document/src/test/java/com/yahoo/vespaxmlparser/PositionParserTestCase.java +++ b/document/src/test/java/com/yahoo/vespaxmlparser/PositionParserTestCase.java @@ -26,7 +26,7 @@ public class PositionParserTestCase { mgr.registerDocumentType(docType); VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test_position.xml", mgr); - Iterator<VespaXMLFeedReader.Operation> it = parser.readAll().iterator(); + Iterator<FeedOperation> it = parser.readAll().iterator(); assertTrue(it.hasNext()); assertDocument(PositionDataType.valueOf(1, 2), it.next()); assertTrue(it.hasNext()); @@ -38,9 +38,9 @@ public class PositionParserTestCase { assertFalse(it.hasNext()); } - private static void assertDocument(Struct expected, VespaXMLFeedReader.Operation operation) { + private static void assertDocument(Struct expected, FeedOperation operation) { assertNotNull(operation); - assertEquals(VespaXMLFeedReader.OperationType.DOCUMENT, operation.getType()); + assertEquals(FeedOperation.Type.DOCUMENT, operation.getType()); Document doc = operation.getDocument(); assertNotNull(doc); assertEquals(expected, doc.getFieldValue("my_pos")); diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/UriParserTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/UriParserTestCase.java index dcdea0975ad..0ccae4dbde5 100644 --- a/document/src/test/java/com/yahoo/vespaxmlparser/UriParserTestCase.java +++ b/document/src/test/java/com/yahoo/vespaxmlparser/UriParserTestCase.java @@ -28,7 +28,7 @@ public class UriParserTestCase { mgr.registerDocumentType(docType); VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test_uri.xml", mgr); - Iterator<VespaXMLFeedReader.Operation> it = parser.readAll().iterator(); + Iterator<FeedOperation> it = parser.readAll().iterator(); Document doc = nextDocument(it); assertNotNull(doc); @@ -59,21 +59,21 @@ public class UriParserTestCase { assertFalse(it.hasNext()); } - private static Document nextDocument(Iterator<VespaXMLFeedReader.Operation> it) { + private static Document nextDocument(Iterator<FeedOperation> it) { assertTrue(it.hasNext()); - VespaXMLFeedReader.Operation op = it.next(); + FeedOperation op = it.next(); assertNotNull(op); - assertEquals(VespaXMLFeedReader.OperationType.DOCUMENT, op.getType()); + assertEquals(FeedOperation.Type.DOCUMENT, op.getType()); Document doc = op.getDocument(); assertNotNull(doc); return doc; } - private static DocumentUpdate nextUpdate(Iterator<VespaXMLFeedReader.Operation> it) { + private static DocumentUpdate nextUpdate(Iterator<FeedOperation> it) { assertTrue(it.hasNext()); - VespaXMLFeedReader.Operation op = it.next(); + FeedOperation op = it.next(); assertNotNull(op); - assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType()); + assertEquals(FeedOperation.Type.UPDATE, op.getType()); DocumentUpdate upd = op.getDocumentUpdate(); assertNotNull(upd); return upd; diff --git a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXMLReaderTestCase.java b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXMLReaderTestCase.java index 1aad59f4c56..e33dbfe8898 100755 --- a/document/src/test/java/com/yahoo/vespaxmlparser/VespaXMLReaderTestCase.java +++ b/document/src/test/java/com/yahoo/vespaxmlparser/VespaXMLReaderTestCase.java @@ -40,7 +40,7 @@ public class VespaXMLReaderTestCase { } @Test - public void testMapNoKey() throws Exception { + public void testMapNoKey() { try { VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/testmapnokey.xml", manager); parser.readAll(); @@ -51,7 +51,7 @@ public class VespaXMLReaderTestCase { } @Test - public void testMapNoValue() throws Exception { + public void testMapNoValue() { try { VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/testmapnovalue.xml", manager); parser.readAll(); @@ -64,10 +64,9 @@ public class VespaXMLReaderTestCase { @Test public void testNews1() throws Exception { VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/testalltypes.xml", manager); - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); - assertTrue(VespaXMLFeedReader.OperationType.INVALID != op.getType()); + assertTrue(FeedOperation.Type.INVALID != op.getType()); Document doc = op.getDocument(); assertEquals(new StringFieldValue("testUrl"), doc.getFieldValue("url")); assertEquals(new StringFieldValue("testTitle"), doc.getFieldValue("title")); @@ -149,10 +148,9 @@ public class VespaXMLReaderTestCase { public void testNews3() throws Exception { // Updating all elements in a documentType VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test03.xml", manager); - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); - assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType()); + assertEquals(FeedOperation.Type.UPDATE, op.getType()); DocumentUpdate docUpdate = op.getDocumentUpdate(); @@ -214,10 +212,9 @@ public class VespaXMLReaderTestCase { // Test on adding just a few fields to a DocumentUpdate (implies other fields to null) VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test04.xml", manager); - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); - assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType()); + assertEquals(FeedOperation.Type.UPDATE, op.getType()); DocumentUpdate docUpdate = op.getDocumentUpdate(); //url @@ -269,10 +266,9 @@ public class VespaXMLReaderTestCase { // Adding a few new fields to a Document using different syntax VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test05.xml", manager); - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); - assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType()); + assertEquals(FeedOperation.Type.UPDATE, op.getType()); DocumentUpdate docUpdate = op.getDocumentUpdate(); @@ -334,20 +330,19 @@ public class VespaXMLReaderTestCase { // long value with txt try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); } // empty string - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); assertEquals("doc:news:http://news6b", op.getDocument().getId().toString()); // int array with text try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -355,7 +350,7 @@ public class VespaXMLReaderTestCase { // long array with whitespace try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -363,7 +358,7 @@ public class VespaXMLReaderTestCase { // byte array with value try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -371,7 +366,7 @@ public class VespaXMLReaderTestCase { // float array with string try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -379,7 +374,7 @@ public class VespaXMLReaderTestCase { // weighted set of int with string try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -387,7 +382,7 @@ public class VespaXMLReaderTestCase { // weighted set of int with string as weight try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -395,17 +390,17 @@ public class VespaXMLReaderTestCase { // weighted set of string with string as weight try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); } - parser.read(op = new VespaXMLFeedReader.Operation()); + op = parser.read(); assertEquals("doc:news:http://news6j", op.getDocument().getId().toString()); - parser.read(op = new VespaXMLFeedReader.Operation()); - assertEquals(VespaXMLFeedReader.OperationType.INVALID, op.getType()); + op = parser.read(); + assertEquals(FeedOperation.Type.INVALID, op.getType()); } @Test @@ -414,10 +409,9 @@ public class VespaXMLReaderTestCase { // are also some updates that will fail (be skipped). VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test07.xml", manager); - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); - assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType()); + assertEquals(FeedOperation.Type.UPDATE, op.getType()); DocumentUpdate docUpdate = op.getDocumentUpdate(); @@ -449,7 +443,7 @@ public class VespaXMLReaderTestCase { // Trying arithmetic on string (b) try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -457,7 +451,7 @@ public class VespaXMLReaderTestCase { // "By" as string (c) try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -465,7 +459,7 @@ public class VespaXMLReaderTestCase { // Empty key in weighted set of int (d) try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -473,7 +467,7 @@ public class VespaXMLReaderTestCase { // No "by" attribute (e) try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -481,7 +475,7 @@ public class VespaXMLReaderTestCase { // Float key as string (f) try { - parser.read(new VespaXMLFeedReader.Operation()); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -492,10 +486,9 @@ public class VespaXMLReaderTestCase { public void testNews8() throws Exception { VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test08.xml", manager); - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); - assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType()); + assertEquals(FeedOperation.Type.UPDATE, op.getType()); DocumentUpdate docUpdate = op.getDocumentUpdate(); @@ -518,24 +511,21 @@ public class VespaXMLReaderTestCase { VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test09.xml", manager); { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); - assertEquals(VespaXMLFeedReader.OperationType.REMOVE, op.getType()); + assertEquals(FeedOperation.Type.REMOVE, op.getType()); assertEquals("doc:news:http://news9a", op.getRemove().toString()); } { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); - assertEquals(VespaXMLFeedReader.OperationType.REMOVE, op.getType()); + assertEquals(FeedOperation.Type.REMOVE, op.getType()); assertEquals("doc:news:http://news9b", op.getRemove().toString()); } { // Remove without documentid. Not supported. - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); try { - parser.read(op); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -547,8 +537,7 @@ public class VespaXMLReaderTestCase { public void testNews10() throws Exception { VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test10.xml", manager); { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); Document doc = op.getDocument(); assertEquals(new StringFieldValue("testUrl"), doc.getFieldValue("url")); @@ -585,15 +574,13 @@ public class VespaXMLReaderTestCase { assertEquals(Integer.valueOf(14), strWset.get(new StringFieldValue("string14"))); } { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); Document doc = op.getDocument(); assertNotNull(doc); assertEquals(new StringFieldValue("testUrl2"), doc.getFieldValue("url")); } { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); DocumentUpdate upd = op.getDocumentUpdate(); assertNull(upd.getFieldUpdate("url")); @@ -629,8 +616,7 @@ public class VespaXMLReaderTestCase { .getWeight()); } { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); DocumentUpdate upd = op.getDocumentUpdate(); assertEquals(new StringFieldValue("assignUrl"), @@ -661,15 +647,13 @@ public class VespaXMLReaderTestCase { assertNull(upd.getFieldUpdate("weightedsetstring")); } { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); assertEquals("doc:news:http://news10e", op.getRemove().toString()); } { // Illegal remove without documentid attribute try { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + parser.read(); fail(); } catch (Exception e) { System.out.println(e.getMessage()); @@ -682,10 +666,9 @@ public class VespaXMLReaderTestCase { // Adding a few new fields to a Document using different syntax VespaXMLFeedReader parser = new VespaXMLFeedReader("src/tests/vespaxml/fieldpathupdates.xml", manager); - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); - assertEquals(VespaXMLFeedReader.OperationType.UPDATE, op.getType()); + assertEquals(FeedOperation.Type.UPDATE, op.getType()); DocumentUpdate docUpdate = op.getDocumentUpdate(); @@ -801,16 +784,15 @@ public class VespaXMLReaderTestCase { .configure(m, "file:src/test/java/com/yahoo/document/documentmanager.docindoc.cfg"); VespaXMLFeedReader parser = new VespaXMLFeedReader("src/test/vespaxmlparser/test_docindoc.xml", m); - List<VespaXMLFeedReader.Operation> ops = parser.readAll(); + List<FeedOperation> ops = parser.readAll(); assertEquals(1, ops.size()); - VespaXMLFeedReader.Operation op = ops.get(0); + FeedOperation op = ops.get(0); System.err.println(op); - assertEquals(VespaXMLFeedReader.OperationType.DOCUMENT, op.getType()); + assertEquals(FeedOperation.Type.DOCUMENT, op.getType()); assertNull(op.getRemove()); assertNull(op.getDocumentUpdate()); - assertNull(op.getFeedOperation()); assertNotNull(op.getDocument()); Document doc = op.getDocument(); @@ -887,8 +869,7 @@ public class VespaXMLReaderTestCase { final int NUM_OPERATIONS_IN_FEED = 3; for (int i = 0; i < NUM_OPERATIONS_IN_FEED; i++) { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - parser.read(op); + FeedOperation op = parser.read(); assertTrue("Missing test and set condition", op.getCondition().isPresent()); assertEquals("Condition is not the same as in xml feed", "news.value_long == 1", op.getCondition().getSelection()); diff --git a/documentapi/abi-spec.json b/documentapi/abi-spec.json index cbfe07eb93d..e1e5f778423 100644 --- a/documentapi/abi-spec.json +++ b/documentapi/abi-spec.json @@ -2035,10 +2035,9 @@ "abstract" ], "methods": [ - "public void <init>(java.util.Map)", "public void init()", "public com.yahoo.jrt.slobrok.api.IMirror getMirror()", - "public com.yahoo.jrt.slobrok.api.Mirror$Entry[] lookup(com.yahoo.messagebus.routing.RoutingContext, java.lang.String)", + "public java.util.List lookup(com.yahoo.messagebus.routing.RoutingContext, java.lang.String)", "public synchronized void configure(com.yahoo.cloud.config.SlobroksConfig)", "public void destroy()", "public bridge synthetic void configure(com.yahoo.config.ConfigInstance)" @@ -2207,7 +2206,7 @@ "public void <init>(java.lang.String)", "public java.util.List getNodeWeights()", "public int getIndex(java.lang.String)", - "public com.yahoo.documentapi.messagebus.protocol.LoadBalancer$Node getRecipient(com.yahoo.jrt.slobrok.api.Mirror$Entry[])", + "public com.yahoo.documentapi.messagebus.protocol.LoadBalancer$Node getRecipient(java.util.List)", "public void received(com.yahoo.documentapi.messagebus.protocol.LoadBalancer$Node, boolean)" ], "fields": [] diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java index 312fd6e5964..e81ac4ae05e 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternPolicy.java @@ -103,7 +103,7 @@ public class ExternPolicy implements DocumentProtocolRoutingPolicy { if (generation != upd) { generation = upd; recipients.clear(); - Mirror.Entry[] arr = mirror.lookup(pattern); + List<Mirror.Entry> arr = mirror.lookup(pattern); for (Mirror.Entry entry : arr) { recipients.add(Hop.parse(entry.getSpec() + session)); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternalSlobrokPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternalSlobrokPolicy.java index 3ebe5b7281e..39242bb6cab 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternalSlobrokPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ExternalSlobrokPolicy.java @@ -11,6 +11,7 @@ import com.yahoo.jrt.slobrok.api.SlobrokList; import com.yahoo.messagebus.routing.RoutingContext; import com.yahoo.cloud.config.SlobroksConfig; +import java.util.List; import java.util.Map; /** @@ -19,16 +20,16 @@ import java.util.Map; */ public abstract class ExternalSlobrokPolicy extends AsyncInitializationPolicy implements ConfigSubscriber.SingleSubscriber<SlobroksConfig> { String error; - Supervisor orb = null; - Mirror mirror = null; - SlobrokList slobroks = null; - boolean firstTry = true; + private Supervisor orb = null; + private Mirror mirror = null; + private SlobrokList slobroks = null; + private boolean firstTry = true; private ConfigSubscriber subscriber; String[] configSources = null; - String slobrokConfigId = "admin/slobrok.0"; + private final static String slobrokConfigId = "admin/slobrok.0"; - public ExternalSlobrokPolicy(Map<String, String> param) { + ExternalSlobrokPolicy(Map<String, String> param) { super(); String conf = param.get("config"); @@ -72,16 +73,16 @@ public abstract class ExternalSlobrokPolicy extends AsyncInitializationPolicy im return mirror; } - public Mirror.Entry[] lookup(RoutingContext context, String pattern) { + public List<Mirror.Entry> lookup(RoutingContext context, String pattern) { IMirror mirror1 = (mirror != null ? mirror : context.getMirror()); - Mirror.Entry[] arr = mirror1.lookup(pattern); + List<Mirror.Entry> arr = mirror1.lookup(pattern); - if ((arr.length == 0) && firstTry) { + if ((arr.isEmpty()) && firstTry) { synchronized(this) { try { int count = 0; - while (arr.length == 0 && count < 100) { + while (arr.isEmpty() && count < 100) { Thread.sleep(50); arr = mirror1.lookup(pattern); count++; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancer.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancer.java index 167a480e1aa..a2875f14ab5 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancer.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancer.java @@ -62,8 +62,8 @@ public class LoadBalancer { * @param choices the node choices, represented as Slobrok entries * @return the chosen node, or null only if the given choices were zero */ - public Node getRecipient(Mirror.Entry[] choices) { - if (choices.length == 0) return null; + public Node getRecipient(List<Mirror.Entry> choices) { + if (choices.isEmpty()) return null; double weightSum = 0.0; Node selectedNode = null; @@ -79,7 +79,7 @@ public class LoadBalancer { } if (selectedNode == null) { // Position>sum of all weights: Wrap around (but keep the remainder for some reason) position -= weightSum; - selectedNode = new Node(choices[0], getNodeMetrics(choices[0])); + selectedNode = new Node(choices.get(0), getNodeMetrics(choices.get(0))); } position += 1.0; selectedNode.metrics.sent.incrementAndGet(); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerPolicy.java index 7c5345351a8..9cf82144e71 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LoadBalancerPolicy.java @@ -9,6 +9,7 @@ import com.yahoo.messagebus.routing.Route; import com.yahoo.messagebus.routing.RoutingContext; import com.yahoo.messagebus.routing.RoutingNodeIterator; +import java.util.List; import java.util.Map; /** @@ -75,7 +76,7 @@ public class LoadBalancerPolicy extends ExternalSlobrokPolicy { @return Returns a hop representing the TCP address of the target, or null if none could be found. */ private LoadBalancer.Node getRecipient(RoutingContext context) { - Mirror.Entry [] lastLookup = lookup(context, pattern); + List<Mirror.Entry> lastLookup = lookup(context, pattern); return loadBalancer.getRecipient(lastLookup); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LocalServicePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LocalServicePolicy.java index eb56a85e378..d4ebd4ecd81 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LocalServicePolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/LocalServicePolicy.java @@ -83,7 +83,7 @@ public class LocalServicePolicy implements DocumentProtocolRoutingPolicy { entry.generation = upd; entry.recipients.clear(); - Mirror.Entry[] arr = ctx.getMirror().lookup(ctx.getHopPrefix() + "*" + ctx.getHopSuffix()); + List<Mirror.Entry> arr = ctx.getMirror().lookup(ctx.getHopPrefix() + "*" + ctx.getHopSuffix()); String self = localAddress != null ? localAddress : toAddress(ctx.getMessageBus().getConnectionSpec()); for (Mirror.Entry item : arr) { if (self.equals(toAddress(item.getSpec()))) { diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoundRobinPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoundRobinPolicy.java index 244d101b36f..3a1599ab71b 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoundRobinPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoundRobinPolicy.java @@ -82,7 +82,7 @@ public class RoundRobinPolicy implements DocumentProtocolRoutingPolicy { entry.generation = upd; entry.recipients.clear(); for (int i = 0; i < ctx.getNumRecipients(); ++i) { - Mirror.Entry[] arr = ctx.getMirror().lookup(ctx.getRecipient(i).getHop(0).toString()); + List<Mirror.Entry> arr = ctx.getMirror().lookup(ctx.getRecipient(i).getHop(0).toString()); for (Mirror.Entry item : arr) { entry.recipients.add(Hop.parse(item.getName())); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java index 341589643d2..048149e86ab 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java @@ -107,7 +107,7 @@ public class StoragePolicy extends ExternalSlobrokPolicy { this.policy = policy; } - private Mirror.Entry[] getEntries(String hostPattern, RoutingContext context) { + private List<Mirror.Entry> getEntries(String hostPattern, RoutingContext context) { return policy.lookup(context, hostPattern); } @@ -116,16 +116,16 @@ public class StoragePolicy extends ExternalSlobrokPolicy { public IMirror getMirror(RoutingContext context) { return context.getMirror(); } public String getTargetSpec(Integer distributor, RoutingContext context) { - Mirror.Entry[] arr = getEntries(patternGenerator.getDistributorHostPattern(distributor), context); - if (arr.length == 0) return null; + List<Mirror.Entry> arr = getEntries(patternGenerator.getDistributorHostPattern(distributor), context); + if (arr.isEmpty()) return null; if (distributor != null) { - if (arr.length == 1) { - return convertSlobrokNameToSessionName(arr[0].getSpec()); + if (arr.size() == 1) { + return convertSlobrokNameToSessionName(arr.get(0).getSpec()); } else { - log.log(LogLevel.WARNING, "Got " + arr.length + " matches for a distributor."); + log.log(LogLevel.WARNING, "Got " + arr.size() + " matches for a distributor."); } } else { - return convertSlobrokNameToSessionName(arr[randomizer.nextInt(arr.length)].getSpec()); + return convertSlobrokNameToSessionName(arr.get(randomizer.nextInt(arr.size())).getSpec()); } return null; } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java index 31802f2872f..3251c038fc7 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java @@ -109,10 +109,10 @@ public class SubsetServicePolicy implements DocumentProtocolRoutingPolicy { entry.generation = upd; entry.recipients.clear(); - Mirror.Entry[] arr = ctx.getMirror().lookup(ctx.getHopPrefix() + "*" + ctx.getHopSuffix()); + List<Mirror.Entry> arr = ctx.getMirror().lookup(ctx.getHopPrefix() + "*" + ctx.getHopSuffix()); int pos = ctx.getMessageBus().getConnectionSpec().hashCode(); - for (int i = 0; i < subsetSize && i < arr.length; ++i) { - entry.recipients.add(Hop.parse(arr[((pos + i) & Integer.MAX_VALUE) % arr.length].getName())); + for (int i = 0; i < subsetSize && i < arr.size(); ++i) { + entry.recipients.add(Hop.parse(arr.get(((pos + i) & Integer.MAX_VALUE) % arr.size()).getName())); } } return entry; diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/TargetCachingSlobrokHostFetcherTest.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/TargetCachingSlobrokHostFetcherTest.java index 7e6c7bc468a..4413b657739 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/TargetCachingSlobrokHostFetcherTest.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/TargetCachingSlobrokHostFetcherTest.java @@ -7,6 +7,8 @@ import com.yahoo.messagebus.routing.RoutingContext; import org.junit.Test; import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -23,26 +25,25 @@ import static org.mockito.Mockito.when; */ public class TargetCachingSlobrokHostFetcherTest { - static String idOfIndex(int index) { + private static String idOfIndex(int index) { return String.format("storage/cluster.foo/distributor/%d/default", index); } - static String idOfWildcardLookup() { + private static String idOfWildcardLookup() { return "storage/cluster.foo/distributor/*/default"; } - static String lookupSpecOfIndex(int index) { + private static String lookupSpecOfIndex(int index) { return String.format("tcp/localhost:%d", index); } - static String resolvedSpecOfIndex(int index) { + private static String resolvedSpecOfIndex(int index) { return String.format("tcp/localhost:%d/default", index); } - static Mirror.Entry[] dummyEntries(int... indices) { + private static List<Mirror.Entry> dummyEntries(int... indices) { return Arrays.stream(indices) - .mapToObj(index -> new Mirror.Entry(idOfIndex(index), lookupSpecOfIndex(index))) - .toArray(Mirror.Entry[]::new); + .mapToObj(index -> new Mirror.Entry(idOfIndex(index), lookupSpecOfIndex(index))).collect(Collectors.toList()); } static class Fixture { diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/LoadBalancerTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/LoadBalancerTestCase.java index 698b778c312..51dd1ac12b8 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/LoadBalancerTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/LoadBalancerTestCase.java @@ -5,6 +5,7 @@ import com.yahoo.documentapi.messagebus.protocol.LoadBalancer; import com.yahoo.jrt.slobrok.api.Mirror; import org.junit.Test; +import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; @@ -42,9 +43,9 @@ public class LoadBalancerTestCase { public void testLoadBalancer() { LoadBalancer lb = new LoadBalancer("foo"); - Mirror.Entry[] entries = new Mirror.Entry[]{ new Mirror.Entry("foo/0/default", "tcp/bar:1"), - new Mirror.Entry("foo/1/default", "tcp/bar:2"), - new Mirror.Entry("foo/2/default", "tcp/bar:3") }; + List<Mirror.Entry> entries = Arrays.asList(new Mirror.Entry("foo/0/default", "tcp/bar:1"), + new Mirror.Entry("foo/1/default", "tcp/bar:2"), + new Mirror.Entry("foo/2/default", "tcp/bar:3")); List<LoadBalancer.NodeMetrics> weights = lb.getNodeWeights(); { @@ -100,7 +101,7 @@ public class LoadBalancerTestCase { public void testLoadBalancerOneItemOnly() { LoadBalancer lb = new LoadBalancer("foo"); - Mirror.Entry[] entries = new Mirror.Entry[]{ new Mirror.Entry("foo/0/default", "tcp/bar:1") }; + List<Mirror.Entry> entries = Arrays.asList(new Mirror.Entry("foo/0/default", "tcp/bar:1") ); List<LoadBalancer.NodeMetrics> weights = lb.getNodeWeights(); assertEquals("foo/0/default" , lb.getRecipient(entries).entry.getName()); diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java index e2f1c9cd937..fd9d3f78ca8 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestCase.java @@ -1,21 +1,61 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi.messagebus.protocol.test; -import com.yahoo.document.*; -import com.yahoo.documentapi.messagebus.protocol.*; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; +import com.yahoo.document.DocumentPut; +import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentTypeManagerConfigurer; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.documentapi.messagebus.protocol.ANDPolicy; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.documentapi.messagebus.protocol.DocumentRouteSelectorPolicy; +import com.yahoo.documentapi.messagebus.protocol.ErrorPolicy; +import com.yahoo.documentapi.messagebus.protocol.ExternPolicy; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.GetDocumentReply; +import com.yahoo.documentapi.messagebus.protocol.LoadBalancerPolicy; +import com.yahoo.documentapi.messagebus.protocol.LocalServicePolicy; +import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.RoundRobinPolicy; +import com.yahoo.documentapi.messagebus.protocol.SubsetServicePolicy; +import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; import com.yahoo.jrt.ListenFailedException; import com.yahoo.jrt.slobrok.api.IMirror; import com.yahoo.jrt.slobrok.api.Mirror; import com.yahoo.jrt.slobrok.server.Slobrok; -import com.yahoo.messagebus.*; +import com.yahoo.messagebus.DestinationSession; +import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.IntermediateSession; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.MessageBus; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.SourceSession; +import com.yahoo.messagebus.SourceSessionParams; import com.yahoo.messagebus.network.rpc.test.TestServer; -import com.yahoo.messagebus.routing.*; +import com.yahoo.messagebus.routing.HopBlueprint; +import com.yahoo.messagebus.routing.HopSpec; +import com.yahoo.messagebus.routing.PolicyDirective; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.messagebus.routing.RouteSpec; +import com.yahoo.messagebus.routing.RoutingNode; +import com.yahoo.messagebus.routing.RoutingPolicy; +import com.yahoo.messagebus.routing.RoutingSpec; +import com.yahoo.messagebus.routing.RoutingTableSpec; import com.yahoo.messagebus.test.Receptor; import org.junit.Before; import org.junit.Test; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -721,7 +761,7 @@ public class PolicyTestCase { throws InterruptedException, TimeoutException { for (int i = 0; i < TIMEOUT_MILLIS / 10; ++i) { - if (slobrok.lookup(pattern).length == numEntries) { + if (slobrok.lookup(pattern).size() == numEntries) { return; } Thread.sleep(10); diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java index 875ef276b01..89d5db62899 100755 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/PolicyTestFrame.java @@ -6,19 +6,32 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.jrt.ListenFailedException; import com.yahoo.jrt.slobrok.api.Mirror; import com.yahoo.jrt.slobrok.server.Slobrok; -import com.yahoo.messagebus.*; +import com.yahoo.messagebus.EmptyReply; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.MessageBus; +import com.yahoo.messagebus.MessageBusParams; +import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.network.Identity; import com.yahoo.messagebus.network.Network; import com.yahoo.messagebus.network.ServiceAddress; import com.yahoo.messagebus.network.rpc.RPCNetwork; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; import com.yahoo.messagebus.network.rpc.test.TestServer; -import com.yahoo.messagebus.routing.*; +import com.yahoo.messagebus.routing.HopSpec; +import com.yahoo.messagebus.routing.Route; +import com.yahoo.messagebus.routing.RoutingNode; +import com.yahoo.messagebus.routing.RoutingSpec; +import com.yahoo.messagebus.routing.RoutingTableSpec; import com.yahoo.messagebus.test.Receptor; import com.yahoo.messagebus.test.SimpleProtocol; import com.yahoo.messagebus.test.SimpleReply; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import static org.junit.Assert.assertEquals; @@ -272,8 +285,8 @@ public class PolicyTestFrame { */ public boolean waitSlobrok(String pattern, int cnt) { for (int i = 0; i < 1000 && !Thread.currentThread().isInterrupted(); ++i) { - Mirror.Entry[] res = net.getMirror().lookup(pattern); - if (res.length == cnt) { + List<Mirror.Entry> res = net.getMirror().lookup(pattern); + if (res.size() == cnt) { return true; } try { Thread.sleep(10); } catch (InterruptedException e) { /* ignore */ } diff --git a/fnet/src/vespa/fnet/connection.h b/fnet/src/vespa/fnet/connection.h index b52d0147ce1..b4272b91cef 100644 --- a/fnet/src/vespa/fnet/connection.h +++ b/fnet/src/vespa/fnet/connection.h @@ -58,9 +58,9 @@ public: }; enum { - FNET_READ_SIZE = 8192, + FNET_READ_SIZE = 16384, FNET_READ_REDO = 10, - FNET_WRITE_SIZE = 8192, + FNET_WRITE_SIZE = 16384, FNET_WRITE_REDO = 10 }; diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/metrics/package-info.java b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/metrics/package-info.java deleted file mode 100644 index 9c425f0a0df..00000000000 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/metrics/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -@ExportPackage -package com.yahoo.messagebus.metrics; - -import com.yahoo.osgi.annotation.ExportPackage; diff --git a/jrt/src/com/yahoo/jrt/Connection.java b/jrt/src/com/yahoo/jrt/Connection.java index d4938f8ecbb..b8ed1b32eda 100644 --- a/jrt/src/com/yahoo/jrt/Connection.java +++ b/jrt/src/com/yahoo/jrt/Connection.java @@ -18,9 +18,9 @@ class Connection extends Target { private static Logger log = Logger.getLogger(Connection.class.getName()); - private static final int READ_SIZE = 8192; + private static final int READ_SIZE = 16384; private static final int READ_REDO = 10; - private static final int WRITE_SIZE = 8192; + private static final int WRITE_SIZE = 16384; private static final int WRITE_REDO = 10; private static final int INITIAL = 0; diff --git a/jrt/src/com/yahoo/jrt/slobrok/api/IMirror.java b/jrt/src/com/yahoo/jrt/slobrok/api/IMirror.java index 9b3bd81fc3c..0079e2c9d67 100644 --- a/jrt/src/com/yahoo/jrt/slobrok/api/IMirror.java +++ b/jrt/src/com/yahoo/jrt/slobrok/api/IMirror.java @@ -1,6 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jrt.slobrok.api; +import java.util.List; + /** * Defines an interface for the name server lookup. * @@ -21,7 +23,7 @@ public interface IMirror { * @return a list of all matching services, with corresponding connect specs * @param pattern The pattern used for matching **/ - Mirror.Entry[] lookup(String pattern); + List<Mirror.Entry> lookup(String pattern); /** * Obtain the number of updates seen by this mirror. The value may wrap, but will never become 0 again. This can be diff --git a/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java b/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java index 127aa3d0d4f..c632191e31d 100644 --- a/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java +++ b/jrt/src/com/yahoo/jrt/slobrok/api/Mirror.java @@ -14,6 +14,7 @@ import com.yahoo.jrt.Values; import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.logging.Logger; import java.util.logging.Level; @@ -86,7 +87,7 @@ public class Mirror implements IMirror { } @Override - public Entry[] lookup(String pattern) { + public List<Entry> lookup(String pattern) { ArrayList<Entry> found = new ArrayList<>(); char[] p = pattern.toCharArray(); for (Entry specEntry : specs) { @@ -94,7 +95,7 @@ public class Mirror implements IMirror { found.add(specEntry); } } - return found.toArray(new Entry[found.size()]); + return found; } @Override diff --git a/jrt/tests/com/yahoo/jrt/SlobrokTest.java b/jrt/tests/com/yahoo/jrt/SlobrokTest.java index ee15c7cd1de..20266b0826a 100644 --- a/jrt/tests/com/yahoo/jrt/SlobrokTest.java +++ b/jrt/tests/com/yahoo/jrt/SlobrokTest.java @@ -1,10 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jrt; - import java.util.ArrayList; -import java.util.Arrays; import java.util.Comparator; +import java.util.List; import com.yahoo.jrt.slobrok.api.SlobrokList; import com.yahoo.jrt.slobrok.api.Mirror; @@ -17,7 +16,6 @@ import org.junit.Before; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; - public class SlobrokTest { private static class SpecList extends ArrayList<Mirror.Entry> { @@ -71,14 +69,13 @@ public class SlobrokTest { return a.compareTo(b); } }; - Mirror.Entry[] expect = - result.toArray(new Mirror.Entry[result.size()]); - Arrays.sort(expect, cmp); - Mirror.Entry[] actual = new Mirror.Entry[0]; + List<Entry> expect = result; + expect.sort(cmp); + List<Entry> actual = new ArrayList<>(); for (int i = 0; i < 1000; i++) { actual = mirror.lookup(pattern); - Arrays.sort(actual, cmp); - if (Arrays.equals(actual, expect)) { + actual.sort(cmp); + if (actual.equals(expect)) { // err("lookup successful for pattern: " + pattern); return; } @@ -87,18 +84,18 @@ public class SlobrokTest { error = true; err("lookup failed for pattern: " + pattern); err("actual values:"); - if (actual.length == 0) { + if (actual.isEmpty()) { err(" { EMPTY }"); } - for (int i = 0; i < actual.length; i++) { - err(" {" + actual[i].getName() + ", " + actual[i].getSpec() + "}"); + for (Entry e : actual) { + err(" {" + e.getName() + ", " + e.getSpec() + "}"); } err("expected values:"); - if (expect.length == 0) { + if (expect.isEmpty()) { err(" { EMPTY }"); } - for (int i = 0; i < expect.length; i++) { - err(" {" + expect[i].getName() + ", " + expect[i].getSpec() + "}"); + for (Entry e : expect) { + err(" {" + e.getName() + ", " + e.getSpec() + "}"); } } @@ -113,9 +110,9 @@ public class SlobrokTest { assertTrue(mirror.ready()); assertTrue(mirror.updates() > 0); - Mirror.Entry[] oneArr = mirror.lookup("*/*/*"); - assertTrue(oneArr.length == 1); - Mirror.Entry one = oneArr[0]; + List<Entry> oneArr = mirror.lookup("*/*/*"); + assertTrue(oneArr.size() == 1); + Mirror.Entry one = oneArr.get(0); assertTrue(one.equals(new Mirror.Entry(wantName, mySpec))); assertFalse(one.equals(new Mirror.Entry("B/x/w", mySpec))); assertFalse(one.equals(new Mirror.Entry(wantName, "foo:99"))); diff --git a/messagebus-disc/pom.xml b/messagebus-disc/pom.xml index 843936e3290..78b3a3ca8e8 100644 --- a/messagebus-disc/pom.xml +++ b/messagebus-disc/pom.xml @@ -80,7 +80,7 @@ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> <manifestEntries> <Bundle-SymbolicName>${project.groupId}.${project.artifactId}</Bundle-SymbolicName> - <Export-Package>com.yahoo.messagebus,com.yahoo.messagebus.jdisc,com.yahoo.messagebus.metrics,com.yahoo.messagebus.network, + <Export-Package>com.yahoo.messagebus,com.yahoo.messagebus.jdisc,com.yahoo.messagebus.network, com.yahoo.messagebus.network.rpc,com.yahoo.messagebus.routing,com.yahoo.messagebus.shared,com.yahoo.messagebus.test</Export-Package> <Import-Package> com.google.inject;version="1.3", diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/local/LocalWire.java b/messagebus/src/main/java/com/yahoo/messagebus/network/local/LocalWire.java index 4f5fd6ab30a..9f6295b1ad2 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/local/LocalWire.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/local/LocalWire.java @@ -41,7 +41,7 @@ public class LocalWire implements IMirror { } @Override - public Mirror.Entry[] lookup(String pattern) { + public List<Mirror.Entry> lookup(String pattern) { List<Mirror.Entry> out = new ArrayList<>(); Pattern regex = Pattern.compile(pattern.replace("*", "[a-zA-Z0-9_-]+")); for (String key : services.keySet()) { @@ -49,7 +49,7 @@ public class LocalWire implements IMirror { out.add(new Mirror.Entry(key, key)); } } - return out.toArray(new Mirror.Entry[out.size()]); + return out; } @Override diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCService.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCService.java index abe1b7b4db3..7c404207737 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCService.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCService.java @@ -4,6 +4,7 @@ package com.yahoo.messagebus.network.rpc; import com.yahoo.jrt.slobrok.api.IMirror; import com.yahoo.jrt.slobrok.api.Mirror; +import java.util.List; import java.util.concurrent.ThreadLocalRandom; /** @@ -18,7 +19,7 @@ public class RPCService { private final String pattern; private int addressIdx = ThreadLocalRandom.current().nextInt(Integer.MAX_VALUE); private int addressGen = 0; - private Mirror.Entry[] addressList = null; + private List<Mirror.Entry> addressList = null; /** * Create a new RPCService backed by the given network and using the given service pattern. @@ -51,9 +52,9 @@ public class RPCService { addressGen = mirror.updates(); addressList = mirror.lookup(pattern); } - if (addressList != null && addressList.length > 0) { - addressIdx = ++addressIdx % addressList.length; - Mirror.Entry entry = addressList[addressIdx]; + if (addressList != null && !addressList.isEmpty()) { + addressIdx = ++addressIdx % addressList.size(); + Mirror.Entry entry = addressList.get(addressIdx); return new RPCServiceAddress(entry.getName(), entry.getSpec()); } } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/test/TestServer.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/test/TestServer.java index d7ce31a5223..b17bb892f7d 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/test/TestServer.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/test/TestServer.java @@ -17,6 +17,7 @@ import com.yahoo.messagebus.routing.RoutingSpec; import com.yahoo.messagebus.routing.RoutingTableSpec; import com.yahoo.messagebus.test.SimpleProtocol; +import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Logger; @@ -121,8 +122,8 @@ public class TestServer { for (int i = 0; i < 6000 && !Thread.currentThread().isInterrupted(); ++i) { boolean done = true; for (String pattern : slobrokState.getPatterns()) { - Mirror.Entry[] res = net.getMirror().lookup(pattern); - if (res.length != slobrokState.getCount(pattern)) { + List<Mirror.Entry> res = net.getMirror().lookup(pattern); + if (res.size() != slobrokState.getCount(pattern)) { done = false; } } diff --git a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/ServiceAddressTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/ServiceAddressTestCase.java index 476d85f59e9..1dbb30de585 100755 --- a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/ServiceAddressTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/ServiceAddressTestCase.java @@ -11,6 +11,7 @@ import org.junit.Before; import org.junit.Test; import java.net.UnknownHostException; +import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -67,8 +68,8 @@ public class ServiceAddressTestCase { private boolean waitSlobrok(String pattern, int num) { for (int i = 0; i < 1000 && !Thread.currentThread().isInterrupted(); ++i) { - Mirror.Entry[] res = network.getMirror().lookup(pattern); - if (res.length == num) { + List<Mirror.Entry> res = network.getMirror().lookup(pattern); + if (res.size() == num) { return true; } try { diff --git a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/SlobrokTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/SlobrokTestCase.java index c6737385f4c..dd779fd84c0 100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/SlobrokTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/network/rpc/SlobrokTestCase.java @@ -31,8 +31,8 @@ public class SlobrokTestCase { lst.add(new Mirror.Entry(fullName, spec)); return this; } - public Mirror.Entry[] toArray() { - return lst.toArray(new Mirror.Entry[lst.size()]); + public List<Mirror.Entry> toArray() { + return lst; } } @@ -44,18 +44,18 @@ public class SlobrokTestCase { int port2; int port3; - void check(RPCNetwork net, String pattern, Mirror.Entry[] expect) { + void check(RPCNetwork net, String pattern, List<Mirror.Entry> expect) { Comparator<Mirror.Entry> cmp = new Comparator<Mirror.Entry>() { public int compare(Mirror.Entry a, Mirror.Entry b) { return a.compareTo(b); } }; - Arrays.sort(expect, cmp); - Mirror.Entry[] actual = null; + expect.sort(cmp); + List<Mirror.Entry> actual = null; for (int i = 0; i < 1000; i++) { actual = net.getMirror().lookup(pattern); - Arrays.sort(actual, cmp); - if (Arrays.equals(actual, expect)) { + actual.sort(cmp); + if (actual.equals(expect)) { System.out.printf("lookup successful for pattern: %s\n", pattern); return; } @@ -65,7 +65,7 @@ public class SlobrokTestCase { } System.out.printf("lookup failed for pattern: %s\n", pattern); System.out.printf("actual values:\n"); - if (actual == null || actual.length == 0) { + if (actual == null || actual.isEmpty()) { System.out.printf(" { EMPTY }\n"); } else { for (Mirror.Entry entry : actual) { @@ -73,7 +73,7 @@ public class SlobrokTestCase { } } System.out.printf("expected values:\n"); - if (expect.length == 0) { + if (expect.isEmpty()) { System.out.printf(" { EMPTY }\n"); } else { for (Mirror.Entry entry : expect) { diff --git a/metrics/src/main/java/com/yahoo/metrics/ConsumerSpec.java b/metrics/src/main/java/com/yahoo/metrics/ConsumerSpec.java deleted file mode 100644 index f97d8fd85ee..00000000000 --- a/metrics/src/main/java/com/yahoo/metrics/ConsumerSpec.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -import java.util.HashSet; -import java.util.Set; -import java.util.StringTokenizer; - -/** - * Spec saved from config. If metricSetChildren has content, metric pointed - * to is a metric set. - */ -class ConsumerSpec { - Set<String> includedMetrics = new HashSet<String>(); - - public boolean contains(Metric m) { - return includedMetrics.contains(m.getPath()); - } - - public void register(String path) { - StringTokenizer tokenizer = new StringTokenizer(path, "."); - - String total = ""; - - while (tokenizer.hasMoreTokens()) { - if (!total.isEmpty()) { - total += "."; - } - total += tokenizer.nextToken(); - includedMetrics.add(total); - } - } -} diff --git a/metrics/src/main/java/com/yahoo/metrics/CountMetric.java b/metrics/src/main/java/com/yahoo/metrics/CountMetric.java deleted file mode 100644 index 6d28a83aa0b..00000000000 --- a/metrics/src/main/java/com/yahoo/metrics/CountMetric.java +++ /dev/null @@ -1,222 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -import com.yahoo.metrics.util.MetricValueSet; -import com.yahoo.metrics.util.ValueType; -import com.yahoo.metrics.util.HasCopy; -import com.yahoo.text.Utf8String; -import com.yahoo.text.XMLWriter; - -import java.util.Collection; -import java.util.Locale; -import java.util.logging.Logger; - -/** - * A metric that counts something. The value should always be positive. - */ -@SuppressWarnings("unchecked") -public class CountMetric extends Metric { - - public static final int LOG_IF_UNSET = 2; - - private static final Utf8String AVERAGE_CHANGE_PER_SECOND = new Utf8String("average_change_per_second"); - private static final Utf8String COUNT = new Utf8String("count"); - private static final Logger log = Logger.getLogger(CountMetric.class.getName()); - private final MetricValueSet<CountValue> values; - private int flags; - - public CountMetric(String name, String tags, String description, MetricSet owner) { - super(name, tags, description, owner); - values = new MetricValueSet<CountValue>(); - flags = LOG_IF_UNSET; - } - - public CountMetric(CountMetric other, CopyType copyType, MetricSet owner) { - super(other, owner); - values = new MetricValueSet<CountValue>(other.values, copyType == CopyType.CLONE ? other.values.size() : 1); - flags = other.flags; - } - - private CountValue getValues() { - return values.getValue(); - } - - public long getValue() { - CountValue val = getValues(); - return (val == null ? 0 : val.value); - } - - @SuppressWarnings("UnusedDeclaration") - public void logOnlyIfSet() { - flags &= LOG_IF_UNSET; - } - - public void set(long value) { - while (!values.setValue(new CountValue(value))) { - // try again - } - } - - public void inc() { - inc(1); - } - - public void dec() { - dec(1); - } - - public void inc(long i) { - boolean overflow; - CountValue val; - do { - val = getValues(); - if (val == null) { - val = new CountValue(0); - } - overflow = (val.value + i < val.value); - val.value += i; - } while (!values.setValue(val)); - - if (overflow) { - reset(); - log.fine("Overflow in metric " + getName() + ". Resetting it."); - } - } - - public void dec(long i) { - boolean underflow; - CountValue val; - do { - val = getValues(); - if (val == null) { - val = new CountValue(0); - } - underflow = (val.value - i > val.value); - val.value -= i; - } while (!values.setValue(val)); - - if (underflow) { - reset(); - log.fine("Underflow in metric " + getName() + ". Resetting it."); - } - } - - @Override - public void reset() { - values.reset(); - } - - @Override - public boolean logFromTotalMetrics() { - return true; - } - - @Override - public void logEvent(EventLogger logger, String fullName) { - CountValue val = getValues(); - - if ((flags & LOG_IF_UNSET) != 0 || val != null) { - logger.count(fullName, val == null ? 0 : val.value); - } - } - - @Override - public void printXml(XMLWriter writer, - int secondsPassed, - int verbosity) - { - CountValue valRef = getValues(); - if (valRef == null && verbosity < 2) { - return; - } - long val = valRef != null ? valRef.value : 0; - openXMLTag(writer, verbosity); - writer.attribute(COUNT, String.valueOf(val)); - - if (secondsPassed > 0) { - writer.attribute(AVERAGE_CHANGE_PER_SECOND, - String.format(Locale.US, "%.2f", (double)val / secondsPassed)); - } - - writer.closeTag(); - } - - // Only one metric in valuemetric, so return it on any id. - @Override - public long getLongValue(String id) { - CountValue val = getValues(); - return (val == null ? 0 : val.value); - } - - @Override - public double getDoubleValue(String id) { - CountValue val = getValues(); - return (val == null ? 0 : val.value); - } - - @Override - public boolean used() { - return getValues() != null; - } - - @Override - public void addToSnapshot(Metric m) { - CountValue val = getValues(); - if (val != null) { - ((CountMetric)m).inc(val.value); - } - } - - @Override - public void addToPart(Metric m) { - CountValue val = getValues(); - if (val != null) { - ((CountMetric)m).inc(val.value); - } - } - - @Override - public Metric clone(CopyType type, MetricSet owner, boolean includeUnused) { - return new CountMetric(this, type, owner); - } - - private static class CountValue implements ValueType, HasCopy<CountValue> { - - long value; - - private CountValue(long value) { - this.value = value; - } - - public CountValue clone() { - try { - return (CountValue)super.clone(); - } catch (CloneNotSupportedException e) { - return null; - } - } - - public void add(ValueType other) { - value += ((CountValue)other).value; - } - - public ValueType join(Collection<ValueType> sources, JoinBehavior joinBehavior) { - CountValue result = new CountValue(0); - for (ValueType t : sources) { - result.add(t); - } - if (joinBehavior == JoinBehavior.AVERAGE_ON_JOIN) { - result.value /= sources.size(); - } - return result; - } - - public String toString() { - return Long.toString(value); - } - - public CountValue copyObject() { - return new CountValue(value); - } - } -} diff --git a/metrics/src/main/java/com/yahoo/metrics/DoubleValue.java b/metrics/src/main/java/com/yahoo/metrics/DoubleValue.java deleted file mode 100644 index d102f5f7bd8..00000000000 --- a/metrics/src/main/java/com/yahoo/metrics/DoubleValue.java +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -import com.yahoo.metrics.util.ValueType; - -import java.util.Collection; -import java.util.Locale; - -public class DoubleValue implements ValueMetric.Value<Double> { - private int count; - private double min, max, last; - private double total; - - public DoubleValue() { - count = 0; - min = Double.POSITIVE_INFINITY; - max = Double.NEGATIVE_INFINITY; - last = 0; - total = 0; - } - - public String toString() { - return "(count " + count + ", min " + min + ", max " + max + ", last " + last + ", total " + total + ")"; - } - - public void add(Double v) { - count = count + 1; - total = total + v; - min = Math.min(min, v); - max = Math.max(max, v); - last = v; - } - - public void join(ValueMetric.Value<Double> v2, boolean createAverageOnJoin) { - //StringBuffer sb = new StringBuffer(); - //sb.append("Adding " + this + " to " + v2); - if (createAverageOnJoin) { - count += v2.getCount(); - total += v2.getTotal(); - last = v2.getLast(); - - } else { - double totalAverage = getAverage() + v2.getAverage(); - count += v2.getCount(); - total = totalAverage * count; - last += v2.getLast(); - } - min = Math.min(min, v2.getMin()); - max = Math.max(max, v2.getMax()); - //sb.append(" and got " + this); - //System.err.println(sb.toString()); - } - - public void add(ValueType other) { - DoubleValue dv = (DoubleValue) other; - count = count + dv.count; - total = total + dv.total; - min = Math.min(min, dv.min); - max = Math.max(max, dv.max); - last = dv.last; - } - - public ValueType join(Collection<ValueType> sources, JoinBehavior joinBehavior) { - DoubleValue result = new DoubleValue(); - for (ValueType t : sources) { - DoubleValue dv = (DoubleValue) t; - result.count = result.count + dv.count; - result.total = result.total + dv.total; - result.min = Math.min(result.min, dv.min); - result.max = Math.max(result.max, dv.max); - result.last += dv.last; - } - if (joinBehavior == JoinBehavior.AVERAGE_ON_JOIN) { - result.last /= sources.size(); - } else { - result.total *= sources.size(); - } - return result; - } - - public boolean overflow(ValueMetric.Value<Double> v2) { - if (count > (count + v2.getCount())) { - return true; - } - if (v2.getTotal() > 0 && getTotal() > getTotal() + v2.getTotal()) { - return true; - } - if (v2.getTotal() < 0 && getTotal() < getTotal() + v2.getTotal()) { - return true; - } - - return false; - } - - public int getCount() { return count; } - public Double getMin() { return (count > 0) ? min : 0; } - public Double getMax() { return (count > 0) ? max : 0; } - public Double getLast() { return last; } - public Double getTotal() { return total; } - - public Double getAverage() { - if (count == 0) { - return 0.0; - } - - return total / count; - } - - public String valueToString(Double val) { - if (val == Double.MIN_VALUE || val == Double.MAX_VALUE) { - return "0.00"; - } - - return String.format(Locale.US, "%.2f", val); - } - - public DoubleValue clone() { - try{ - return (DoubleValue) super.clone(); - } catch (CloneNotSupportedException e) { return null; } - } - - public ValueMetric.Value<Double> copyObject() { - return clone(); - } - -} diff --git a/metrics/src/main/java/com/yahoo/metrics/EventLogger.java b/metrics/src/main/java/com/yahoo/metrics/EventLogger.java deleted file mode 100644 index 3798b0e2e55..00000000000 --- a/metrics/src/main/java/com/yahoo/metrics/EventLogger.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -public interface EventLogger { - public void value(String name, double value); - public void count(String name, long value); -} diff --git a/metrics/src/main/java/com/yahoo/metrics/JoinBehavior.java b/metrics/src/main/java/com/yahoo/metrics/JoinBehavior.java deleted file mode 100644 index d995e116dd0..00000000000 --- a/metrics/src/main/java/com/yahoo/metrics/JoinBehavior.java +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -/** - * When joining multiple metrics as a result of dimension - * removal. Should the result be an average or a sum? As an example, - * a latency metric should likely be averaged, while a number of - * pending metric should likely be summed. This join behavior property - * lets the metric framework know how to remove dimensions. - **/ -public enum JoinBehavior { AVERAGE_ON_JOIN, SUM_ON_JOIN } diff --git a/metrics/src/main/java/com/yahoo/metrics/LongValue.java b/metrics/src/main/java/com/yahoo/metrics/LongValue.java deleted file mode 100644 index c18d2a5f25b..00000000000 --- a/metrics/src/main/java/com/yahoo/metrics/LongValue.java +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -import com.yahoo.metrics.util.ValueType; - -import java.util.Collection; - -/** - * @author thomasg - */ -public class LongValue - implements ValueMetric.Value<Long> -{ - private int count; - private long min, max, last; - private long total; - - public LongValue() { - count = 0; - min = Long.MAX_VALUE; - max = Long.MIN_VALUE; - last = 0; - total = 0; - } - - @Override - public void add(Long v) { - LongValue val = this; - val.count = count + 1; - val.total = total + v; - val.min = Math.min(min, v); - val.max = Math.max(max, v); - val.last = v; - } - - @Override - public void join(ValueMetric.Value<Long> v2, boolean createAverageOnJoin) { - LongValue value = this; - - if (createAverageOnJoin) { - value.count = count + v2.getCount(); - value.total = total + v2.getTotal(); - value.last = v2.getLast(); - } else { - double totalAverage = getAverage() + v2.getAverage(); - value.count = count + v2.getCount(); - value.total = (long) (totalAverage * value.count); // Total is "wrong" I guess. - value.last = last + v2.getLast(); - } - - value.min = Math.min(min, v2.getMin()); - value.max = Math.max(max, v2.getMax()); - } - - @Override - public void add(ValueType other) { - LongValue dv = (LongValue) other; - count = count + dv.count; - total = total + dv.total; - min = Math.min(min, dv.min); - max = Math.max(max, dv.max); - last = dv.last; - } - - @Override - public ValueType join(Collection<ValueType> sources, JoinBehavior joinBehavior) { - LongValue result = new LongValue(); - for (ValueType t : sources) { - LongValue dv = (LongValue) t; - result.count = result.count + dv.count; - result.total = result.total + dv.total; - result.min = Math.min(result.min, dv.min); - result.max = Math.max(result.max, dv.max); - result.last += dv.last; - } - if (joinBehavior == JoinBehavior.AVERAGE_ON_JOIN) { - result.last /= sources.size(); - } else { - result.total *= sources.size(); - } - return result; - } - - @Override - public boolean overflow(ValueMetric.Value<Long> v2) { - if (count > (count + v2.getCount())) { - return true; - } - if (v2.getTotal() > 0 && getTotal() > getTotal() + v2.getTotal()) { - return true; - } - if (v2.getTotal() < 0 && getTotal() < getTotal() + v2.getTotal()) { - return true; - } - - return false; - } - - @Override - public int getCount() { return count; } - - @Override - public Long getMin() { return (count > 0) ? min : 0; } - - @Override - public Long getMax() { return (count > 0) ? max : 0; } - - @Override - public Long getLast() { return last; } - - @Override - public Long getTotal() { return total; } - - @Override - public Double getAverage() { - if (count == 0) { - return 0.0; - } - return ((double) total) / count; - } - - @Override - public String valueToString(Long val) { - if (val == Long.MIN_VALUE || val == Long.MAX_VALUE) { - return valueToString((long)0); - } - - return val.toString(); - } - - @Override - public LongValue clone() { - try{ - return (LongValue) super.clone(); - } catch (CloneNotSupportedException e) { return null; } - } - - @Override - public ValueMetric.Value<Long> copyObject() { - return clone(); - } -} diff --git a/metrics/src/main/java/com/yahoo/metrics/Metric.java b/metrics/src/main/java/com/yahoo/metrics/Metric.java deleted file mode 100644 index ad7ffc971f6..00000000000 --- a/metrics/src/main/java/com/yahoo/metrics/Metric.java +++ /dev/null @@ -1,256 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -import com.yahoo.text.XMLWriter; -import com.yahoo.text.Utf8String; - -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.List; - -public abstract class Metric { - - private String name; - private String tags; - private String description; - - public String getXMLTag() { - return getName(); - } - - public void setName(String name) { - this.name = name; - } - - public void setTags(String tags) { - this.tags = tags; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getTags() { - return tags; - } - - MetricSet owner; - - public Metric(String name, String tags, String description) { - this(name, tags, description, null); - } - - public Metric(String name, String tags, String description, MetricSet owner) { - this.name = name; - this.tags = tags; - this.description = description; - - if (owner != null) { - owner.registerMetric(this); - } - } - - public Metric(Metric other, MetricSet owner) { - this(other.name, other.tags, other.description, owner); - } - - public String getName() { return name; } - - public String getPath() { - if (owner == null || owner.owner == null) { - return getName(); - } - - return owner.getPath() + "." + getName(); - } - - public List<String> getPathVector() { - List<String> result = new ArrayList<>(); - result.add(getName()); - MetricSet owner = this.owner; - while (owner != null) { - result.add(0, owner.getName()); - owner = owner.owner; - } - return result; - } - - public String getDescription() { return description; } - - public String[] getTagVector() { - return getTags().split("[ \r\t\f]"); - } - - /** - * Returns true if the given tag exists in this metric's tag list. - * - * @return true if tag exists in tag list - */ - public boolean hasTag(String tag) { - for (String s : getTagVector()) { - if (s.equals(tag)) { - return true; - } - } - return false; - } - - public enum CopyType { CLONE, INACTIVE } - - /** - * The clone function will clone metrics to an identical subtree of - * metrics. Clone is primarily used for load metrics that wants to clone - * a template metric for each loadtype. But it should work generically. - * - * @param type If set to inactive, sum metrics will evaluate to primitives - * and metrics can save memory by knowing no updates are coming. - * @param includeUnused When creating snapshots we do not want to include - * unused metrics, but while generating sum metric sum in active - * metrics we want to. This has no affect if type is CLONE. - */ - public abstract Metric clone(CopyType type, MetricSet owner, boolean includeUnused); - - /** - * Utility function for assigning values from one metric of identical type - * to this metric. For simplicity sake it does a const cast and calls - * addToSnapshot, which should not alter source if reset is false. This can - * not be used to copy between active metrics and inactive copies. - * - * @return Returns itself. - */ - public Metric assignValues(Metric m) { - m.addToSnapshot(this); - // As this should only be called among active metrics, all metrics - // should exist and owner list should thus always end up empty. - return this; - } - - /** Reset all metric values. */ - public abstract void reset(); - - public boolean logFromTotalMetrics() { return false; } - - /** Implement to make metric able to log event. - * - * @param logger An event logger to use for logging. - * @param fullName The name to use for the event. - */ - public abstract void logEvent(EventLogger logger, String fullName); - - public static final Utf8String TAG_NAME = new Utf8String("name"); - public static final Utf8String TAG_TAGS = new Utf8String("tags"); - public static final Utf8String TAG_DESC = new Utf8String("description"); - - void openXMLTag(XMLWriter writer, int verbosity) { - String[] tags = getTagVector(); - - writer.openTag(getXMLTag()); - - if ( ! getXMLTag().equals(getName())) { - writer.attribute(TAG_NAME, getName()); - } - - if (verbosity >= 3 && tags.length > 0) { - String tagStr = ""; - for (String tag : tags) { - if (!tagStr.isEmpty()) { - tagStr = ","; - } - tagStr += tag; - } - - writer.attribute(TAG_TAGS, tagStr); - } - - if (verbosity >= 1 && !getDescription().isEmpty()) { - writer.attribute(TAG_DESC, getDescription()); - } - } - - /** - * The verbosity says how much to print. - * At verbosity level 0, only the most critical parts are printed. - * At verbosity level 1, descriptions are added. - * At verbosity level 2, metrics without data is added. - * At verbosity level 3, tags are included too. - */ - public abstract void printXml(XMLWriter writer, - int secondsPassed, - int verbosity); - - public String toXml(int secondsPassed, int verbosity) { - StringWriter writer = new StringWriter(); - printXml(new XMLWriter(writer), secondsPassed, verbosity); - return writer.toString(); - } - - /** - * Most metrics report numbers of some kind. To be able to report numbers - * without having code to handle each possible metric type, these functions - * exist to extract raw data to present easily. - * @param id The part of the metric to extract. For instance, an average - * metric have average, - */ - public abstract long getLongValue(String id); - public abstract double getDoubleValue(String id); - - /** - * When snapshotting we need to be able to join data from one set of metrics - * to another set of metrics taken at another time. MetricSet doesn't know - * the type of the metrics it contains, so we need a generic function for - * doing this. This function assumes metric given as input is of the exact - * same type as the one it is called on for simplicity. This is true when - * adding to snapshots as they have been created with clone and is thus - * always exactly equal. - * - * @param m Metric of exact same type as this one. (Will core if wrong) - */ - abstract void addToSnapshot(Metric m); - - /** - * For sum metrics to work with metric sets, metric sets need operator+=. - * To implement this, we need a function to join any metric type together. - * This is different from adding to snapshot. When adding to snapshots we - * join different time periods to the same metric, but when adding parts - * together we join different metrics for the same time. For instance, an - * average metric of queuesize, should just join new values to create new - * average when adding to snapshot, but when adding parts, the averages - * themselves should be added together. - * - * @param m Metric of exact same type as this one. (Will core if wrong) - */ - abstract void addToPart(Metric m); - - public boolean visit(MetricVisitor visitor, boolean tagAsAutoGenerated) { - return visitor.visitPrimitiveMetric(this, tagAsAutoGenerated); - } - - /** Set whether metrics have ever been set. */ - public abstract boolean used(); - - /** Returns true if this metric is registered in a metric set. */ - public boolean isRegistered() { return (owner != null); } - - /** - * If this metric is registered with an owner, remove itself from that owner. - */ - public void unregister() { - if (isRegistered()) { - getOwner().unregisterMetric(this); - } - } - - public MetricSet getOwner() { return owner; } - - public MetricSet getRoot() { - if (owner == null) { - if (this instanceof MetricSet) { - return (MetricSet)this; - } else { - return null; - } - } else { - return owner.getRoot(); - } - } -} diff --git a/metrics/src/main/java/com/yahoo/metrics/MetricManager.java b/metrics/src/main/java/com/yahoo/metrics/MetricManager.java deleted file mode 100644 index d22cf77c9c9..00000000000 --- a/metrics/src/main/java/com/yahoo/metrics/MetricManager.java +++ /dev/null @@ -1,704 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -import com.yahoo.collections.Pair; -import com.yahoo.text.XMLWriter; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.logging.Logger; - -/** - * A metrics-enabled application should have a single MetricManager. You can register a number of MetricSets in the - * MetricManager. Each metric in the metrics sets can be used by zero or more consumers, configurable using - * readConfig(). - * - * The consumers get their data by calling the getMetrics() method, which gives them a snapshot of all the current - * metrics which are configured for the given name. - * - * Locking strategy: - * - * There are three locks in this class: - * - * Config lock: - This protects the class on config changes. It protects the _config and _consumerConfig members. - * - * Thread monitor (waiter): - This lock is kept by the worker thread while it is doing a work cycle, and it uses this - * monitor to sleep. It is used to make shutdown quick by interrupting thread, and to let functions called by clients be - * able to do a change while the worker thread is idle. - The log period is protected by the thread monitor. - The - * update hooks is protected by the thread monitor. - * - * Metric lock: - The metric log protects the active metric set when adding or removing metrics. Clients need to grab - * this lock before altering active metrics. The metric manager needs to grab this lock everytime it visits active - * metrics. - The metric log protects the snapshots. The snapshot writer is the metric worker thread and will grab the - * lock while editing them. Readers that aren't the worker thread itself must grab lock to be sure. - * - * If multiple locks is taken, the allowed locking order is: 1. Thread monitor. 2. Metric lock. 3. Config lock. - */ -public class MetricManager implements Runnable { - - private static final int STATE_CREATED = 0; - private static final int STATE_RUNNING = 1; - private static final int STATE_STOPPED = 2; - private static final Logger log = Logger.getLogger(MetricManager.class.getName()); - private final CountDownLatch termination = new CountDownLatch(1); - private final MetricSnapshot activeMetrics = new MetricSnapshot("Active metrics showing updates since " + - "last snapshot"); - private final Map<String, ConsumerSpec> consumerConfig = new HashMap<>(); - private final List<UpdateHook> periodicUpdateHooks = new ArrayList<>(); - private final List<UpdateHook> snapshotUpdateHooks = new ArrayList<>(); - private final Timer timer; - private Pair<Integer, Integer> logPeriod; - private List<MetricSnapshotSet> snapshots = new ArrayList<>(); - private MetricSnapshot totalMetrics = new MetricSnapshot("Empty metrics before init", 0, - activeMetrics.getMetrics(), false); - private int state = STATE_CREATED; - private int lastProcessedTime = 0; - private boolean forceEventLogging = false; - private boolean snapshotUnsetMetrics = false; // TODO: add to config - - public MetricManager() { - this(new Timer()); - } - - MetricManager(Timer timer) { - this.timer = timer; - initializeSnapshots(); - logPeriod = new Pair<>(snapshots.get(0).getPeriod(), 0); - } - - void initializeSnapshots() { - int currentTime = timer.secs(); - - List<Pair<Integer, String>> snapshotPeriods = new ArrayList<>(); - snapshotPeriods.add(new Pair<>(60 * 5, "5 minute")); - snapshotPeriods.add(new Pair<>(60 * 60, "1 hour")); - snapshotPeriods.add(new Pair<>(60 * 60 * 24, "1 day")); - snapshotPeriods.add(new Pair<>(60 * 60 * 24 * 7, "1 week")); - - int count = 1; - for (int i = 0; i < snapshotPeriods.size(); ++i) { - int nextCount = 1; - if (i + 1 < snapshotPeriods.size()) { - nextCount = snapshotPeriods.get(i + 1).getFirst() - / snapshotPeriods.get(i).getFirst(); - if (snapshotPeriods.get(i + 1).getFirst() % snapshotPeriods.get(i).getFirst() != 0) { - throw new IllegalStateException("Snapshot periods must be multiplum of each other"); - } - } - snapshots.add(new MetricSnapshotSet(snapshotPeriods.get(i).getSecond(), - snapshotPeriods.get(i).getFirst(), - count, - activeMetrics.getMetrics(), - snapshotUnsetMetrics)); - count = nextCount; - } - // Add all time snapshot. - totalMetrics = new MetricSnapshot("All time snapshot", - 0, activeMetrics.getMetrics(), - snapshotUnsetMetrics); - totalMetrics.reset(currentTime); - } - - public void stop() { - synchronized (this) { - int prevState = state; - state = STATE_STOPPED; - if (prevState == STATE_CREATED) { - return; - } - notifyAll(); - } - try { - termination.await(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - @SuppressWarnings("UnusedDeclaration") - void setSnapshotUnsetMetrics(boolean value) { - snapshotUnsetMetrics = value; - } - - /** - * Add a metric update hook. This will always be called prior to snapshotting and metric logging, to make the - * metrics the best as they can be at those occasions. - * - * @param hook The hook to add. - * @param period Period in seconds for how often callback should be called. The default value of 0, means only - * before snapshotting or logging, while another value will give callbacks each period seconds. - * Expensive metrics to calculate will typically only want to do it before snapshotting, while - * inexpensive metrics might want to log their value every 5 seconds or so. Any value of period >= the - * smallest snapshot time will behave identically as if period is set to 0. - */ - @SuppressWarnings("UnusedDeclaration") - public synchronized void addMetricUpdateHook(UpdateHook hook, int period) { - hook.period = period; - - // If we've already initialized manager, log period has been set. - // In this case. Call first time after period - hook.nextCall = (logPeriod.getSecond() == 0 ? 0 : timer.secs() + period); - if (period == 0) { - if (!snapshotUpdateHooks.contains(hook)) { - snapshotUpdateHooks.add(hook); - } - } else { - if (!periodicUpdateHooks.contains(hook)) { - periodicUpdateHooks.add(hook); - } - } - } - - @SuppressWarnings("UnusedDeclaration") - public synchronized void removeMetricUpdateHook(UpdateHook hook) { - if (hook.period == 0) { - snapshotUpdateHooks.remove(hook); - } else { - periodicUpdateHooks.remove(hook); - } - } - - /** - * Force a metric update for all update hooks. Useful if you want to ensure nice values before reporting something. - * This function can not be called from an update hook callback. - * - * @param includeSnapshotOnlyHooks True to also run snapshot hooks. - */ - @SuppressWarnings("UnusedDeclaration") - public synchronized void updateMetrics(boolean includeSnapshotOnlyHooks) { - log.fine("Giving " + periodicUpdateHooks.size() + " periodic update hooks."); - - updatePeriodicMetrics(0, true); - - if (includeSnapshotOnlyHooks) { - log.fine("Giving " + snapshotUpdateHooks.size() + " snapshot update hooks."); - updateSnapshotMetrics(); - } - } - - /** - * Force event logging to happen now. This function can not be called from an update hook callback. - */ - @SuppressWarnings("UnusedDeclaration") - public void forceEventLogging() { - log.fine("Forcing event logging to happen."); - // Ensure background thread is not in a current cycle during change. - - synchronized (this) { - forceEventLogging = true; - this.notifyAll(); - } - } - - /** - * Register a new metric to be included in the active metric set. You need to have grabbed the metric lock in order - * to do this. (You also need to grab that lock if you alter registration of already registered metric set.) This - * function can not be called from an update hook callback. - * - * @param m The metric to register. - */ - public void registerMetric(Metric m) { - activeMetrics.getMetrics().registerMetric(m); - } - - /** - * Unregister a metric from the active metric set. You need to have grabbed the metric lock in order to do this. - * (You also need to grab that lock if you alter registration of already registered metric set.) This function can - * not be called from an update hook callback. - * - * @param m The Metric to unregister. - */ - @SuppressWarnings("UnusedDeclaration") - public void unregisterMetric(Metric m) { - activeMetrics.getMetrics().unregisterMetric(m); - } - - /** - * Reset all metrics including all snapshots. This function can not be called from an update hook callback. - * - * @param currentTime The current time. - */ - public synchronized void reset(int currentTime) { - activeMetrics.reset(currentTime); - - for (MetricSnapshotSet m : snapshots) { - m.reset(currentTime); - } - totalMetrics.reset(currentTime); - } - - /** - * Read configuration. Before reading config, all metrics should be set up first. By doing this, the metrics manager - * can optimize reporting of consumers. readConfig() will start a config subscription. It should not be called - * multiple times. - */ -/* public synchronized void init(String configId, ThreadPool pool) { - log.fine("Initializing metric manager") - - LOG(debug, "Initializing metric manager."); - _configSubscription = Config::subscribe(configId, *this); - LOG(debug, "Starting worker thread, waiting for first " - "iteration to complete."); - Runnable::start(pool); - // Wait for first iteration to have completed, such that it is safe - // to access snapshots afterwards. - vespalib::MonitorGuard sync(_waiter); - while (_lastProcessedTime == 0) { - sync.wait(1); - } - LOG(debug, "Metric manager completed initialization."); -} - -*/ - - class ConsumerMetricVisitor extends MetricVisitor { - - ConsumerSpec metricsToMatch; - MetricVisitor clientVisitor; - - ConsumerMetricVisitor(ConsumerSpec spec, - MetricVisitor clientVisitor) - { - metricsToMatch = spec; - this.clientVisitor = clientVisitor; - log.fine("Consuming metrics: " + spec.includedMetrics); - } - - public boolean visitMetricSet(MetricSet metricSet, boolean autoGenerated) { - if (metricSet.getOwner() == null) { - return true; - } - - if (!metricsToMatch.contains(metricSet)) { - log.fine("Metric doesn't match " + metricSet.getPath()); - return false; - } - - return clientVisitor.visitMetricSet(metricSet, autoGenerated); - } - - public void doneVisitingMetricSet(MetricSet metricSet) { - if (metricSet.getOwner() != null) { - clientVisitor.doneVisitingMetricSet(metricSet); - } - } - - public boolean visitPrimitiveMetric(Metric metric, boolean autoGenerated) { - if (metricsToMatch.contains(metric)) { - return clientVisitor.visitPrimitiveMetric(metric, autoGenerated); - } else { - log.fine("Metric doesn't match " + metric.getPath()); - } - return true; - } - } - - public synchronized void visit(MetricSet metrics, MetricVisitor visitor, String consumer) { - if (consumer.isEmpty()) { - metrics.visit(visitor, false); - return; - } - - ConsumerSpec spec = getConsumerSpec(consumer); - - if (spec != null) { - ConsumerMetricVisitor consumerVis = new ConsumerMetricVisitor(spec, visitor); - metrics.visit(consumerVis, false); - } else { - log.warning("Requested metrics for non-defined consumer " + consumer); - } - } - - class XmlWriterMetricVisitor extends MetricVisitor { - - int period; - XMLWriter writer; - int verbosity; - - XmlWriterMetricVisitor(XMLWriter writer, int period, int verbosity) { - this.period = period; - this.verbosity = verbosity; - this.writer = writer; - } - - public boolean visitMetricSet(MetricSet set, boolean autoGenerated) { - if (set.used() || verbosity >= 2) { - set.openXMLTag(writer, verbosity); - return true; - } - return false; - } - - public void doneVisitingMetricSet(MetricSet set) { - writer.closeTag(); - } - - public boolean visitPrimitiveMetric(Metric metric, boolean autoGenerated) { - metric.printXml(writer, period, verbosity); - return true; - } - } - - void printXml(MetricSet set, XMLWriter writer, int period, String consumer, int verbosity) { - visit(set, new XmlWriterMetricVisitor(writer, period, verbosity), consumer); - } - - /** - * Synchronize over this while the returned object - * - * @return The MetricSnapshot of all active metrics. - */ - public MetricSnapshot getActiveMetrics() { - return activeMetrics; - } - - /** - * Synchronize over this while the returned object - * - * @return The MetricSnapshot for the total metric. - */ - public MetricSnapshot getTotalMetricSnapshot() { - return totalMetrics; - } - - public synchronized List<Integer> getSnapshotPeriods() { - List<Integer> retVal = new ArrayList<Integer>(); - - for (MetricSnapshotSet m : snapshots) { - retVal.add(m.getPeriod()); - } - return retVal; - } - - /** - * While accessing snapshots you should synchronize over this - * - * @param period The id of the snapshot period to access. - * @param getInProgressSet True to retrieve the snapshot currently being built. - * @return The appropriate MetricSnapshot. - */ - MetricSnapshot getMetricSnapshot(int period, boolean getInProgressSet) { - return getMetricSnapshotSet(period).getSnapshot(getInProgressSet); - } - - MetricSnapshot getMetricSnapshot(int period) { - return getMetricSnapshot(period, false); - } - - public MetricSnapshotSet getMetricSnapshotSet(int period) { - for (MetricSnapshotSet m : snapshots) { - if (m.getPeriod() == period) { - return m; - } - } - - throw new IllegalArgumentException("No snapshot for period of length " + period + " exists."); - } - - @SuppressWarnings("UnusedDeclaration") - public synchronized boolean hasTemporarySnapshot(int period) { - return getMetricSnapshotSet(period).hasTemporarySnapshot(); - } - - public synchronized void addMetricToConsumer(String consumerName, String metricPath) { - ConsumerSpec spec = getConsumerSpec(consumerName); - if (spec == null) { - spec = new ConsumerSpec(); - consumerConfig.put(consumerName, spec); - } - spec.register(metricPath); - } - - public synchronized ConsumerSpec getConsumerSpec(String consumer) { - return consumerConfig.get(consumer); - } - - /** - * If you join or remove metrics from the active metric sets, normally, snapshots will be recreated next snapshot - * period. However, if you want to see the effects of such changes in status pages ahead of that, you can call this - * function in order to check whether snapshots needs to be regenerated and regenerate them if needed. - */ - public synchronized void checkMetricsAltered() { - if (activeMetrics.getMetrics().isRegistrationAltered()) { - handleMetricsAltered(); - } - } - - /** - * Used by unit tests to verify that we have processed for a given time. - * - * @return Returns the timestamp of the previous tick. - */ - @SuppressWarnings("UnusedDeclaration") - public int getLastProcessedTime() { - return lastProcessedTime; - } - - class LogMetricVisitor extends MetricVisitor { - - boolean total; - EventLogger logger; - - LogMetricVisitor(boolean totalVals, EventLogger logger) { - total = totalVals; - this.logger = logger; - } - - public boolean visitPrimitiveMetric(Metric metric, boolean autoGenerated) { - if (metric.logFromTotalMetrics() == total) { - String logName = metric.getPath().replace('.', '_'); - metric.logEvent(logger, logName); - } - return true; - } - } - - public void logTotal(int currentTime, EventLogger logger) { - LogMetricVisitor totalVisitor = new LogMetricVisitor(true, logger); - LogMetricVisitor fiveMinVisitor = new LogMetricVisitor(false, logger); - - if (logPeriod.getSecond() <= currentTime) { - log.fine("Logging total metrics."); - visit(totalMetrics.getMetrics(), totalVisitor, "log"); - visit(snapshots.get(0).getSnapshot().getMetrics(), fiveMinVisitor, "log"); - if (logPeriod.getSecond() + logPeriod.getFirst() < currentTime) { - logPeriod = new Pair<Integer, Integer>(logPeriod.getFirst(), - snapshots.get(0).getFromTime() + logPeriod.getFirst()); - } else { - logPeriod = - new Pair<Integer, Integer>(logPeriod.getFirst(), logPeriod.getSecond() + logPeriod.getFirst()); - } - } - } - - public void logOutOfSequence(int currentTime, EventLogger logger) { - LogMetricVisitor totalVisitor = new LogMetricVisitor(true, logger); - LogMetricVisitor fiveMinVisitor = new LogMetricVisitor(false, logger); - - log.fine("Logging total metrics out of sequence."); - MetricSnapshot snapshot = new MetricSnapshot( - "Total out of sequence metrics from start until current time", - 0, - totalMetrics.getMetrics(), - snapshotUnsetMetrics); - - activeMetrics.addToSnapshot(snapshot, currentTime, false); - snapshot.setFromTime(totalMetrics.getFromTime()); - visit(snapshot.getMetrics(), totalVisitor, "log"); - visit(snapshot.getMetrics(), fiveMinVisitor, "log"); - } - - /** - * Runs one iteration of the thread activity. - * - * @param logger An event logger to use for any new events generated. - * @return The number of milliseconds to sleep until waking up again - */ - public synchronized int tick(EventLogger logger) { - int currentTime = timer.secs(); - - log.finest("Worker thread starting to process for time " + currentTime); - - boolean firstIteration = (logPeriod.getSecond() == 0); - // For a slow system to still be doing metrics tasks each n'th - // second, rather than each n'th + time to do something seconds, - // we constantly join next time to do something from the last timer. - // For that to work, we need to initialize timers on first iteration - // to set them to current time. - if (firstIteration) { - // Setting next log period to now, such that we log metrics - // straight away - logPeriod = new Pair<Integer, Integer>(logPeriod.getFirst(), currentTime); - for (MetricSnapshotSet m : snapshots) { - m.setFromTime(currentTime); - } - for (UpdateHook h : periodicUpdateHooks) { - h.nextCall = currentTime; - } - } - - // If metrics have changed since last time we did a snapshot, - // work that out before taking the snapshot, such that new - // metric can be included - checkMetricsAltered(); - - // Set next work time to the time we want to take next snapshot. - int nextWorkTime = snapshots.get(0).getPeriod() + snapshots.get(0).getFromTime(); - - int nextUpdateHookTime; - - if (nextWorkTime <= currentTime) { - log.fine("Time to do snapshot. Calling update hooks"); - nextUpdateHookTime = updatePeriodicMetrics(currentTime, true); - updateSnapshotMetrics(); - takeSnapshots(nextWorkTime); - } else if (forceEventLogging) { - log.fine("Out of sequence event logging. Calling update hooks"); - nextUpdateHookTime = updatePeriodicMetrics(currentTime, true); - updateSnapshotMetrics(); - } else { - // If not taking a new snapshot. Only give update hooks to - // periodic hooks wanting it. - nextUpdateHookTime = updatePeriodicMetrics(currentTime, false); - } - - // Log if it is time - if (logPeriod.getSecond() <= currentTime || forceEventLogging) { - logTotal(currentTime, logger); - } else { - logOutOfSequence(currentTime, logger); - } - - forceEventLogging = false; - lastProcessedTime = (nextWorkTime <= currentTime ? nextWorkTime : currentTime); - log.fine("Worker thread done with processing for time " + lastProcessedTime); - - int next = Math.min(logPeriod.getSecond(), snapshots.get(0).getPeriod() + snapshots.get(0).getFromTime()); - next = Math.min(next, nextUpdateHookTime); - if (currentTime < next) { - return (next - currentTime) * 1000; - } - - return 0; - } - - @Override - public synchronized void run() { - if (state != STATE_CREATED) { - throw new IllegalStateException(); - } - try { - for (state = STATE_RUNNING; state == STATE_RUNNING; ) { - int timeout = tick(new VespaLogEventLogger()); - if (timeout > 0) { - wait(timeout); - } - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } finally { - termination.countDown(); - } - } - - public synchronized void takeSnapshots(int timeToProcess) { - // If not time to do dump data from active snapshot yet, nothing to do - if (!snapshots.get(0).timeForAnotherSnapshot(timeToProcess)) { - return; - } - - log.fine("Updating " + snapshots.get(0).getName() + " snapshot from active metrics"); - // int fromTime = snapshots.get(0).getSnapshot().getToTime(); - MetricSnapshot firstTarget = (snapshots.get(0).getNextTarget()); - firstTarget.reset(timeToProcess); - activeMetrics.addToSnapshot(firstTarget, timeToProcess, false); - log.fine("Updating total metrics with five minute period of active metrics"); - activeMetrics.addToSnapshot(totalMetrics, timeToProcess, false); - activeMetrics.reset(timeToProcess); - - for (int i = 1; i < snapshots.size(); ++i) { - MetricSnapshotSet s = snapshots.get(i); - - log.fine("Adding data from last snapshot to building snapshot of " + - "next period snapshot " + s.getName()); - - MetricSnapshot target = s.getNextTarget(); - snapshots.get(i - 1).getSnapshot().addToSnapshot(target, timeToProcess, false); - target.setToTime(timeToProcess); - - if (!snapshots.get(i).haveCompletedNewPeriod(timeToProcess)) { - log.fine("Not time to roll snapshot " + s.getName() + " yet. " + - s.getBuilderCount() + " of " + s.getCount() + " snapshot " + - "taken at time" + (s.getBuilderCount() * s.getPeriod() + s.getFromTime()) + - ", and period of " + s.getPeriod() + " is not up " + - "yet as we're currently processing for time " + timeToProcess); - break; - } else { - log.fine("Rolled snapshot " + s.getName() + " at time " + timeToProcess); - } - } - } - - /** - * Utility function for updating periodic metrics. - * - * @param updateTime Update metrics timed to update at this time. - * @param outOfSchedule Force calls to all hooks. Don't screw up normal schedule though. If not time to update yet, - * update without adjusting schedule for next update. - * @return Time of next hook to be called in the future. - */ - int updatePeriodicMetrics(int updateTime, boolean outOfSchedule) { - int nextUpdateTime = Integer.MAX_VALUE; - for (UpdateHook h : periodicUpdateHooks) { - if (h.nextCall <= updateTime) { - h.updateMetrics(); - if (h.nextCall + h.period < updateTime) { - h.nextCall = updateTime + h.period; - } else { - h.nextCall += h.period; - } - } else if (outOfSchedule) { - h.updateMetrics(); - } - nextUpdateTime = Math.min(nextUpdateTime, h.nextCall); - } - return nextUpdateTime; - } - - void updateSnapshotMetrics() { - for (UpdateHook h : snapshotUpdateHooks) { - h.updateMetrics(); - } - } - - synchronized void handleMetricsAltered() { -/* if (consumerConfig.isEmpty()) { - log.fine("Setting up consumers for the first time."); - } else { - log.info("Metrics registration changes detected. Handling changes."); - } - - Map<String, ConsumerSpec> configMap = new HashMap<String, ConsumerSpec>(); - activeMetrics.getMetrics().clearRegistrationAltered(); - - - for (<config::MetricsmanagerConfig::Consumer>::const_iterator it - = _config.consumer.begin(); it != _config.consumer.end(); ++it) - { - ConsumerMetricBuilder consumerMetricBuilder(*it); - _activeMetrics.getMetrics().visit(consumerMetricBuilder); - configMap[it->name] = ConsumerSpec::SP( - new ConsumerSpec(consumerMetricBuilder._matchedMetrics)); - } - _consumerConfig.swap(configMap); - */ - log.fine("Recreating snapshots to include altered metrics"); - totalMetrics.recreateSnapshot(activeMetrics.getMetrics(), snapshotUnsetMetrics); - - for (MetricSnapshotSet set : snapshots) { - set.recreateSnapshot(activeMetrics.getMetrics(), snapshotUnsetMetrics); - } - } - - abstract class UpdateHook { - - String name; - int nextCall; - int period; - - public UpdateHook(String name) { - this.name = name; - nextCall = 0; - period = 0; - } - - public abstract void updateMetrics(); - - public String getName() { - return name; - } - } -} diff --git a/metrics/src/main/java/com/yahoo/metrics/MetricSet.java b/metrics/src/main/java/com/yahoo/metrics/MetricSet.java deleted file mode 100644 index 8f3039bff18..00000000000 --- a/metrics/src/main/java/com/yahoo/metrics/MetricSet.java +++ /dev/null @@ -1,244 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -import com.yahoo.text.XMLWriter; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.logging.Logger; - -public abstract class MetricSet extends Metric { - - private static Logger log = Logger.getLogger(MetricSet.class.getName()); - - List<Metric> metricOrder = new ArrayList<>(); // Keep added order for reporting - boolean registrationAltered; // Set to true if metrics have been - // registered/unregistered since last time - // it was reset - - public MetricSet(String name, String tags, String description, MetricSet owner) { - super(name, tags, description, owner); - } - - public MetricSet(MetricSet other, CopyType copyType, MetricSet owner, boolean includeUnused) { - super(other, owner); - - for (Metric m : other.metricOrder) { - if (copyType != CopyType.INACTIVE || includeUnused || m.used()) { - m.clone(copyType, this, includeUnused); - } - } - } - - /** - * @return Returns true if registration has been altered since it was last - * cleared. Used by the metric manager to know when it needs to recalculate - * which consumers will see what. - */ - public boolean isRegistrationAltered() { return registrationAltered; } - - /** Clear all registration altered flags. */ - void clearRegistrationAltered() { - visit(new MetricVisitor() { - public boolean visitMetricSet(MetricSet set, boolean autoGenerated) { - if (autoGenerated) { - return false; - } - - set.registrationAltered = false; - return true; - } - }, false); - } - - public void registerMetric(Metric m) { - if (m.isRegistered()) { - throw new IllegalStateException("Metric " + m.getName() + - " is already registered in a metric set. Cannot register it twice."); - } - - if (getMetricInternal(m.getName()) != null) { - throw new IllegalStateException("A metric named " + m.getName() + " is already registered " - + "in metric set " + getPath()); - } - - metricOrder.add(m); - m.owner = this; - tagRegistrationAltered(); - } - - public void unregisterMetric(Metric m) { - // In case of abrubt shutdowns, don't die hard on attempts to unregister - // non-registered metrics. Just warn and ignore. - if (!metricOrder.remove(m)) { - log.warning("Attempt to unregister metric " + m.getName() + " in metric set " + getPath() + - ", where it wasn't registered to begin with."); - return; - } - - m.owner = null; - tagRegistrationAltered(); - - log.finest("Unregistered metric " + m.getName() + " from metric set " + getPath() + "."); - } - - @Override - public void reset() { - for (Metric m : metricOrder) { - m.reset(); - } - } - - @Override - public boolean visit(MetricVisitor visitor, boolean tagAsAutoGenerated) { - if (!visitor.visitMetricSet(this, tagAsAutoGenerated)) { - return true; - } - - for (Metric m : metricOrder) { - if (!m.visit(visitor, tagAsAutoGenerated)) { - break; - } - } - - visitor.doneVisitingMetricSet(this); - return true; - } - - @Override - public void logEvent(EventLogger logger, String fullName) { - throw new IllegalStateException("Can't log event from a MetricsSet: " + fullName); - } - - // These should never be called on metrics set. - @Override - public long getLongValue(String id) { - throw new IllegalStateException("Tried to get long from metricset"); - } - - @Override - public double getDoubleValue(String id) { - throw new IllegalStateException("Tried to get double from metricset"); - } - - public Metric getMetric(String name) { - int pos = name.indexOf('.'); - if (pos == -1) { - return getMetricInternal(name); - } else { - String child = name.substring(0, pos); - String rest = name.substring(pos + 1); - - Metric m = getMetricInternal(child); - if (m == null || !(m instanceof MetricSet)) { - return null; - } else { - return ((MetricSet)m).getMetric(rest); - } - } - } - - private Metric getMetricInternal(String name) { - for (Metric m : metricOrder) { - if (m.getName().equals(name)) { - return m; - } - } - return null; - } - - Map<String, Metric> createMetricMap() { - Map<String, Metric> map = new TreeMap<>(); - - for (Metric m : metricOrder) { - map.put(m.getName(), m); - } - - return map; - } - - @Override - public void addToSnapshot(Metric snapshotMetric) { - MetricSet o = (MetricSet)snapshotMetric; - - Map<String, Metric> map1 = createMetricMap(); - Set<String> seen = new HashSet<>(); - - // For all the metrics in the other's order, join ours to the snapshot. - for (Metric m : o.metricOrder) { - Metric myCopy = map1.get(m.getName()); - - if (myCopy != null) { - seen.add(m.getName()); - myCopy.addToSnapshot(m); - } - } - - // For all the remaining metrics, just join them to the other one. - for (Metric m : metricOrder) { - if (!seen.contains(m.getName())) { - m.clone(CopyType.INACTIVE, o, false); - } - } - } - - List<Metric> getRegisteredMetrics() - { return metricOrder; } - - @Override - public boolean used() { - for (Metric m : metricOrder) { - if (m.used()) { - return true; - } - } - - return false; - } - - @Override - public void addToPart(Metric partMetric) { - MetricSet o = (MetricSet)partMetric; - - Map<String, Metric> map2 = o.createMetricMap(); - - for (Metric m : metricOrder) { - Metric other = map2.get(m.getName()); - if (other != null) { - m.addToPart(other); - } else { - m.clone(CopyType.INACTIVE, o, false); - } - } - } - - private void tagRegistrationAltered() { - registrationAltered = true; - if (owner != null) { - owner.tagRegistrationAltered(); - } - } - - @Override - public void printXml(XMLWriter writer, - int secondsPassed, int verbosity) - { - if (!used() && verbosity < 3) { - return; - } - - openXMLTag(writer, verbosity); - - for (Metric m : metricOrder) { - m.printXml(writer, secondsPassed, verbosity); - } - - writer.closeTag(); - } - - -} diff --git a/metrics/src/main/java/com/yahoo/metrics/MetricSnapshot.java b/metrics/src/main/java/com/yahoo/metrics/MetricSnapshot.java deleted file mode 100644 index b64330ddbc7..00000000000 --- a/metrics/src/main/java/com/yahoo/metrics/MetricSnapshot.java +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -import com.yahoo.text.XMLWriter; -import com.yahoo.text.Utf8String; - -import java.io.StringWriter; - -public class MetricSnapshot -{ - String name; - int period; - int fromTime; - int toTime; - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public int getPeriod() { - return period; - } - - public void setPeriod(int period) { - this.period = period; - } - - public int getFromTime() { - return fromTime; - } - - public void setFromTime(int fromTime) { - this.fromTime = fromTime; - } - - public int getToTime() { - return toTime; - } - - public void setToTime(int toTime) { - this.toTime = toTime; - } - - public MetricSet getMetrics() { - return snapshot; - } - - MetricSet snapshot; - - public MetricSnapshot(String name) { - this.name = name; - this.period = 0; - this.fromTime = 0; - this.toTime = 0; - snapshot = new SimpleMetricSet("metrics", "", ""); - } - - public MetricSnapshot(String name, - int period, - MetricSet source, - boolean copyUnset) { - this(name); - this.period = period; - snapshot = (MetricSet)source.clone(Metric.CopyType.INACTIVE, null, copyUnset); - } - - void reset(int currentTime) { - fromTime = currentTime; - toTime = 0; - snapshot.reset(); - } - - public void recreateSnapshot(MetricSet source, boolean copyUnset) { - MetricSet newSnapshot = (MetricSet)source.clone(Metric.CopyType.INACTIVE, null, copyUnset); - newSnapshot.reset(); - snapshot.addToSnapshot(newSnapshot); - snapshot = newSnapshot; - } - - public static final Utf8String TAG_NAME = new Utf8String("name"); - public static final Utf8String TAG_FROM = new Utf8String("from"); - public static final Utf8String TAG_TO = new Utf8String("to"); - public static final Utf8String TAG_PERIOD = new Utf8String("period"); - - public void printXml(MetricManager man, String consumer, int verbosity, XMLWriter writer) { - writer.openTag("snapshot"); - writer.attribute(TAG_NAME, name); - writer.attribute(TAG_FROM, fromTime); - writer.attribute(TAG_TO, toTime); - writer.attribute(TAG_PERIOD, period); - - for (Metric m : snapshot.getRegisteredMetrics()) { - if (m instanceof MetricSet) { - man.printXml((MetricSet)m, writer, toTime > fromTime ? (toTime - fromTime) : period, consumer, verbosity); - } - } - - writer.closeTag(); - } - - public String toXml(MetricManager man, String consumer, int verbosity) { - StringWriter str = new StringWriter(); - XMLWriter writer = new XMLWriter(str); - printXml(man, consumer, verbosity, writer); - return str.toString(); - } - - public void addToSnapshot(MetricSnapshot other, int currentTime, boolean reset) { - snapshot.addToSnapshot(other.getMetrics()); - if (reset) { - reset(currentTime); - } - other.toTime = currentTime; - } -} diff --git a/metrics/src/main/java/com/yahoo/metrics/MetricSnapshotSet.java b/metrics/src/main/java/com/yahoo/metrics/MetricSnapshotSet.java deleted file mode 100644 index 2cb07cf7dbe..00000000000 --- a/metrics/src/main/java/com/yahoo/metrics/MetricSnapshotSet.java +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -/** - * Represents two snapshots for the same time period. - */ -public class MetricSnapshotSet { - int count; // Number of times we need to join to building period - // before we have a full time window. - int builderCount; // Number of times we've currently added to the - // building instance. - MetricSnapshot current = null; // The last full period - MetricSnapshot building = null; // The building period - - MetricSnapshotSet(String name, int period, int count, MetricSet source, boolean snapshotUnsetMetrics) { - this.count = count; - this.builderCount = 0; - current = new MetricSnapshot(name, period, source, snapshotUnsetMetrics); - current.reset(0); - if (count != 1) { - building = new MetricSnapshot(name, period, source, snapshotUnsetMetrics); - building.reset(0); - } - } - - MetricSnapshot getNextTarget() { - if (count == 1) { - return current; - } else { - return building; - } - } - - public boolean haveCompletedNewPeriod(int newFromTime) { - if (count == 1) { - current.setToTime(newFromTime); - return true; - } - building.setToTime(newFromTime); - - // If not time to roll yet, just return - if (++builderCount < count) return false; - // Building buffer done. Use that as current and reset current. - MetricSnapshot tmp = current; - current = building; - building = tmp; - building.setFromTime(newFromTime); - building.setToTime(0); - builderCount = 0; - return true; - } - - public boolean timeForAnotherSnapshot(int currentTime) { - int lastTime = getFromTime() + builderCount * getPeriod(); - return currentTime >= lastTime + getPeriod(); - } - - public void reset(int currentTime) { - if (count != 1) building.reset(currentTime); - current.reset(currentTime); - builderCount = 0; - } - - public void recreateSnapshot(MetricSet metrics, boolean copyUnset) { - if (count != 1) building.recreateSnapshot(metrics, copyUnset); - current.recreateSnapshot(metrics, copyUnset); - } - - public void setFromTime(int fromTime) - { - if (count != 1) { - building.setFromTime(fromTime); - } else { - current.setFromTime(fromTime); - } - } - - public int getPeriod() { - return current.getPeriod(); - } - - public int getFromTime() { - return current.getFromTime(); - } - - public int getCount() { - return count; - } - - public MetricSnapshot getSnapshot() { - return getSnapshot(false); - } - - public MetricSnapshot getSnapshot(boolean getBuilding) { - if (getBuilding) { - if (count == 1) { - throw new IllegalStateException("No temporary snapshot for set " + current.name); - } - return building; - } - - return current; - } - - public boolean hasTemporarySnapshot() { - return count > 1; - } - - public String getName() { - return current.getName(); - } - - public int getBuilderCount() { - return builderCount; - } -} diff --git a/metrics/src/main/java/com/yahoo/metrics/MetricVisitor.java b/metrics/src/main/java/com/yahoo/metrics/MetricVisitor.java deleted file mode 100644 index c26d28caf83..00000000000 --- a/metrics/src/main/java/com/yahoo/metrics/MetricVisitor.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -public abstract class MetricVisitor { - /** - * Visit a metric set. - * - * @param autoGenerated True for metric sets that are generated on the - * fly such as in sum metrics. - * @return True if you want to visit the content of this metric set. - */ - public boolean visitMetricSet(MetricSet set, boolean autoGenerated) { - return true; - } - - /** - * Callback visitors can use if they need to know the tree traversal of - * metric sets. This function is not called if visitMetricSet returned - * false. - */ - public void doneVisitingMetricSet(MetricSet set) { - } - - /** - * Visit a primitive metric within an accepted metric set. - * - * @return True if you want to continue visiting, false to abort. - */ - public boolean visitPrimitiveMetric(Metric m, boolean autoGenerated) { - return true; - } -} diff --git a/metrics/src/main/java/com/yahoo/metrics/SimpleMetricSet.java b/metrics/src/main/java/com/yahoo/metrics/SimpleMetricSet.java deleted file mode 100644 index c6671cc07d9..00000000000 --- a/metrics/src/main/java/com/yahoo/metrics/SimpleMetricSet.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -/** - * A final metric set. - */ -public final class SimpleMetricSet extends MetricSet { - - public SimpleMetricSet(String name, String tags, String description) { - this(name, tags, description, null); - } - - public SimpleMetricSet(String name, String tags, String description, MetricSet owner) { - super(name, tags, description, owner); - } - - public SimpleMetricSet(SimpleMetricSet other, Metric.CopyType copyType, MetricSet owner, boolean includeUnused) { - super(other, copyType, owner, includeUnused); - } - - @Override - public Metric clone(Metric.CopyType type, MetricSet owner, boolean includeUnused) - { return new SimpleMetricSet(this, type, owner, includeUnused); } - -} diff --git a/metrics/src/main/java/com/yahoo/metrics/SumMetric.java b/metrics/src/main/java/com/yahoo/metrics/SumMetric.java deleted file mode 100644 index 3607ea7939b..00000000000 --- a/metrics/src/main/java/com/yahoo/metrics/SumMetric.java +++ /dev/null @@ -1,238 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class metrics::CounterMetric - * @ingroup metrics - * - * @brief Counts a value that only moves upwards. - * - * NB! If you have a MetricSet subclass you want to create a sum for, use - * MetricSet itself as the template argument. Otherwise you'll need to override - * clone(...) in order to make it return the correct type for your - * implementation. - */ - -package com.yahoo.metrics; - -import com.yahoo.text.XMLWriter; - -import java.util.ArrayList; -import java.util.List; - -public class SumMetric extends Metric -{ - ArrayList<Metric> metricsToSum = new ArrayList<Metric>(); - - public SumMetric(String name, String tags, String description, MetricSet owner) { - super(name, tags, description, owner); - metricsToSum = new ArrayList<Metric>(); - } - - public SumMetric(SumMetric other, MetricSet owner) { - super(other, owner); - - - if (other.owner == null) { - throw new IllegalStateException( - "Cannot copy a sum metric not registered in a metric set, as " + - "we need to use parent to detect new metrics to point to."); - } - if (owner == null) { - throw new IllegalStateException( - "Cannot copy a sum metric directly. One needs to at least " + - "include metric set above it in order to include metrics " + - "summed."); - } - - metricsToSum.ensureCapacity(other.metricsToSum.size()); - List<String> parentPath = other.owner.getPathVector(); - - for (Metric metric : other.metricsToSum) { - List<String> addendPath = metric.getPathVector(); - MetricSet newAddendParent = owner; - - for (int i = parentPath.size(); i < addendPath.size() - 1; ++i) { - String path = addendPath.get(i); - Metric child = newAddendParent.getMetric(path); - if (child == null) { - throw new IllegalStateException( - "Metric " + path + " in metric set " - + newAddendParent.getPath() + " was expected to " + - "exist. This sounds like a bug."); - } - - newAddendParent = (MetricSet)child; - } - - String path = addendPath.get(addendPath.size() - 1); - Metric child = newAddendParent.getMetric(path); - if (child == null) { - throw new IllegalStateException( - "Metric " + path + " in metric set " - + newAddendParent.getPath() + " was expected to " + - "exist. This sounds like a bug."); - } - - metricsToSum.add(child); - } - } - - @Override - public boolean visit(MetricVisitor visitor, boolean tagAsAutoGenerated) { - if (metricsToSum.isEmpty()) return true; - - Metric sum = generateSum(); - - if (sum == null) { - return true; - } - - if (sum instanceof MetricSet) { - sum.visit(visitor, true); - return true; - } else { - return visitor.visitPrimitiveMetric(sum, true); - } - } - - @Override - public Metric clone(CopyType copyType, MetricSet owner, boolean includeUnused) { - if (copyType == CopyType.CLONE) { - return new SumMetric(this, owner); - } - - Metric sum = null; - for (Metric metric : metricsToSum) { - if (sum == null) { - sum = metric.clone(CopyType.INACTIVE, null, includeUnused); - sum.setName(getName()); - sum.setDescription(getDescription()); - sum.setTags(getTags()); - - if (owner != null) { - owner.registerMetric(sum); - } - } else { - metric.addToPart(sum); - } - } - - return sum; - } - - @Override - public void addToPart(Metric partMetric) { - Metric m = generateSum(); - if (m != null) { - m.addToPart(partMetric); - } - } - - @Override - public void addToSnapshot(Metric snapshotMetric) { - Metric m = generateSum(); - if (m != null) { - m.addToSnapshot(snapshotMetric); - } - } - - public void addMetricToSum(Metric metric) { - if (owner == null) { - throw new IllegalStateException( - "Sum metric needs to be registered in a parent metric set " + - "prior to adding metrics to sum."); - } - if (!metricsToSum.isEmpty() && !(metric.getClass().equals(metricsToSum.get(0).getClass()))) { - throw new IllegalStateException( - "All metrics in a metric set must be of the same type."); - } - - List<String> sumParentPath = owner.getPathVector(); - List<String> addedPath = metric.getPathVector(); - - boolean error = false; - if (addedPath.size() <= sumParentPath.size()) { - error = true; - } else for (int i=0; i<sumParentPath.size(); ++i) { - if (!sumParentPath.get(i).equals(addedPath.get(i))) { - error = true; - break; - } - } - if (error) { - throw new IllegalStateException( - "Metric added to sum is required to be a child of the sum's " + - "direct parent metric set. (Need not be a direct child) " + - "Metric set " + metric.getPath() + " is not a child of " + - owner.getPath()); - } - - ArrayList<Metric> metrics = new ArrayList<Metric>(metricsToSum.size() + 1); - for (int i = 0; i < metricsToSum.size(); ++i) { - metrics.add(metricsToSum.get(i)); - } - metrics.add(metric); - metricsToSum = metrics; - } - - public void removeMetricFromSum(Metric metric) { - metricsToSum.remove(metric); - } - - public Metric generateSum() { - Metric m = clone(CopyType.INACTIVE, null, true); - - if (m != null) { - m.owner = owner; - } - return m; - } - - @Override - public long getLongValue(String id) { - Metric s = generateSum(); - if (s == null) { - return 0; - } - return s.getLongValue(id); - } - - @Override - public double getDoubleValue(String id) { - Metric s = generateSum(); - if (s == null) { - return 0.0; - } - return generateSum().getDoubleValue(id); - } - - @Override - public void logEvent(EventLogger logger, String fullName) { - Metric s = generateSum(); - if (s != null) { - s.logEvent(logger, fullName); - } - } - - @Override - public void printXml(XMLWriter writer, int secondsPassed, int verbosity) { - Metric s = generateSum(); - if (s == null) { - openXMLTag(writer, verbosity); - writer.closeTag(); - return; - } - - generateSum().printXml(writer, secondsPassed, verbosity); - } - - @Override - public boolean used() { - for (Metric m : metricsToSum) { - if (m.used()) return true; - } - return false; - } - - @Override - public void reset() {} -} diff --git a/metrics/src/main/java/com/yahoo/metrics/Timer.java b/metrics/src/main/java/com/yahoo/metrics/Timer.java deleted file mode 100644 index 0e980b816b9..00000000000 --- a/metrics/src/main/java/com/yahoo/metrics/Timer.java +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -/** -* @author thomasg -*/ -class Timer { - int secs() { return (int)(milliSecs() / 1000); } - long milliSecs() { return System.currentTimeMillis(); } -} diff --git a/metrics/src/main/java/com/yahoo/metrics/ValueMetric.java b/metrics/src/main/java/com/yahoo/metrics/ValueMetric.java deleted file mode 100644 index d0bc8f39140..00000000000 --- a/metrics/src/main/java/com/yahoo/metrics/ValueMetric.java +++ /dev/null @@ -1,265 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -import com.yahoo.log.LogLevel; -import com.yahoo.metrics.util.HasCopy; -import com.yahoo.metrics.util.MetricValueKeeper; -import com.yahoo.metrics.util.SimpleMetricValueKeeper; -import com.yahoo.metrics.util.ThreadLocalDirectoryValueKeeper; -import com.yahoo.metrics.util.ValueType; -import com.yahoo.text.XMLWriter; -import com.yahoo.text.Utf8String; - -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.logging.Logger; - -/** - * A metric that represents the value of something. - */ -public class ValueMetric<N extends Number> - extends Metric -{ - private static Logger log = Logger.getLogger(ValueMetric.class.getName()); - private static AtomicBoolean hasWarnedOnNonFinite = new AtomicBoolean(false); - - public interface Value<N extends Number> extends ValueType, HasCopy<Value<N>> { - void add(N v); - void join(Value<N> v2, boolean createAverageOnJoin); - boolean overflow(Value<N> v2); - - int getCount(); - N getMin(); - N getMax(); - N getLast(); - N getTotal(); - Double getAverage(); - - String valueToString(N value); - } - - public boolean hasFlag(int flag) { return (flags & flag) != 0; } - public ValueMetric<N> setFlag(int flag) { flags |= flag; return this; } - - MetricValueKeeper<Value<N>> values; - int flags = 0; - - static int AVERAGE_METRIC = 1; - static int CREATE_AVERAGE_ON_JOIN = 2; - static int UNSET_ON_ZERO_VALUE = 4; - static int LOG_IF_UNSET = 8; - - boolean isAverageMetric() { return hasFlag(AVERAGE_METRIC); } - boolean doCreateAverageOnJoin() { return hasFlag(CREATE_AVERAGE_ON_JOIN); } - boolean isUnsetOnZeroValue() { return hasFlag(UNSET_ON_ZERO_VALUE); } - boolean doLogIfUnset() { return hasFlag(LOG_IF_UNSET); } - JoinBehavior getJoinBehavior() { - return hasFlag(CREATE_AVERAGE_ON_JOIN) ? JoinBehavior.AVERAGE_ON_JOIN - : JoinBehavior.SUM_ON_JOIN; - } - - public ValueMetric<N> averageMetric() { return setFlag(AVERAGE_METRIC); } - public ValueMetric<N> createAverageOnJoin() { return setFlag(CREATE_AVERAGE_ON_JOIN); } - public ValueMetric<N> unsetOnZeroValue() { return setFlag(UNSET_ON_ZERO_VALUE); } - public ValueMetric<N> logIfUnset() { return setFlag(LOG_IF_UNSET); } - - public ValueMetric(String name, String tags, String description, MetricSet owner) { - super(name, tags, description, owner); - //values = new MetricValueSet(); - values = new ThreadLocalDirectoryValueKeeper<>(); - } - - public ValueMetric(ValueMetric<N> other, CopyType copyType, MetricSet owner) { - super(other, owner); - if (copyType == CopyType.INACTIVE || other.values instanceof SimpleMetricValueKeeper) { - values = new SimpleMetricValueKeeper<>(); - values.set(other.values.get(getJoinBehavior())); - } else { - //values = new MetricValueSet((MetricValueSet) other.values, ((MetricValueSet) other.values).size()); - values = new ThreadLocalDirectoryValueKeeper<>(other.values); - } - this.flags = other.flags; - } - - private void logNonFiniteValueWarning() { - if (!hasWarnedOnNonFinite.getAndSet(true)) { - log.log(LogLevel.WARNING, - "Metric '" + getPath() + "' attempted updated with a value that is NaN or " + - "Infinity; update ignored! No further warnings will be printed for " + - "such updates on any metrics, but they can be observed with debug " + - "logging enabled on component " + log.getName()); - } else if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, - "Metric '" + getPath() + "' attempted updated with a value that is NaN or " + - "Infinity; update ignored!"); - } - } - - public void addValue(N v) { - if (v == null) throw new NullPointerException("Cannot have null value"); - if (v instanceof Long) { - LongValue lv = new LongValue(); - lv.add((Long) v); - values.add((Value<N>) lv); - } else { - Double d = (Double)v; - if (d.isNaN() || d.isInfinite()) { - logNonFiniteValueWarning(); - return; - } - DoubleValue dv = new DoubleValue(); - dv.add(d); - values.add((Value<N>) dv); - } - } - - private Value<N> getValues() { - return values.get(getJoinBehavior()); - } - - @Override - public void addToSnapshot(Metric snapshotMetric) { - Value<N> v = getValues(); - if (v != null) ((ValueMetric<N>)snapshotMetric).join(v, true); - } - - @Override - void addToPart(Metric m) { - Value<N> v = getValues(); - if (v != null) ((ValueMetric<N>) m).join(v, doCreateAverageOnJoin()); - } - - public void join(Value<N> v2, boolean createAverageOnJoin) { - Value<N> tmpVals = getValues(); - if (tmpVals == null) { - if (v2 instanceof LongValue) { - tmpVals = (Value<N>) new LongValue(); - } else { - tmpVals = (Value<N>) new DoubleValue(); - } - } - if (tmpVals.overflow(v2)) { - this.values.reset(); - log.fine("Metric " + getPath() + " overflowed, resetting it."); - return; - } - if (tmpVals.getCount() == 0) { - tmpVals = v2; - } else if (v2.getCount() == 0) { - // Do nothing - } else { - tmpVals.join(v2, createAverageOnJoin); - } - this.values.set(tmpVals); - } - - public static final Utf8String TAG_AVG = new Utf8String("average"); - public static final Utf8String TAG_LAST = new Utf8String("last"); - public static final Utf8String TAG_MIN = new Utf8String("min"); - public static final Utf8String TAG_MAX = new Utf8String("max"); - public static final Utf8String TAG_CNT = new Utf8String("count"); - public static final Utf8String TAG_TOT = new Utf8String("total"); - - @Override - public void printXml(XMLWriter writer, - int timePassed, - int verbosity) - { - Value<N> val = getValues(); - if (!inUse(val) && verbosity < 2) { - return; - } - if (val == null) val = (Value<N>) new LongValue(); - - openXMLTag(writer, verbosity); - writer.attribute(TAG_AVG, new DoubleValue().valueToString(val.getAverage())); - writer.attribute(TAG_LAST, val.valueToString(val.getLast())); - - if (val.getCount() > 0) { - writer.attribute(TAG_MIN, val.valueToString(val.getMin())); - writer.attribute(TAG_MAX, val.valueToString(val.getMax())); - } - writer.attribute(TAG_CNT, val.getCount()); - if (verbosity >= 2) { - writer.attribute(TAG_TOT, val.valueToString(val.getTotal())); - } - - writer.closeTag(); - } - - public Number getValue(String id) { - Value<N> val = getValues(); - if (val == null) return 0; - - if (id.equals("last") || (!isAverageMetric() && id.equals("value"))) { - return val.getLast(); - } else if (id.equals("average") || (isAverageMetric() && id.equals("value"))) { - return val.getAverage(); - } else if (id.equals("count")) { - return val.getCount(); - } else if (id.equals("total")) { - return val.getTotal(); - } else if (id.equals("min")) { - return val.getMin(); - } else if (id.equals("max")) { - return val.getMax(); - } else { - throw new IllegalArgumentException("No id " + id + " in value metric " + getName()); - } - } - - @Override - public long getLongValue(String id) { - return getValue(id).longValue(); - } - - @Override - public double getDoubleValue(String id) { - return getValue(id).doubleValue(); - } - - @Override - public ValueMetric<N> clone(CopyType type, MetricSet owner, boolean includeUnused) { - return new ValueMetric<>(this, type, owner); - } - - @Override - public void reset() { values.reset(); } - - @Override - public void logEvent(EventLogger logger, String fullName) { - Value<N> val = getValues(); - if (!doLogIfUnset() && !inUse(val)) { - return; - } - logger.value(fullName, val == null ? 0 - : isAverageMetric() ? val.getAverage() - : val.getLast().doubleValue()); - } - - public boolean inUse(Value<?> value) { - return (value != null - && (value.getTotal().longValue() != 0 - || (value.getCount() != 0 && !isUnsetOnZeroValue()))); - } - - @Override - public boolean used() { - return inUse(getValues()); - } - - @Override - public String toString() { - Value<N> tmpVals = getValues(); - if (tmpVals == null) { - tmpVals = (Value<N>) new LongValue(); - } - return ("count=\"" + tmpVals.getCount() + - "\" min=\"" + tmpVals.valueToString(tmpVals.getMin()) + - "\" max=\"" + tmpVals.valueToString(tmpVals.getMax()) + - "\" last=\""+ tmpVals.valueToString(tmpVals.getLast()) + - "\" total=\"" + tmpVals.valueToString(tmpVals.getTotal()) + - "\" average=\"" + new DoubleValue().valueToString(tmpVals.getAverage()) + - "\""); - } - -} diff --git a/metrics/src/main/java/com/yahoo/metrics/VespaLogEventLogger.java b/metrics/src/main/java/com/yahoo/metrics/VespaLogEventLogger.java deleted file mode 100644 index a2ee2133b13..00000000000 --- a/metrics/src/main/java/com/yahoo/metrics/VespaLogEventLogger.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -import com.yahoo.log.event.Event; - -/** - * @author thomasg - */ -public class VespaLogEventLogger implements EventLogger { - public void value(String name, double value) { - Event.value(name, value); - } - - public void count(String name, long value) { - Event.count(name, value); - } -} diff --git a/metrics/src/main/java/com/yahoo/metrics/util/HasCopy.java b/metrics/src/main/java/com/yahoo/metrics/util/HasCopy.java deleted file mode 100644 index c4b53b24adc..00000000000 --- a/metrics/src/main/java/com/yahoo/metrics/util/HasCopy.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics.util; - -public interface HasCopy<T> { - public T copyObject(); -} - diff --git a/metrics/src/main/java/com/yahoo/metrics/util/MetricValueKeeper.java b/metrics/src/main/java/com/yahoo/metrics/util/MetricValueKeeper.java deleted file mode 100644 index c40b57c29fb..00000000000 --- a/metrics/src/main/java/com/yahoo/metrics/util/MetricValueKeeper.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics.util; - -import com.yahoo.metrics.JoinBehavior; -import java.util.Collection; - -/** - * - */ -public interface MetricValueKeeper<Value> { - /** Add a value to a given value. Used when metrics are written to by metric writer. */ - public void add(Value v); - /** Set the value to exactly this, forgetting old value. This operation needs only be supported by value keepers used in inactive snapshots. */ - public void set(Value v); - /** Get the value of the metrics written. If multiple values exist, use given join behavior. */ - public Value get(JoinBehavior joinBehavior); - /** Reset the value of the metrics to zero. */ - public void reset(); - /** Return copy of current view */ - public Collection<Value> getDirectoryView(); -} diff --git a/metrics/src/main/java/com/yahoo/metrics/util/MetricValueSet.java b/metrics/src/main/java/com/yahoo/metrics/util/MetricValueSet.java deleted file mode 100644 index 0ba3138f3d7..00000000000 --- a/metrics/src/main/java/com/yahoo/metrics/util/MetricValueSet.java +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class MetricValueSet - * \ingroup metrics - * - * \brief Utility for doing lockless metric updates and reads. - * - * We don't want to use regular locking while updating metrics due to overhead. - * We use this class to make metric updates as safe as possible without - * requiring locks. - * - * It keeps the set of values a metric wants to set. Thus it is templated on - * the class keeping the values. All that is required of this class is that it - * has an empty constructor and a copy constructor. - * - * The locking works, by keeping a set of values, with an active pointer into - * the value vector. Assuming only one thread calls setValues(), it can update - * the active pointer safely. We assume updating the active pointer is a - * non-interruptable operations, such that other threads will see either the new - * or the old value correctly. This should be the case on our platforms. - * - * Due to the reset functionality, it is possible to miss out on a metrics - * added during the reset, but this is very unlikely. For that to happen, when - * someone sets the reset flag, the writer thread must be in setValues(), - * having already passed the check for the reset flag, but not finished setting - * the values yet. - */ - -package com.yahoo.metrics.util; - -import java.util.Collection; -import com.yahoo.metrics.JoinBehavior; - -public class MetricValueSet<Value extends HasCopy<Value> & ValueType> - implements MetricValueKeeper<Value> -{ - private final ValueType[] values; - - private void setIndexedValue(int idx, Value v) { - values[idx] = v; - } - - @SuppressWarnings("unchecked") - private Value getIndexedValue(int idx) { - return (Value) values[idx]; - } - - - private volatile int activeValueIndex; - private volatile boolean reset = false; - - public MetricValueSet() { - values = new ValueType[3]; - activeValueIndex = 0; - } - - public MetricValueSet(MetricValueSet<Value> other, int copyCount) { - values = new ValueType[copyCount]; - activeValueIndex = 0; - setValue(other.getValue()); - } - - /** Get the current values. */ - public Value getValue() { - if (reset) return null; - Value v = getIndexedValue(activeValueIndex); - return (v == null ? null : v.copyObject()); - } - - /** - * Get the current values from the metric. This function should not be - * called in parallel. Only call it from a single thread or use external - * locking. If it returns false, it means the metric have just been reset. - * In which case, redo getValues(), apply the update again, and call - * setValues() again. - */ - public boolean setValue(Value value) { - int nextIndex = (activeValueIndex + 1) % values.length; - if (reset) { - reset = false; - setIndexedValue(nextIndex, null); - activeValueIndex = nextIndex; - return false; - } else { - setIndexedValue(nextIndex, value); - activeValueIndex = nextIndex; - return true; - } - } - - /** - * Retrieve and reset in a single operation, to minimize chance of - * alteration in the process. - */ - public Value getValuesAndReset() { - Value result = getValue(); - reset = true; - return result; - } - - public String toString() { - String retVal = "MetricValueSet(reset=" + reset - + ", active " + activeValueIndex + "["; - for (ValueType n : values) retVal += n + ","; - retVal += "])"; - return retVal; - } - - public int size() { return values.length; } - - public void add(Value v) { - while (true) { - Value current = getValue(); - if (current != null) { - current.add(v); - if (setValue(current)) { - return; - } - } else { - if (setValue(v)) { - return; - } - } - } - } - - public void set(Value v) { - setValue(v); - } - - public Value get(JoinBehavior joinBehavior) { - return getValue(); - } - - public void reset() { reset = true; } - - public Collection<Value> getDirectoryView() { - return null; - } - -} diff --git a/metrics/src/main/java/com/yahoo/metrics/util/SimpleMetricValueKeeper.java b/metrics/src/main/java/com/yahoo/metrics/util/SimpleMetricValueKeeper.java deleted file mode 100644 index bf5b18f782e..00000000000 --- a/metrics/src/main/java/com/yahoo/metrics/util/SimpleMetricValueKeeper.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics.util; - -import java.util.Collection; -import com.yahoo.metrics.JoinBehavior; - -/** - * Simple implementation of the metric value keeper for use with snapshots. Only keeps - * one instance of data, does not need to worry about threads, and should have a small - * memory footprint. - */ -public class SimpleMetricValueKeeper<Value> implements MetricValueKeeper<Value> { - private Value value; - - public SimpleMetricValueKeeper() { - } - - public void add(Value v) { - throw new UnsupportedOperationException("Not supported. This value keeper is not intended to be used for live metrics."); - } - - public void set(Value v) { - this.value = v; - } - - public Value get(JoinBehavior joinBehavior) { - return value; - } - - public void reset() { - value = null; - } - - public Collection<Value> getDirectoryView() { - return null; - } - -} diff --git a/metrics/src/main/java/com/yahoo/metrics/util/ThreadLocalDirectoryValueKeeper.java b/metrics/src/main/java/com/yahoo/metrics/util/ThreadLocalDirectoryValueKeeper.java deleted file mode 100644 index a6ef28eac0f..00000000000 --- a/metrics/src/main/java/com/yahoo/metrics/util/ThreadLocalDirectoryValueKeeper.java +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics.util; - -import com.yahoo.concurrent.ThreadLocalDirectory; -import com.yahoo.metrics.DoubleValue; -import com.yahoo.metrics.LongValue; -import com.yahoo.metrics.JoinBehavior; - -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; - -/** - * Value keeper class using ThreadLocalDirectory to maintain thread safety from multiple threads writing. - */ -public class ThreadLocalDirectoryValueKeeper<Value extends ValueType & HasCopy<Value>> - implements MetricValueKeeper<Value>, ThreadLocalDirectory.ObservableUpdater<Value, Value> -{ - private final ThreadLocalDirectory<Value, Value> directory = new ThreadLocalDirectory<Value, Value>(this); - private List<Value> initialValues = null; - - public ThreadLocalDirectoryValueKeeper() { - } - - public ThreadLocalDirectoryValueKeeper(MetricValueKeeper<Value> other) { - initialValues = new LinkedList<Value>(); - for (Value v : other.getDirectoryView()) initialValues.add(v.copyObject()); - } - - public Value createGenerationInstance(Value value) { - return null; - } - - public Value update(Value current, Value newValue) { - if (current == null) { - if (newValue instanceof DoubleValue) { - current = (Value) new DoubleValue(); - } else { - current = (Value) new LongValue(); - } - } - current.add(newValue); - return current; - } - - public Value copy(Value value) { - return value.copyObject(); - } - - public void add(Value v) { - directory.update(v, directory.getLocalInstance()); - } - - public void set(Value v) { - throw new UnsupportedOperationException("This value keeper is only intended to use with active metrics. Set operation not supported."); - } - - public Value get(JoinBehavior joinBehavior) { - List<Value> values = directory.view(); - if (initialValues != null) values.addAll(initialValues); - if (values.isEmpty()) { - return null; - } - return (Value) values.get(0).join(Collections.<ValueType>unmodifiableList(values), joinBehavior); - } - - public void reset() { - directory.fetch(); - initialValues = null; - } - - public Collection<Value> getDirectoryView() { - return directory.view(); - } -} diff --git a/metrics/src/main/java/com/yahoo/metrics/util/ValueType.java b/metrics/src/main/java/com/yahoo/metrics/util/ValueType.java deleted file mode 100644 index e9c8e27b613..00000000000 --- a/metrics/src/main/java/com/yahoo/metrics/util/ValueType.java +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics.util; - -import com.yahoo.metrics.JoinBehavior; - -import java.util.Collection; - -public interface ValueType extends Cloneable { - public void add(ValueType other); - public ValueType join(Collection<ValueType> sources, JoinBehavior joinBehavior); -} diff --git a/metrics/src/main/metrics-with-dependencies.xml b/metrics/src/main/metrics-with-dependencies.xml deleted file mode 100644 index 3b76662ebed..00000000000 --- a/metrics/src/main/metrics-with-dependencies.xml +++ /dev/null @@ -1,19 +0,0 @@ -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<assembly> - <id>jar-with-dependencies</id> - <formats> - <format>jar</format> - </formats> - <includeBaseDirectory>false</includeBaseDirectory> - <dependencySets> - <dependencySet> - <unpack>true</unpack> - <scope>runtime</scope> - </dependencySet> - </dependencySets> - <fileSets> - <fileSet> - <directory>${project.build.outputDirectory}</directory> - </fileSet> - </fileSets> -</assembly> diff --git a/metrics/src/test/java/com/yahoo/metrics/AveragedDoubleValueMetric.java b/metrics/src/test/java/com/yahoo/metrics/AveragedDoubleValueMetric.java deleted file mode 100644 index 41fe7f177dc..00000000000 --- a/metrics/src/test/java/com/yahoo/metrics/AveragedDoubleValueMetric.java +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -/** - * @author Simon Thoresen Hult - */ -class AveragedDoubleValueMetric extends ValueMetric<Double> { - - public AveragedDoubleValueMetric(String name, String tags, String description, MetricSet owner) { - super(name, tags, description, owner); - averageMetric(); - createAverageOnJoin(); - } - - public AveragedDoubleValueMetric(AveragedDoubleValueMetric other, CopyType copyType, MetricSet owner) { - super(other, copyType, owner); - } -} diff --git a/metrics/src/test/java/com/yahoo/metrics/AveragedLongValueMetric.java b/metrics/src/test/java/com/yahoo/metrics/AveragedLongValueMetric.java deleted file mode 100644 index 2f27720cfc1..00000000000 --- a/metrics/src/test/java/com/yahoo/metrics/AveragedLongValueMetric.java +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -/** - * @author Simon Thoresen Hult - */ -class AveragedLongValueMetric extends ValueMetric<Long> { - - public AveragedLongValueMetric(String name, String tags, String description, MetricSet owner) { - super(name, tags, description, owner); - averageMetric(); - createAverageOnJoin(); - } - - public AveragedLongValueMetric(AveragedLongValueMetric other, CopyType copyType, MetricSet owner) { - super(other, copyType, owner); - } -} diff --git a/metrics/src/test/java/com/yahoo/metrics/CountMetricTest.java b/metrics/src/test/java/com/yahoo/metrics/CountMetricTest.java deleted file mode 100644 index ef802a73c4c..00000000000 --- a/metrics/src/test/java/com/yahoo/metrics/CountMetricTest.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author thomasg - */ -public class CountMetricTest { - - @Test - public void testCountMetric() { - CountMetric m = new CountMetric("test", "tag", "description", null); - assertEquals(false, m.used()); - m.set(100); - assertEquals(true, m.used()); - assertEquals(100, m.getValue()); - m.inc(5); - assertEquals(105, m.getValue()); - m.dec(15); - assertEquals(90, m.getValue()); - - CountMetric m2 = new CountMetric(m, Metric.CopyType.CLONE, null); - m.reset(); - assertEquals(90, m2.getValue()); - assertEquals(0, m.getValue()); - - CountMetric n = new CountMetric("m2", "", "desc", null); - n.set(6); - - n.addToSnapshot(m2); - assertEquals(96, m2.getValue()); - n.addToPart(m); - assertEquals(6, m.getValue()); - - assertEquals(6, n.getValue()); - - assertEquals("<test description=\"description\" count=\"96\"/>\n", m2.toXml(0, 2)); - assertEquals("<test description=\"description\" count=\"96\" average_change_per_second=\"9.60\"/>\n", m2.toXml(10, 2)); - assertEquals(96.0, m2.getDoubleValue("value"), 0.00000001); - assertEquals(96, m2.getLongValue("value")); - } - -} diff --git a/metrics/src/test/java/com/yahoo/metrics/DummyTimer.java b/metrics/src/test/java/com/yahoo/metrics/DummyTimer.java deleted file mode 100644 index 7cba2f6d42c..00000000000 --- a/metrics/src/test/java/com/yahoo/metrics/DummyTimer.java +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -/** - * @author thomasg - */ -public class DummyTimer extends Timer { - long ms = 0; - - public long milliSecs() { return ms; } -} diff --git a/metrics/src/test/java/com/yahoo/metrics/MetricManagerTest.java b/metrics/src/test/java/com/yahoo/metrics/MetricManagerTest.java deleted file mode 100644 index 6ee4f5b55c2..00000000000 --- a/metrics/src/test/java/com/yahoo/metrics/MetricManagerTest.java +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -import org.junit.Test; - -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.*; - -/** - * @author thomasg - */ -public class MetricManagerTest { - - @Test - public void requireThatManagerCanBeStopped() throws InterruptedException { - final MetricManager manager = new MetricManager(new DummyTimer()); - Thread managerThread = new Thread(manager); - managerThread.start(); - Thread stopThread = new Thread(new Runnable() { - - @Override - public void run() { - manager.stop(); - } - }); - stopThread.start(); - stopThread.join(TimeUnit.SECONDS.toMillis(60)); - assertFalse(stopThread.isAlive()); - managerThread.join(TimeUnit.SECONDS.toMillis(60)); - assertFalse(managerThread.isAlive()); - } - - @Test - public void requireThatManagerCanBeStoppedBeforeStarting() throws InterruptedException { - final MetricManager manager = new MetricManager(new DummyTimer()); - Thread thread = new Thread(new Runnable() { - - @Override - public void run() { - manager.stop(); - } - }); - thread.start(); - thread.join(TimeUnit.SECONDS.toMillis(60)); - assertFalse(thread.isAlive()); - } - - @Test - public void requireThatManagerCanNotBeStartedAfterItHasBeenStopped() { - final MetricManager manager = new MetricManager(new DummyTimer()); - manager.stop(); - try { - manager.run(); - fail(); - } catch (IllegalStateException e) { - - } - } - - @Test - public void testSimpleLogging() { - DummyTimer timer = new DummyTimer(); - MetricManager manager = new MetricManager(timer); - SimpleMetricSet sms = new SimpleMetricSet("foo", "", "", null); - manager.registerMetric(sms); - - CountMetric bar = new CountMetric("bar", "", "", sms); - bar.inc(3); - - manager.addMetricToConsumer("log", "foo.bar"); - - DummyEventLogger logger = new DummyEventLogger(); - manager.tick(logger); - //assertEquals("CNT foo_bar: 0\n", logger.output); - - timer.ms = 300000; - - manager.tick(logger); - assertEquals("CNT foo_bar: 0\nCNT foo_bar: 3\n", logger.output.toString()); - } - - @Test - public void testSumLogging() { - DummyTimer timer = new DummyTimer(); - MetricManager manager = new MetricManager(timer); - SimpleMetricSet sms = new SimpleMetricSet("foo", "", "", null); - manager.registerMetric(sms); - - CountMetric v1 = new CountMetric("v1", "", "", sms); - CountMetric v2 = new CountMetric("v2", "", "", sms); - CountMetric v3 = new CountMetric("v3", "", "", sms); - SumMetric sum = new SumMetric("sum", "", "", sms); - - sum.addMetricToSum(v1); - sum.addMetricToSum(v2); - sum.addMetricToSum(v3); - - v1.inc(3); - v2.inc(1); - v3.inc(2); - - manager.addMetricToConsumer("log", "foo.sum"); - - DummyEventLogger logger = new DummyEventLogger(); - manager.tick(logger); - assertEquals("CNT foo_sum: 0\n", logger.output.toString()); - - timer.ms = 300000; - - manager.tick(logger); - assertEquals("CNT foo_sum: 0\nCNT foo_sum: 6\n", logger.output.toString()); - } - - @Test - public void testSumAndSetLogging() { - DummyTimer timer = new DummyTimer(); - timer.ms = 1000000; - MetricManager manager = new MetricManager(timer); - SimpleMetricSet sms = new SimpleMetricSet("foo", "", "", null); - manager.registerMetric(sms); - - SimpleMetricSet v1 = new SimpleMetricSet("v1", "", "", sms); - SimpleMetricSet v2 = new SimpleMetricSet("v2", "", "", sms); - - CountMetric v1_a = new CountMetric("a", "", "", v1); - SummedLongValueMetric v1_b = new SummedLongValueMetric("b", "", "", v1); - AveragedLongValueMetric v1_c = new AveragedLongValueMetric("c", "", "", v1); - - CountMetric v2_a = new CountMetric("a", "", "", v2); - SummedLongValueMetric v2_b = new SummedLongValueMetric("b", "", "", v2); - AveragedLongValueMetric v2_c = new AveragedLongValueMetric("c", "", "", v2); - - SumMetric sum = new SumMetric("sum", "", "", sms); - - sum.addMetricToSum(v1); - sum.addMetricToSum(v2); - - v1_a.inc(3); - v1_b.addValue((long)1); - v1_c.addValue((long)4); - v2_a.inc(2); - v2_b.addValue((long)2); - v2_c.addValue((long)8); - - manager.addMetricToConsumer("log", "foo.sum.a"); - manager.addMetricToConsumer("log", "foo.sum.b"); - manager.addMetricToConsumer("log", "foo.sum.c"); - - { - DummyEventLogger logger = new DummyEventLogger(); - assertEquals(300000, manager.tick(logger)); - assertEquals("CNT foo_sum_a: 0\n", logger.output.toString()); - } - - timer.ms = 1300000; - - { - DummyEventLogger logger = new DummyEventLogger(); - assertEquals(300000, manager.tick(logger)); - assertEquals("CNT foo_sum_a: 5\nVAL foo_sum_b: 3.0\nVAL foo_sum_c: 6.0\n", logger.output.toString()); - } - - v1_a.inc(4); - v1_b.addValue((long)2); - v1_b.addValue((long)6); - v2_b.addValue((long)4); - - timer.ms = 1600000; - - { - DummyEventLogger logger = new DummyEventLogger(); - assertEquals(300000, manager.tick(logger)); - assertEquals("CNT foo_sum_a: 9\nVAL foo_sum_b: 10.0\n", logger.output.toString()); - } - } - - private static class DummyEventLogger implements EventLogger { - - StringBuilder output = new StringBuilder(); - - @Override - public void value(String name, double value) { - output.append("VAL " + name + ": " + value + "\n"); - } - - @Override - public void count(String name, long value) { - output.append("CNT " + name + ": " + value + "\n"); - } - } -} diff --git a/metrics/src/test/java/com/yahoo/metrics/MetricSetTest.java b/metrics/src/test/java/com/yahoo/metrics/MetricSetTest.java deleted file mode 100644 index bfe10825f4f..00000000000 --- a/metrics/src/test/java/com/yahoo/metrics/MetricSetTest.java +++ /dev/null @@ -1,344 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - -public class MetricSetTest { - - private final double delta = 0.000000001; - - class TestMetricVisitor extends MetricVisitor { - String output = ""; - int setsToVisit; - - TestMetricVisitor(int setsToVisit) { - this.setsToVisit = setsToVisit; - } - - public boolean visitMetricSet(MetricSet set, boolean autoGenerated) { - output += "[" + (autoGenerated ? "*" : "") + set.getName() + "]\n"; - if (setsToVisit > 0) { - --setsToVisit; - return true; - } - return false; - } - public boolean visitPrimitiveMetric(Metric m, boolean autoGenerated) { - output += (autoGenerated ? "*" : "") + m.getName() + "\n"; - return true; - } - } - - @Test - public void testNormalUsage() { - // Set up some metrics to test.. - MetricSet set = new SimpleMetricSet("a", "foo", ""); - SummedDoubleValueMetric v1 = new SummedDoubleValueMetric("c", "foo", "", set); - AveragedLongValueMetric v2 = new AveragedLongValueMetric("b", "", "", set); - CountMetric v3 = new CountMetric("d", "bar", "", set); - MetricSet set2 = new SimpleMetricSet("e", "bar", "", set); - CountMetric v4 = new CountMetric("f", "foo", "", set2); - - // Give them some values - v1.addValue(4.2); - v2.addValue((long)8); - v3.inc(); - v4.inc(3); - - // Check that we can register through registerMetric function too. - CountMetric v5 = new CountMetric("g", "", "", null); - set.registerMetric(v5); - v5.inc(3); - v5.dec(); - - // Check that getMetric works, and doesn't return copy. - AveragedLongValueMetric v2copy = (AveragedLongValueMetric)set.getMetric("b"); - assertNotNull(v2copy); - v2copy.addValue((long)9); - - assertNull(set.getMetric("nonexisting")); - assertNull(set.getMetric("non.existing")); - - // Check that paths are set - MetricSet topSet = new SimpleMetricSet("top", "", "", null); - topSet.registerMetric(set); - - assertEquals("a", set.getPath()); - assertEquals("a.c", v1.getPath()); - assertEquals("a.b", v2.getPath()); - assertEquals("a.d", v3.getPath()); - assertEquals("a.e", set2.getPath()); - assertEquals("a.e.f", v4.getPath()); - assertEquals("a.g", v5.getPath()); - assertEquals("a.b", v2copy.getPath()); - - // Verify XML output. Should be in register order. - assertEquals( - "<a>\n\n" + - " <c average=\"4.20\" last=\"4.20\" min=\"4.20\" max=\"4.20\" count=\"1\" total=\"4.20\"/>\n\n" + - " <b average=\"8.50\" last=\"9\" min=\"8\" max=\"9\" count=\"2\" total=\"17\"/>\n\n" + - " <d count=\"1\"/>\n\n" + - " <e>\n"+ - " <f count=\"3\"/>\n" + - " </e>\n\n" + - " <g count=\"2\"/>\n\n"+ - "</a>\n", set.toXml(0,2)); - - // Verify that visiting works. That you get all metrics if you answer - // true to all sets, and that you don't get members of sets you answer - // false to get. - { - TestMetricVisitor visitor = new TestMetricVisitor(2); - set.visit(visitor, false); - assertEquals("[a]\nc\nb\nd\n[e]\nf\ng\n", visitor.output); - } - { - TestMetricVisitor visitor = new TestMetricVisitor(1); - set.visit(visitor, false); - assertEquals("[a]\nc\nb\nd\n[e]\ng\n", visitor.output); - } - { - TestMetricVisitor visitor = new TestMetricVisitor(0); - set.visit(visitor, false); - assertEquals("[a]\n", visitor.output); - } - } - - class MyTimer extends Timer { - long timeNow = 0; - - @Override - long milliSecs() { return timeNow; } - } - - @SuppressWarnings("rawtypes") - @Test - public void testSumInSet() { - MetricSet ms_a = new SimpleMetricSet("a", "", "", null); - MyTimer timer = new MyTimer(); - timer.timeNow = 1; - - MetricManager manager = new MetricManager(timer); - manager.registerMetric(ms_a); - - manager.takeSnapshots(1); - - SumMetric s_a = new SumMetric("sum", "", "", ms_a); - MetricSet ms_b = new SimpleMetricSet("b", "", "", ms_a); - s_a.addMetricToSum(ms_b); - MetricSet ms_c = new SimpleMetricSet("c", "", "", ms_a); - s_a.addMetricToSum(ms_c); - - SumMetric s_b_a = new SumMetric("sum", "", "", ms_b); - SumMetric s_c_a = new SumMetric("sum", "", "", ms_c); - - AveragedDoubleValueMetric v1_b = new AveragedDoubleValueMetric("v1", "", "", ms_b); - s_b_a.addMetricToSum(v1_b); - AveragedDoubleValueMetric v3_b = new AveragedDoubleValueMetric("v3", "", "", ms_b); - s_b_a.addMetricToSum(v3_b); - - AveragedDoubleValueMetric v1_c = new AveragedDoubleValueMetric("v1", "", "", ms_c); - s_c_a.addMetricToSum(v1_c); - AveragedDoubleValueMetric v2_c = new AveragedDoubleValueMetric("v2", "", "", ms_c); - s_c_a.addMetricToSum(v2_c); - - v1_b.addValue(1.0); - v3_b.addValue(2.0); - v1_c.addValue(3.0); - v2_c.addValue(4.0); - - SimpleMetricSet sms = (SimpleMetricSet)s_a.generateSum(); - ValueMetric av_sum = (ValueMetric)sms.getMetric("sum"); - ValueMetric av_1 = (ValueMetric)sms.getMetric("v1"); - ValueMetric av_2 = (ValueMetric)sms.getMetric("v2"); - ValueMetric av_3 = (ValueMetric)sms.getMetric("v3"); - - assertEquals(10.0, av_sum.getDoubleValue("total"), delta); - assertEquals(4, av_sum.getLongValue("count"), delta); - assertEquals(4.0, av_1.getDoubleValue("total"), delta); - assertEquals(4.0, av_2.getDoubleValue("total"), delta); - assertEquals(2.0, av_3.getDoubleValue("total"), delta); - - timer.timeNow = 301; - manager.takeSnapshots(301); - - MetricSnapshot snapshot = manager.getMetricSnapshot(300); - - assertEquals("<snapshot name=\"5 minute\" from=\"301\" to=\"301\" period=\"300\">\n" + - "\n" + - " <a>\n" + - " <sum>\n" + - " <sum average=\"2.50\" last=\"4.00\" min=\"1.00\" max=\"4.00\" count=\"4\" total=\"10.00\"/>\n" + - " <v1 average=\"2.00\" last=\"3.00\" min=\"1.00\" max=\"3.00\" count=\"2\" total=\"4.00\"/>\n" + - " <v3 average=\"2.00\" last=\"2.00\" min=\"2.00\" max=\"2.00\" count=\"1\" total=\"2.00\"/>\n" + - " <v2 average=\"4.00\" last=\"4.00\" min=\"4.00\" max=\"4.00\" count=\"1\" total=\"4.00\"/>\n" + - " </sum>\n" + - " <b>\n" + - " <sum average=\"1.50\" last=\"2.00\" min=\"1.00\" max=\"2.00\" count=\"2\" total=\"3.00\"/>\n" + - " <v1 average=\"1.00\" last=\"1.00\" min=\"1.00\" max=\"1.00\" count=\"1\" total=\"1.00\"/>\n" + - " <v3 average=\"2.00\" last=\"2.00\" min=\"2.00\" max=\"2.00\" count=\"1\" total=\"2.00\"/>\n" + - " </b>\n" + - " <c>\n" + - " <sum average=\"3.50\" last=\"4.00\" min=\"3.00\" max=\"4.00\" count=\"2\" total=\"7.00\"/>\n" + - " <v1 average=\"3.00\" last=\"3.00\" min=\"3.00\" max=\"3.00\" count=\"1\" total=\"3.00\"/>\n" + - " <v2 average=\"4.00\" last=\"4.00\" min=\"4.00\" max=\"4.00\" count=\"1\" total=\"4.00\"/>\n" + - " </c>\n" + - " </a>\n" + - "\n" + - "</snapshot>\n", snapshot.toXml(manager, "", 2)); - - assertEquals("<snapshot name=\"1 hour\" from=\"0\" to=\"301\" period=\"3600\">\n" + - "\n" + - " <a>\n" + - " <sum>\n" + - " <sum average=\"2.50\" last=\"4.00\" min=\"1.00\" max=\"4.00\" count=\"4\" total=\"10.00\"/>\n" + - " <v1 average=\"2.00\" last=\"3.00\" min=\"1.00\" max=\"3.00\" count=\"2\" total=\"4.00\"/>\n" + - " <v3 average=\"2.00\" last=\"2.00\" min=\"2.00\" max=\"2.00\" count=\"1\" total=\"2.00\"/>\n" + - " <v2 average=\"4.00\" last=\"4.00\" min=\"4.00\" max=\"4.00\" count=\"1\" total=\"4.00\"/>\n" + - " </sum>\n" + - " <b>\n" + - " <sum average=\"1.50\" last=\"2.00\" min=\"1.00\" max=\"2.00\" count=\"2\" total=\"3.00\"/>\n" + - " <v1 average=\"1.00\" last=\"1.00\" min=\"1.00\" max=\"1.00\" count=\"1\" total=\"1.00\"/>\n" + - " <v3 average=\"2.00\" last=\"2.00\" min=\"2.00\" max=\"2.00\" count=\"1\" total=\"2.00\"/>\n" + - " </b>\n" + - " <c>\n" + - " <sum average=\"3.50\" last=\"4.00\" min=\"3.00\" max=\"4.00\" count=\"2\" total=\"7.00\"/>\n" + - " <v1 average=\"3.00\" last=\"3.00\" min=\"3.00\" max=\"3.00\" count=\"1\" total=\"3.00\"/>\n" + - " <v2 average=\"4.00\" last=\"4.00\" min=\"4.00\" max=\"4.00\" count=\"1\" total=\"4.00\"/>\n" + - " </c>\n" + - " </a>\n" + - "\n" + - "</snapshot>\n", manager.getMetricSnapshot(3600, true).toXml(manager, "", 2)); - - v1_b.addValue(2.0); - v3_b.addValue(3.0); - v1_c.addValue(4.0); - v2_c.addValue(8.0); - - CountMetric c1_b = new CountMetric("cnt", "", "", ms_b); - c1_b.inc(500); - - timer.timeNow = 601; - manager.takeSnapshots(601); - - // THIS IS WRONG. See ticket - assertEquals("<snapshot name=\"5 minute\" from=\"601\" to=\"601\" period=\"300\">\n" + - "\n" + - " <a>\n" + - " <sum>\n" + - " <sum average=\"4.25\" last=\"8.00\" min=\"2.00\" max=\"8.00\" count=\"4\" total=\"17.00\"/>\n" + - " <v1 average=\"3.00\" last=\"4.00\" min=\"2.00\" max=\"4.00\" count=\"2\" total=\"6.00\"/>\n" + - " <v3 average=\"3.00\" last=\"3.00\" min=\"3.00\" max=\"3.00\" count=\"1\" total=\"3.00\"/>\n" + - " <v2 average=\"8.00\" last=\"8.00\" min=\"8.00\" max=\"8.00\" count=\"1\" total=\"8.00\"/>\n" + - " <cnt count=\"500\" average_change_per_second=\"1.67\"/>\n" + - " </sum>\n" + - " <b>\n" + - " <sum average=\"2.50\" last=\"3.00\" min=\"2.00\" max=\"3.00\" count=\"2\" total=\"5.00\"/>\n" + - " <v1 average=\"2.00\" last=\"2.00\" min=\"2.00\" max=\"2.00\" count=\"1\" total=\"2.00\"/>\n" + - " <v3 average=\"3.00\" last=\"3.00\" min=\"3.00\" max=\"3.00\" count=\"1\" total=\"3.00\"/>\n" + - " <cnt count=\"500\" average_change_per_second=\"1.67\"/>\n" + - " </b>\n" + - " <c>\n" + - " <sum average=\"6.00\" last=\"8.00\" min=\"4.00\" max=\"8.00\" count=\"2\" total=\"12.00\"/>\n" + - " <v1 average=\"4.00\" last=\"4.00\" min=\"4.00\" max=\"4.00\" count=\"1\" total=\"4.00\"/>\n" + - " <v2 average=\"8.00\" last=\"8.00\" min=\"8.00\" max=\"8.00\" count=\"1\" total=\"8.00\"/>\n" + - " </c>\n" + - " </a>\n" + - "\n" + - "</snapshot>\n", manager.getMetricSnapshot(300, false).toXml(manager, "", 2)); - - assertEquals("<snapshot name=\"1 hour\" from=\"0\" to=\"601\" period=\"3600\">\n" + - "\n" + - " <a>\n" + - " <sum>\n" + - " <sum average=\"3.38\" last=\"8.00\" min=\"1.00\" max=\"8.00\" count=\"8\" total=\"27.00\"/>\n" + - " <v1 average=\"2.50\" last=\"4.00\" min=\"1.00\" max=\"4.00\" count=\"4\" total=\"10.00\"/>\n" + - " <v3 average=\"2.50\" last=\"3.00\" min=\"2.00\" max=\"3.00\" count=\"2\" total=\"5.00\"/>\n" + - " <v2 average=\"6.00\" last=\"8.00\" min=\"4.00\" max=\"8.00\" count=\"2\" total=\"12.00\"/>\n" + - " <cnt count=\"500\" average_change_per_second=\"0.83\"/>\n" + - " </sum>\n" + - " <b>\n" + - " <sum average=\"2.00\" last=\"3.00\" min=\"1.00\" max=\"3.00\" count=\"4\" total=\"8.00\"/>\n" + - " <v1 average=\"1.50\" last=\"2.00\" min=\"1.00\" max=\"2.00\" count=\"2\" total=\"3.00\"/>\n" + - " <v3 average=\"2.50\" last=\"3.00\" min=\"2.00\" max=\"3.00\" count=\"2\" total=\"5.00\"/>\n" + - " <cnt count=\"500\" average_change_per_second=\"0.83\"/>\n" + - " </b>\n" + - " <c>\n" + - " <sum average=\"4.75\" last=\"8.00\" min=\"3.00\" max=\"8.00\" count=\"4\" total=\"19.00\"/>\n" + - " <v1 average=\"3.50\" last=\"4.00\" min=\"3.00\" max=\"4.00\" count=\"2\" total=\"7.00\"/>\n" + - " <v2 average=\"6.00\" last=\"8.00\" min=\"4.00\" max=\"8.00\" count=\"2\" total=\"12.00\"/>\n" + - " </c>\n" + - " </a>\n" + - "\n" + - "</snapshot>\n", manager.getMetricSnapshot(3600, true).toXml(manager, "", 2)); - - AveragedDoubleValueMetric v4_c = new AveragedDoubleValueMetric("v4", "", "", ms_c); - s_c_a.addMetricToSum(v4_c); - - v2_c.addValue(4.0); - v4_c.addValue(12.0); - c1_b.inc(3000); - - timer.timeNow = 901; - manager.takeSnapshots(901); - - assertEquals("<snapshot name=\"5 minute\" from=\"901\" to=\"901\" period=\"300\">\n" + - "\n" + - " <a>\n" + - " <sum>\n" + - " <sum average=\"8.00\" last=\"12.00\" min=\"4.00\" max=\"12.00\" count=\"2\" total=\"16.00\"/>\n" + - " <v1 average=\"0.00\" last=\"0\" count=\"0\" total=\"0\"/>\n" + - " <v3 average=\"0.00\" last=\"0\" count=\"0\" total=\"0\"/>\n" + - " <v2 average=\"4.00\" last=\"4.00\" min=\"4.00\" max=\"4.00\" count=\"1\" total=\"4.00\"/>\n" + - " <cnt count=\"3000\" average_change_per_second=\"10.00\"/>\n" + - " <v4 average=\"12.00\" last=\"12.00\" min=\"12.00\" max=\"12.00\" count=\"1\" total=\"12.00\"/>\n" + - " </sum>\n" + - " <b>\n" + - " <sum average=\"0.00\" last=\"0\" count=\"0\" total=\"0\"/>\n" + - " <v1 average=\"0.00\" last=\"0\" count=\"0\" total=\"0\"/>\n" + - " <v3 average=\"0.00\" last=\"0\" count=\"0\" total=\"0\"/>\n" + - " <cnt count=\"3000\" average_change_per_second=\"10.00\"/>\n" + - " </b>\n" + - " <c>\n" + - " <sum average=\"8.00\" last=\"12.00\" min=\"4.00\" max=\"12.00\" count=\"2\" total=\"16.00\"/>\n" + - " <v1 average=\"0.00\" last=\"0\" count=\"0\" total=\"0\"/>\n" + - " <v2 average=\"4.00\" last=\"4.00\" min=\"4.00\" max=\"4.00\" count=\"1\" total=\"4.00\"/>\n" + - " <v4 average=\"12.00\" last=\"12.00\" min=\"12.00\" max=\"12.00\" count=\"1\" total=\"12.00\"/>\n" + - " </c>\n" + - " </a>\n" + - "\n" + - "</snapshot>\n", manager.getMetricSnapshot(300, false).toXml(manager, "", 2)); - - assertEquals("<snapshot name=\"1 hour\" from=\"0\" to=\"901\" period=\"3600\">\n" + - "\n" + - " <a>\n" + - " <sum>\n" + - " <sum average=\"4.30\" last=\"12.00\" min=\"1.00\" max=\"12.00\" count=\"10\" total=\"43.00\"/>\n" + - " <v1 average=\"2.50\" last=\"4.00\" min=\"1.00\" max=\"4.00\" count=\"4\" total=\"10.00\"/>\n" + - " <v3 average=\"2.50\" last=\"3.00\" min=\"2.00\" max=\"3.00\" count=\"2\" total=\"5.00\"/>\n" + - " <v2 average=\"5.33\" last=\"4.00\" min=\"4.00\" max=\"8.00\" count=\"3\" total=\"16.00\"/>\n" + - " <cnt count=\"3500\" average_change_per_second=\"3.88\"/>\n" + - " <v4 average=\"12.00\" last=\"12.00\" min=\"12.00\" max=\"12.00\" count=\"1\" total=\"12.00\"/>\n" + - " </sum>\n" + - " <b>\n" + - " <sum average=\"2.00\" last=\"3.00\" min=\"1.00\" max=\"3.00\" count=\"4\" total=\"8.00\"/>\n" + - " <v1 average=\"1.50\" last=\"2.00\" min=\"1.00\" max=\"2.00\" count=\"2\" total=\"3.00\"/>\n" + - " <v3 average=\"2.50\" last=\"3.00\" min=\"2.00\" max=\"3.00\" count=\"2\" total=\"5.00\"/>\n" + - " <cnt count=\"3500\" average_change_per_second=\"3.88\"/>\n" + - " </b>\n" + - " <c>\n" + - " <sum average=\"5.83\" last=\"12.00\" min=\"3.00\" max=\"12.00\" count=\"6\" total=\"35.00\"/>\n" + - " <v1 average=\"3.50\" last=\"4.00\" min=\"3.00\" max=\"4.00\" count=\"2\" total=\"7.00\"/>\n" + - " <v2 average=\"5.33\" last=\"4.00\" min=\"4.00\" max=\"8.00\" count=\"3\" total=\"16.00\"/>\n" + - " <v4 average=\"12.00\" last=\"12.00\" min=\"12.00\" max=\"12.00\" count=\"1\" total=\"12.00\"/>\n" + - " </c>\n" + - " </a>\n" + - "\n" + - "</snapshot>\n", manager.getMetricSnapshot(3600, true).toXml(manager, "", 2)); - } - -} // metrics diff --git a/metrics/src/test/java/com/yahoo/metrics/SumMetricTest.java b/metrics/src/test/java/com/yahoo/metrics/SumMetricTest.java deleted file mode 100644 index 008575af574..00000000000 --- a/metrics/src/test/java/com/yahoo/metrics/SumMetricTest.java +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -/** - * @author thomasg - */ -public class SumMetricTest { - - private final double delta = 0.000000001; - - @Test - public void testSummedCountMetric() { - MetricSet parent = new SimpleMetricSet("parent", "", ""); - SumMetric sum = new SumMetric("foo", "", "foodesc", parent); - - CountMetric v1 = new CountMetric("aa", "", "", parent); - CountMetric v2 = new CountMetric("bb", "", "", parent); - CountMetric v3 = new CountMetric("cc", "", "", parent); - - sum.addMetricToSum(v1); - sum.addMetricToSum(v2); - sum.addMetricToSum(v3); - - // Give them some values - v1.inc(3); - v2.inc(7); - v3.inc(5); - - assertEquals("<foo description=\"foodesc\" count=\"15\"/>\n", sum.toXml(0, 2)); - assertEquals(15, sum.getLongValue("value")); - - v3.inc(5); - - assertEquals("<foo description=\"foodesc\" count=\"20\"/>\n", sum.toXml(0, 2)); - assertEquals(20, sum.getLongValue("value")); - } - - @Test - public void testSummedValueMetric() { - MetricSet parent = new SimpleMetricSet("parent", "", ""); - SumMetric sum = new SumMetric("foo", "", "foodesc", parent); - - SummedDoubleValueMetric v1 = new SummedDoubleValueMetric("aa", "", "", parent); - SummedDoubleValueMetric v2 = new SummedDoubleValueMetric("bb", "", "", parent); - SummedDoubleValueMetric v3 = new SummedDoubleValueMetric("cc", "", "", parent); - - sum.addMetricToSum(v1); - sum.addMetricToSum(v2); - sum.addMetricToSum(v3); - - v1.addValue(3.0); - v1.addValue(2.0); - - v2.addValue(7.0); - - v3.addValue(5.0); - v3.addValue(10.0); - - assertEquals("<foo description=\"foodesc\" average=\"17.00\" last=\"19.00\" min=\"2.00\" max=\"10.00\" count=\"5\" total=\"85.00\"/>\n", sum.toXml(0,2)); - - assertEquals(19, sum.getLongValue("value")); - assertEquals(2, sum.getLongValue("min")); - assertEquals(10, sum.getLongValue("max")); - } - - @Test - public void testAveragedValueMetric() { - MetricSet parent = new SimpleMetricSet("parent", "", ""); - SumMetric sum = new SumMetric("foo", "", "foodesc", parent); - - AveragedDoubleValueMetric v1 = new AveragedDoubleValueMetric("aa", "", "", parent); - AveragedDoubleValueMetric v2 = new AveragedDoubleValueMetric("bb", "", "", parent); - AveragedDoubleValueMetric v3 = new AveragedDoubleValueMetric("cc", "", "", parent); - - sum.addMetricToSum(v1); - sum.addMetricToSum(v2); - sum.addMetricToSum(v3); - - v1.addValue(3.0); - v1.addValue(2.0); - - v2.addValue(7.0); - - v3.addValue(5.0); - v3.addValue(10.0); - - assertEquals("<foo description=\"foodesc\" average=\"5.40\" last=\"10.00\" min=\"2.00\" max=\"10.00\" count=\"5\" total=\"27.00\"/>\n", sum.toXml(0,2)); - - assertEquals(5.40, sum.getDoubleValue("value"), delta); - assertEquals(2.0, sum.getDoubleValue("min"), delta); - assertEquals(10.0, sum.getDoubleValue("max"), delta); - } - - @Test - public void testMetricSet() { - MetricSet parent = new SimpleMetricSet("parent", "", ""); - SumMetric sum = new SumMetric("foo", "", "bar", parent); - - MetricSet set1 = new SimpleMetricSet("a", "", "", parent); - MetricSet set2 = new SimpleMetricSet("b", "", "", parent); - - SummedLongValueMetric v1 = new SummedLongValueMetric("c", "", "", set1); - SummedLongValueMetric v2 = new SummedLongValueMetric("c", "", "", set2); - CountMetric v3 = new CountMetric("e", "", "", set1); - CountMetric v4 = new CountMetric("e", "", "", set2); - - sum.addMetricToSum(set1); - sum.addMetricToSum(set2); - - // Give them some values - v1.addValue((long)3); - v2.addValue((long)7); - v3.inc(2); - v4.inc(); - - // Verify XML output. Should be in register order. - assertEquals( - "<foo description=\"bar\">\n\n" + - " <c average=\"10.00\" last=\"10\" min=\"3\" max=\"7\" count=\"2\" total=\"20\"/>\n\n" + - " <e count=\"3\"/>\n\n" + - "</foo>\n", sum.toXml(0,2)); - - } - - @Test - public void testRemove() { - MetricSet parent = new SimpleMetricSet("parent", "", ""); - SumMetric sum = new SumMetric("foo", "", "foodesc", parent); - - SummedDoubleValueMetric v1 = new SummedDoubleValueMetric("aa", "", "", parent); - SummedDoubleValueMetric v2 = new SummedDoubleValueMetric("bb", "", "", parent); - SummedDoubleValueMetric v3 = new SummedDoubleValueMetric("cc", "", "", parent); - - sum.addMetricToSum(v1); - sum.addMetricToSum(v2); - sum.addMetricToSum(v3); - - v1.addValue(3.0); - v1.addValue(2.0); - - v2.addValue(7.0); - - v3.addValue(5.0); - v3.addValue(10.0); - - sum.removeMetricFromSum(v1); - - assertEquals("<foo description=\"foodesc\" average=\"14.50\" last=\"17.00\" min=\"5.00\" max=\"10.00\" count=\"3\" total=\"43.50\"/>\n", sum.toXml(0,2)); - } - - @Test - public void testEmpty() { - SumMetric sum = new SumMetric("foo", "", "foodesc", null); - assertEquals("<foo description=\"foodesc\"/>\n", sum.toXml(0,2)); - } - -} diff --git a/metrics/src/test/java/com/yahoo/metrics/SummedDoubleValueMetric.java b/metrics/src/test/java/com/yahoo/metrics/SummedDoubleValueMetric.java deleted file mode 100644 index 5d616d93821..00000000000 --- a/metrics/src/test/java/com/yahoo/metrics/SummedDoubleValueMetric.java +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -/** - * @author Simon Thoresen Hult - */ -class SummedDoubleValueMetric extends ValueMetric<Double> { - - public SummedDoubleValueMetric(String name, String tags, String description, MetricSet owner) { - super(name, tags, description, owner); - } - - public SummedDoubleValueMetric(SummedDoubleValueMetric other, CopyType copyType, MetricSet owner) { - super(other, copyType, owner); - } -} diff --git a/metrics/src/test/java/com/yahoo/metrics/SummedLongValueMetric.java b/metrics/src/test/java/com/yahoo/metrics/SummedLongValueMetric.java deleted file mode 100644 index cabd715cc63..00000000000 --- a/metrics/src/test/java/com/yahoo/metrics/SummedLongValueMetric.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -/** - * @author Simon Thoresen Hult - */ -class SummedLongValueMetric extends ValueMetric<Long> { - - public SummedLongValueMetric(String name, String tags, String description, MetricSet owner) { - super(name, tags, description, owner); - } - - public SummedLongValueMetric(SummedLongValueMetric other, CopyType copyType, MetricSet owner) { - super(other, copyType, owner); - } -} - diff --git a/metrics/src/test/java/com/yahoo/metrics/ValueMetricTest.java b/metrics/src/test/java/com/yahoo/metrics/ValueMetricTest.java deleted file mode 100644 index 40ba70a9749..00000000000 --- a/metrics/src/test/java/com/yahoo/metrics/ValueMetricTest.java +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.metrics; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - -public class ValueMetricTest { - - private static final double delta = 0.000000001; - - @Test - public void testAveragedDoubleValueMetric() { - AveragedDoubleValueMetric m = new AveragedDoubleValueMetric("test", "tag", "description", null); - m.addValue(100.0); - assertEquals("count=\"1\" min=\"100.00\" max=\"100.00\" last=\"100.00\" total=\"100.00\" average=\"100.00\"", m.toString()); - m.addValue(100.0); - assertEquals("count=\"2\" min=\"100.00\" max=\"100.00\" last=\"100.00\" total=\"200.00\" average=\"100.00\"", m.toString()); - m.addValue(40.0); - assertEquals("count=\"3\" min=\"40.00\" max=\"100.00\" last=\"40.00\" total=\"240.00\" average=\"80.00\"", m.toString()); - - AveragedDoubleValueMetric m2 = new AveragedDoubleValueMetric(m, Metric.CopyType.CLONE, null); - assertEquals("count=\"3\" min=\"40.00\" max=\"100.00\" last=\"40.00\" total=\"240.00\" average=\"80.00\"", m.toString()); - assertEquals("count=\"3\" min=\"40.00\" max=\"100.00\" last=\"40.00\" total=\"240.00\" average=\"80.00\"", m2.toString()); - m.reset(); - assertEquals("count=\"0\" min=\"0\" max=\"0\" last=\"0\" total=\"0\" average=\"0.00\"", m.toString()); - assertEquals("count=\"3\" min=\"40.00\" max=\"100.00\" last=\"40.00\" total=\"240.00\" average=\"80.00\"", m2.toString()); - - AveragedDoubleValueMetric m3 = new AveragedDoubleValueMetric("test", "tag", "description", null); - m3.addValue(200.0); - m3.addValue(100.0); - m3.addValue(400.0); - - AveragedDoubleValueMetric m4 = new AveragedDoubleValueMetric("test", "tag", "description", null); - m3.addValue(50.0); - m3.addValue(100.0); - m3.addValue(2000.0); - - AveragedDoubleValueMetric sum = new AveragedDoubleValueMetric(m2, Metric.CopyType.INACTIVE, null); - m3.addToPart(sum); - m4.addToPart(sum); - assertEquals("count=\"9\" min=\"40.00\" max=\"2000.00\" last=\"2000.00\" total=\"3090.00\" average=\"343.33\"", sum.toString()); - - AveragedDoubleValueMetric snapshot = new AveragedDoubleValueMetric(m2, Metric.CopyType.INACTIVE, null); - m3.addToSnapshot(snapshot); - m4.addToSnapshot(snapshot); - assertEquals("count=\"9\" min=\"40.00\" max=\"2000.00\" last=\"2000.00\" total=\"3090.00\" average=\"343.33\"", snapshot.toString()); - - assertEquals("<test description=\"description\" average=\"343.33\" last=\"2000.00\" min=\"40.00\" max=\"2000.00\" count=\"9\" total=\"3090.00\"/>\n", sum.toXml(0, 2)); - - assertEquals(80.0, m2.getDoubleValue("value"), delta); - assertEquals(80.0, m2.getDoubleValue("average"), delta); - assertEquals(40.0, m2.getDoubleValue("min"), delta); - assertEquals(100.0, m2.getDoubleValue("max"), delta); - assertEquals(40.0, m2.getDoubleValue("last"), delta); - assertEquals(3.0, m2.getDoubleValue("count"), delta); - assertEquals(240.0, m2.getDoubleValue("total"), delta); - - assertEquals(80, m2.getLongValue("value")); - assertEquals(80, m2.getLongValue("average")); - assertEquals(40, m2.getLongValue("min")); - assertEquals(100, m2.getLongValue("max")); - assertEquals(40, m2.getLongValue("last")); - assertEquals(3, m2.getLongValue("count")); - assertEquals(240, m2.getLongValue("total")); - } - - @Test - public void testDoubleValueMetricNotUpdatedOnNaN() { - AveragedDoubleValueMetric m = new AveragedDoubleValueMetric("test", "tag", "description", null); - m.addValue(Double.NaN); - assertEquals("count=\"0\" min=\"0\" max=\"0\" last=\"0\" total=\"0\" average=\"0.00\"", m.toString()); - } - - @Test - public void testDoubleValueMetricNotUpdatedOnInfinity() { - AveragedDoubleValueMetric m = new AveragedDoubleValueMetric("test", "tag", "description", null); - m.addValue(Double.POSITIVE_INFINITY); - assertEquals("count=\"0\" min=\"0\" max=\"0\" last=\"0\" total=\"0\" average=\"0.00\"", m.toString()); - } - - @Test - public void testSummedDoubleValueMetric() { - SummedDoubleValueMetric m = new SummedDoubleValueMetric("test", "tag", "description", null); - m.addValue(100.0); - assertEquals("count=\"1\" min=\"100.00\" max=\"100.00\" last=\"100.00\" total=\"100.00\" average=\"100.00\"", m.toString()); - m.addValue(100.0); - assertEquals("count=\"2\" min=\"100.00\" max=\"100.00\" last=\"100.00\" total=\"200.00\" average=\"100.00\"", m.toString()); - m.addValue(40.0); - assertEquals("count=\"3\" min=\"40.00\" max=\"100.00\" last=\"40.00\" total=\"240.00\" average=\"80.00\"", m.toString()); - - SummedDoubleValueMetric m2 = new SummedDoubleValueMetric(m, Metric.CopyType.CLONE, null); - assertEquals("count=\"3\" min=\"40.00\" max=\"100.00\" last=\"40.00\" total=\"240.00\" average=\"80.00\"", m.toString()); - assertEquals("count=\"3\" min=\"40.00\" max=\"100.00\" last=\"40.00\" total=\"240.00\" average=\"80.00\"", m2.toString()); - m.reset(); - assertEquals("count=\"0\" min=\"0\" max=\"0\" last=\"0\" total=\"0\" average=\"0.00\"", m.toString()); - assertEquals("count=\"3\" min=\"40.00\" max=\"100.00\" last=\"40.00\" total=\"240.00\" average=\"80.00\"", m2.toString()); - - SummedDoubleValueMetric m3 = new SummedDoubleValueMetric("test", "tag", "description", null); - m3.addValue(200.0); - m3.addValue(100.0); - m3.addValue(400.0); - - SummedDoubleValueMetric m4 = new SummedDoubleValueMetric("test", "tag", "description", null); - m4.addValue(2000.0); - - SummedDoubleValueMetric sum = new SummedDoubleValueMetric(m2, Metric.CopyType.INACTIVE, null); - m3.addToPart(sum); - m4.addToPart(sum); - assertEquals("count=\"7\" min=\"40.00\" max=\"2000.00\" last=\"2440.00\" total=\"16193.33\" average=\"2313.33\"", sum.toString()); - - SummedDoubleValueMetric snapshot = new SummedDoubleValueMetric(m2, Metric.CopyType.INACTIVE, null); - m3.addToSnapshot(snapshot); - m4.addToSnapshot(snapshot); - assertEquals("count=\"7\" min=\"40.00\" max=\"2000.00\" last=\"2000.00\" total=\"2940.00\" average=\"420.00\"", snapshot.toString()); - - assertEquals("<test description=\"description\" average=\"2313.33\" last=\"2440.00\" min=\"40.00\" max=\"2000.00\" count=\"7\" total=\"16193.33\"/>\n", sum.toXml(0, 2)); - - assertEquals(40.0, m2.getDoubleValue("value"), delta); - assertEquals(80.0, m2.getDoubleValue("average"), delta); - assertEquals(40.0, m2.getDoubleValue("min"), delta); - assertEquals(100.0, m2.getDoubleValue("max"), delta); - assertEquals(40.0, m2.getDoubleValue("last"), delta); - assertEquals(3.0, m2.getDoubleValue("count"), delta); - assertEquals(240.0, m2.getDoubleValue("total"), delta); - - assertEquals(40, m2.getLongValue("value")); - assertEquals(80, m2.getLongValue("average")); - assertEquals(40, m2.getLongValue("min")); - assertEquals(100, m2.getLongValue("max")); - assertEquals(40, m2.getLongValue("last")); - assertEquals(3, m2.getLongValue("count")); - assertEquals(240, m2.getLongValue("total")); - } - - @Test - public void testAveragedLongValueMetric() { - AveragedLongValueMetric m = new AveragedLongValueMetric("test", "tag", "description", null); - - assertEquals(0L, m.getLongValue("max")); - assertEquals(0L, m.getLongValue("min")); - - m.addValue((long)100); - assertEquals("count=\"1\" min=\"100\" max=\"100\" last=\"100\" total=\"100\" average=\"100.00\"", m.toString()); - m.addValue((long)100); - assertEquals("count=\"2\" min=\"100\" max=\"100\" last=\"100\" total=\"200\" average=\"100.00\"", m.toString()); - m.addValue((long)40); - assertEquals("count=\"3\" min=\"40\" max=\"100\" last=\"40\" total=\"240\" average=\"80.00\"", m.toString()); - - AveragedLongValueMetric m2 = new AveragedLongValueMetric(m, Metric.CopyType.CLONE, null); - assertEquals("count=\"3\" min=\"40\" max=\"100\" last=\"40\" total=\"240\" average=\"80.00\"", m.toString()); - assertEquals("count=\"3\" min=\"40\" max=\"100\" last=\"40\" total=\"240\" average=\"80.00\"", m2.toString()); - m.reset(); - assertEquals("count=\"0\" min=\"0\" max=\"0\" last=\"0\" total=\"0\" average=\"0.00\"", m.toString()); - assertEquals("count=\"3\" min=\"40\" max=\"100\" last=\"40\" total=\"240\" average=\"80.00\"", m2.toString()); - - AveragedLongValueMetric m3 = new AveragedLongValueMetric("test", "tag", "description", null); - m3.addValue((long)200); - m3.addValue((long)100); - m3.addValue((long)400); - - AveragedLongValueMetric m4 = new AveragedLongValueMetric("test", "tag", "description", null); - m3.addValue((long)50); - m3.addValue((long)100); - m3.addValue((long)2000); - - AveragedLongValueMetric sum = new AveragedLongValueMetric(m2, Metric.CopyType.INACTIVE, null); - m3.addToPart(sum); - m4.addToPart(sum); - assertEquals("count=\"9\" min=\"40\" max=\"2000\" last=\"2000\" total=\"3090\" average=\"343.33\"", sum.toString()); - - AveragedLongValueMetric snapshot = new AveragedLongValueMetric(m2, Metric.CopyType.INACTIVE, null); - m3.addToSnapshot(snapshot); - m4.addToSnapshot(snapshot); - assertEquals("count=\"9\" min=\"40\" max=\"2000\" last=\"2000\" total=\"3090\" average=\"343.33\"", snapshot.toString()); - - assertEquals("<test description=\"description\" average=\"343.33\" last=\"2000\" min=\"40\" max=\"2000\" count=\"9\" total=\"3090\"/>\n", sum.toXml(0, 2)); - - assertEquals(80, m2.getLongValue("value")); - assertEquals(80, m2.getLongValue("average")); - assertEquals(40, m2.getLongValue("min")); - assertEquals(100, m2.getLongValue("max")); - assertEquals(40, m2.getLongValue("last")); - assertEquals(3, m2.getLongValue("count")); - assertEquals(240, m2.getLongValue("total")); - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 57e504a6ffd..96db16bf1c1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -33,7 +33,6 @@ import com.yahoo.vespa.hosted.node.admin.util.SecretAgentCheckConfig; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; @@ -534,13 +533,13 @@ public class NodeAgentImpl implements NodeAgent { ContainerStats stats = containerStats.get(); final String APP = MetricReceiverWrapper.APPLICATION_NODE; - final int totalNumCpuCores = ((List<Number>) ((Map) stats.getCpuStats().get("cpu_usage")).get("percpu_usage")).size(); - final long cpuContainerKernelTime = ((Number) ((Map) stats.getCpuStats().get("cpu_usage")).get("usage_in_kernelmode")).longValue(); - final long cpuContainerTotalTime = ((Number) ((Map) stats.getCpuStats().get("cpu_usage")).get("total_usage")).longValue(); - final long cpuSystemTotalTime = ((Number) stats.getCpuStats().get("system_cpu_usage")).longValue(); - final long memoryTotalBytes = ((Number) stats.getMemoryStats().get("limit")).longValue(); - final long memoryTotalBytesUsage = ((Number) stats.getMemoryStats().get("usage")).longValue(); - final long memoryTotalBytesCache = ((Number) ((Map) stats.getMemoryStats().get("stats")).get("cache")).longValue(); + final int totalNumCpuCores = stats.getCpuStats().getOnlineCpus(); + final long cpuContainerKernelTime = stats.getCpuStats().getUsageInKernelMode(); + final long cpuContainerTotalTime = stats.getCpuStats().getTotalUsage(); + final long cpuSystemTotalTime = stats.getCpuStats().getSystemCpuUsage(); + final long memoryTotalBytes = stats.getMemoryStats().getLimit(); + final long memoryTotalBytesUsage = stats.getMemoryStats().getUsage(); + final long memoryTotalBytesCache = stats.getMemoryStats().getCache(); final long diskTotalBytes = (long) (node.getMinDiskAvailableGb() * BYTES_IN_GB); final Optional<Long> diskTotalBytesUsed = storageMaintainer.getDiskUsageFor(context); @@ -573,14 +572,13 @@ public class NodeAgentImpl implements NodeAgent { stats.getNetworks().forEach((interfaceName, interfaceStats) -> { Dimensions netDims = dimensionsBuilder.add("interface", interfaceName).build(); - Map<String, Number> infStats = (Map<String, Number>) interfaceStats; DimensionMetrics networkMetrics = new DimensionMetrics.Builder(APP, netDims) - .withMetric("net.in.bytes", infStats.get("rx_bytes").longValue()) - .withMetric("net.in.errors", infStats.get("rx_errors").longValue()) - .withMetric("net.in.dropped", infStats.get("rx_dropped").longValue()) - .withMetric("net.out.bytes", infStats.get("tx_bytes").longValue()) - .withMetric("net.out.errors", infStats.get("tx_errors").longValue()) - .withMetric("net.out.dropped", infStats.get("tx_dropped").longValue()) + .withMetric("net.in.bytes", interfaceStats.getRxBytes()) + .withMetric("net.in.errors", interfaceStats.getRxErrors()) + .withMetric("net.in.dropped", interfaceStats.getRxDropped()) + .withMetric("net.out.bytes", interfaceStats.getTxBytes()) + .withMetric("net.out.errors", interfaceStats.getTxErrors()) + .withMetric("net.out.dropped", interfaceStats.getTxDropped()) .build(); metrics.add(networkMetrics); }); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index dcbbaf792a8..620c59969d6 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -1,10 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.nodeagent; -import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.component.Version; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeType; +import com.yahoo.io.IOUtils; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; @@ -30,7 +30,7 @@ import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import org.junit.Test; import org.mockito.InOrder; -import java.net.URL; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -82,8 +82,6 @@ public class NodeAgentImplTest { private final MetricReceiverWrapper metricReceiver = new MetricReceiverWrapper(MetricReceiver.nullImplementation); private final AclMaintainer aclMaintainer = mock(AclMaintainer.class); private final HealthChecker healthChecker = mock(HealthChecker.class); - private final ContainerStats emptyContainerStats = new ContainerStats(Collections.emptyMap(), - Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap()); private final CredentialsMaintainer credentialsMaintainer = mock(CredentialsMaintainer.class); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); @@ -645,18 +643,10 @@ public class NodeAgentImplTest { @Test @SuppressWarnings("unchecked") public void testGetRelevantMetrics() throws Exception { - final ObjectMapper objectMapper = new ObjectMapper(); ClassLoader classLoader = getClass().getClassLoader(); - URL statsFile = classLoader.getResource("docker.stats.json"); - Map<String, Map<String, Object>> dockerStats = objectMapper.readValue(statsFile, Map.class); - - Map<String, Object> networks = dockerStats.get("networks"); - Map<String, Object> precpu_stats = dockerStats.get("precpu_stats"); - Map<String, Object> cpu_stats = dockerStats.get("cpu_stats"); - Map<String, Object> memory_stats = dockerStats.get("memory_stats"); - Map<String, Object> blkio_stats = dockerStats.get("blkio_stats"); - ContainerStats stats1 = new ContainerStats(networks, precpu_stats, memory_stats, blkio_stats); - ContainerStats stats2 = new ContainerStats(networks, cpu_stats, memory_stats, blkio_stats); + String json = IOUtils.readAll(classLoader.getResourceAsStream("docker.stats.json"), StandardCharsets.UTF_8); + ContainerStats stats2 = ContainerStats.fromJson(json); + ContainerStats stats1 = ContainerStats.fromJson(json.replace("\"cpu_stats\"", "\"cpu_stats2\"").replace("\"precpu_stats\"", "\"cpu_stats\"")); NodeOwner owner = new NodeOwner("tester", "testapp", "testinstance"); NodeMembership membership = new NodeMembership("clustType", "clustId", "grp", 3, false); @@ -759,7 +749,6 @@ public class NodeAgentImplTest { private NodeAgentImpl makeNodeAgent(DockerImage dockerImage, boolean isRunning) { mockGetContainer(dockerImage, isRunning); - when(dockerOperations.getContainerStats(any())).thenReturn(Optional.of(emptyContainerStats)); doNothing().when(storageMaintainer).writeMetricsConfig(any()); return new NodeAgentImpl(contextSupplier, nodeRepository, orchestrator, dockerOperations, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 999eae73475..ccdfbf0b402 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -15,7 +15,6 @@ import com.yahoo.config.provisioning.NodeRepositoryConfig; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.curator.Curator; -import com.yahoo.vespa.hosted.provision.flag.Flags; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerList; @@ -87,7 +86,6 @@ public class NodeRepository extends AbstractComponent { private final OsVersions osVersions; private final FirmwareChecks firmwareChecks; private final DockerImages dockerImages; - private final Flags flags; /** * Creates a node repository from a zookeeper provider. @@ -112,7 +110,6 @@ public class NodeRepository extends AbstractComponent { this.osVersions = new OsVersions(this.db); this.firmwareChecks = new FirmwareChecks(db, clock); this.dockerImages = new DockerImages(db, dockerImage); - this.flags = new Flags(this.db); // read and write all nodes to make sure they are stored in the latest version of the serialized format for (Node.State state : Node.State.values()) @@ -137,11 +134,6 @@ public class NodeRepository extends AbstractComponent { /** Returns the docker images to use for nodes in this. */ public DockerImages dockerImages() { return dockerImages; } - /** Returns feature flags of this node repository */ - public Flags flags() { - return flags; - } - // ---------------- Query API ---------------------------------------------------------------- /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/Flag.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/Flag.java deleted file mode 100644 index a22e1dea024..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/Flag.java +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.flag; - -import com.google.common.collect.ImmutableSet; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; - -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.Objects; -import java.util.Set; - -/** - * Represents a feature flag and its status. Use {@link Flags#get(FlagId)} to lookup status for a specific flag. - * - * @author mpolden - */ -public class Flag { - - private final FlagId id; - private final boolean enabled; - private final Set<String> hostnames; - private final Set<ApplicationId> applications; - - public Flag(FlagId id, boolean enabled, Set<String> hostnames, Set<ApplicationId> applications) { - this.id = Objects.requireNonNull(id, "id must be non-null"); - this.enabled = enabled; - this.hostnames = ImmutableSet.copyOf(Objects.requireNonNull(hostnames, "hostnames must be non-null")); - this.applications = ImmutableSet.copyOf(Objects.requireNonNull(applications, "applications must be non-null")); - } - - public FlagId id() { - return id; - } - - /** The hostnames this flag should apply to */ - public Set<String> hostnames() { - return hostnames; - } - - /** The applications this flag should apply to */ - public Set<ApplicationId> applications() { - return applications; - } - - /** - * Returns whether this flag is enabled for all dimensions. Note: More specific dimensions always return true when - * this is true - */ - public boolean isEnabled() { - return enabled; - } - - /** Returns whether this flag is enabled for given hostname */ - public boolean isEnabled(HostName hostname) { - return enabled || hostnames.contains(hostname.value()); - } - - /** Returns whether this flag is enabled for given application */ - public boolean isEnabled(ApplicationId application) { - return enabled || applications.contains(application); - } - - /** Returns a copy of this with this flag enabled for all dimensions */ - public Flag withEnabled(boolean enabled) { - return new Flag(id, enabled, hostnames, applications); - } - - /** Returns a copy of this with enabled set for hostname */ - public Flag withEnabled(HostName hostname, boolean enabled) { - Set<String> hostnames = new LinkedHashSet<>(this.hostnames); - if (enabled) { - hostnames.add(hostname.value()); - } else { - hostnames.remove(hostname.value()); - } - return new Flag(id, this.enabled, hostnames, applications); - } - - /** Returns a copy of this with enabled set for application */ - public Flag withEnabled(ApplicationId application, boolean enabled) { - Set<ApplicationId> applications = new LinkedHashSet<>(this.applications); - if (enabled) { - applications.add(application); - } else { - applications.remove(application); - } - return new Flag(id, this.enabled, hostnames, applications); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Flag flag = (Flag) o; - return id == flag.id; - } - - @Override - public int hashCode() { - return Objects.hash(id); - } - - /** Create a flag for given feature that is disabled for all dimensions */ - public static Flag disabled(FlagId id) { - return new Flag(id, false, Collections.emptySet(), Collections.emptySet()); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/FlagId.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/FlagId.java deleted file mode 100644 index 1b798c14588..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/FlagId.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.flag; - -import java.util.Arrays; - -/** - * Features of this node repository that can be toggled. - * - * @author mpolden - */ -public enum FlagId { - - /** Indicates whether a exclusive load balancer should be provisioned */ - exclusiveLoadBalancer("exclusive-load-balancer"); - - private final String serializedValue; - - FlagId(String serializedValue) { - this.serializedValue = serializedValue; - } - - public String serializedValue() { - return serializedValue; - } - - public static FlagId fromSerializedForm(String value) { - return Arrays.stream(FlagId.values()) - .filter(f -> f.serializedValue().equals(value)) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException("Could not find flag ID by serialized value '" + - value + "'")); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/Flags.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/Flags.java deleted file mode 100644 index b4ecf415ede..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/flag/Flags.java +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.flag; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; -import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.function.Function; -import java.util.stream.Collectors; - -/** - * This class provides feature flags for the node repository. A feature flag can be toggled for the following - * dimensions: - * - * 1) The node repository (entire zone) - * 2) A specific node - * 3) A specific application - * - * Code which needs to consider feature flags can access them through {@link NodeRepository#flags()}. - * - * @author mpolden - */ -public class Flags { - - private final CuratorDatabaseClient db; - - public Flags(CuratorDatabaseClient db) { - this.db = Objects.requireNonNull(db, "db must be non-null"); - } - - /** Get status for given feature flag */ - public Flag get(FlagId id) { - return db.readFlag(id).orElseGet(() -> Flag.disabled(id)); - } - - /** Get all known feature flags */ - public List<Flag> list() { - return Arrays.stream(FlagId.values()) - .map(this::get) - .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList)); - } - - /** Enable feature flag in this node repository */ - public void setEnabled(FlagId flag, boolean enabled) { - write(flag, (f) -> f.withEnabled(enabled)); - } - - /** Enable feature flag for given application */ - public void setEnabled(FlagId flag, ApplicationId application, boolean enabled) { - write(flag, (f) -> f.withEnabled(application, enabled)); - } - - /** Enable feature flag for given node */ - public void setEnabled(FlagId flag, HostName hostname, boolean enabled) { - write(flag, (f) -> f.withEnabled(hostname, enabled)); - } - - private void write(FlagId id, Function<Flag, Flag> updateFunc) { - try (Lock lock = db.lockFlags()) { - db.writeFlag(updateFunc.apply(get(id))); - } - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index 3ed8ef9f64a..a78c48418ea 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -20,8 +20,6 @@ import com.yahoo.vespa.curator.recipes.CuratorCounter; import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.flag.Flag; -import com.yahoo.vespa.hosted.provision.flag.FlagId; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -77,6 +75,7 @@ public class CuratorDatabaseClient { public CuratorDatabaseClient(NodeFlavors flavors, Curator curator, Clock clock, Zone zone, boolean useCache) { this.nodeSerializer = new NodeSerializer(flavors); this.zone = zone; + curator.delete(flagsRoot); // TODO: Remove after 7.42 has been released this.curatorDatabase = new CuratorDatabase(curator, root, useCache); this.clock = clock; this.provisionIndexCounter = new CuratorCounter(curator, root.append("provisionIndexCounter").getAbsolute()); @@ -97,7 +96,6 @@ public class CuratorDatabaseClient { curatorDatabase.create(dockerImagesPath()); curatorDatabase.create(firmwareCheckPath()); curatorDatabase.create(loadBalancersRoot); - curatorDatabase.create(flagsRoot); provisionIndexCounter.initialize(100); } @@ -527,28 +525,6 @@ public class CuratorDatabaseClient { return loadBalancersRoot.append(id.serializedForm()); } - public void writeFlag(Flag flag) { - Path path = flagPath(flag.id()); - NestedTransaction transaction = new NestedTransaction(); - CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); - curatorTransaction.add(createOrSet(path, FlagSerializer.toJson(flag))); - transaction.commit(); - } - - - // Flags - public Optional<Flag> readFlag(FlagId id) { - return read(flagPath(id), FlagSerializer::fromJson); - } - - public Lock lockFlags() { - return lock(lockRoot.append("flagsLock"), defaultLockTimeout); - } - - private Path flagPath(FlagId id) { - return flagsRoot.append(id.serializedValue()); - } - private Transaction.Operation createOrSet(Path path, byte[] data) { if (curatorDatabase.exists(path)) { return CuratorOperations.setData(path.getAbsolute(), data); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/FlagSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/FlagSerializer.java deleted file mode 100644 index 431aa92a513..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/FlagSerializer.java +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.persistence; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.slime.ArrayTraverser; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; -import com.yahoo.vespa.hosted.provision.flag.Flag; -import com.yahoo.vespa.hosted.provision.flag.FlagId; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.LinkedHashSet; -import java.util.Set; - -/** - * @author mpolden - */ -public class FlagSerializer { - - private static final String featureField = "feature"; - private static final String enabledField = "enabled"; - private static final String hostnamesField = "hostnames"; - private static final String applicationsField = "applications"; - - public static byte[] toJson(Flag flag) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - - root.setString(featureField, flag.id().serializedValue()); - root.setBool(enabledField, flag.isEnabled()); - - Cursor nodeArray = root.setArray(hostnamesField); - flag.hostnames().forEach(nodeArray::addString); - - Cursor applicationArray = root.setArray(applicationsField); - flag.applications().forEach(application -> applicationArray.addString(application.serializedForm())); - - try { - return SlimeUtils.toJsonBytes(slime); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - public static Flag fromJson(byte[] data) { - Inspector inspect = SlimeUtils.jsonToSlime(data).get(); - - Set<String> hostnames = new LinkedHashSet<>(); - inspect.field(hostnamesField).traverse((ArrayTraverser) (i, hostname) -> hostnames.add(hostname.asString())); - - Set<ApplicationId> applications = new LinkedHashSet<>(); - inspect.field(applicationsField).traverse((ArrayTraverser) (i, application) -> { - applications.add(ApplicationId.fromSerializedForm(application.asString())); - }); - - return new Flag(FlagId.fromSerializedForm(inspect.field(featureField).asString()), - inspect.field(enabledField).asBool(), - hostnames, - applications); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 629bbf56884..f5f8ed53d2a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.NodeType; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.Node; @@ -93,7 +94,7 @@ public class LoadBalancerProvisioner { /** Returns a list of active containers for given application, grouped by cluster spec */ private Map<ClusterSpec, List<Node>> activeContainers(ApplicationId application) { - return new NodeList(nodeRepository.getNodes(Node.State.active)) + return new NodeList(nodeRepository.getNodes(NodeType.tenant, Node.State.active)) .owner(application) .filter(node -> node.state().isAllocated()) .type(ClusterSpec.Type.container) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 246c56ee28b..c91d28e17ce 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -21,7 +21,6 @@ import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.flag.FlagId; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; @@ -116,13 +115,11 @@ public class NodeRepositoryProvisioner implements Provisioner { validate(hosts); activator.activate(application, hosts, transaction); transaction.onCommitted(() -> { - if (nodeRepository.flags().get(FlagId.exclusiveLoadBalancer).isEnabled(application)) { - try { - loadBalancerProvisioner.ifPresent(lbProvisioner -> lbProvisioner.provision(application)); - } catch (Exception e) { - log.log(LogLevel.ERROR, "Failed to provision load balancer for application " + - application.toShortString(), e); - } + try { + loadBalancerProvisioner.ifPresent(lbProvisioner -> lbProvisioner.provision(application)); + } catch (Exception e) { + log.log(LogLevel.ERROR, "Failed to provision load balancer for application " + + application.toShortString(), e); } }); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/FlagsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/FlagsResponse.java deleted file mode 100644 index 1bc016bcda2..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/FlagsResponse.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.hosted.provision.flag.Flag; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.List; - -/** - * @author mpolden - */ -public class FlagsResponse extends HttpResponse { - - private final List<Flag> flags; - - public FlagsResponse(List<Flag> flags) { - super(200); - this.flags = flags; - } - - @Override - public void render(OutputStream out) throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor flagArray = root.setArray("flags"); - flags.forEach(flag -> { - Cursor flagObject = flagArray.addObject(); - flagObject.setString("id", flag.id().serializedValue()); - flagObject.setBool("enabled", flag.isEnabled()); - Cursor nodeArray = flagObject.setArray("enabledHostnames"); - flag.hostnames().forEach(nodeArray::addString); - Cursor applicationArray = flagObject.setArray("enabledApplications"); - flag.applications().stream() - .map(ApplicationId::serializedForm) - .forEach(applicationArray::addString); - }); - new JsonFormat(true).encode(out, slime); - } - - @Override - public String getContentType() { - return "application/json"; - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java index 58125304d6f..a18b7ab72ad 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java @@ -2,10 +2,8 @@ package com.yahoo.vespa.hosted.provision.restapi.v2; import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostFilter; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.container.jdisc.HttpRequest; @@ -20,7 +18,6 @@ import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.provision.NoSuchNodeException; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.flag.FlagId; import com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; @@ -108,7 +105,6 @@ public class NodesApiHandler extends LoggingRequestHandler { if (path.equals( "/nodes/v2/command/")) return ResourcesResponse.fromStrings(request.getUri(), "restart", "reboot"); if (path.equals( "/nodes/v2/maintenance/")) return new JobsResponse(maintenance.jobControl()); if (path.equals( "/nodes/v2/upgrade/")) return new UpgradeResponse(maintenance.infrastructureVersions(), nodeRepository.osVersions(), nodeRepository.dockerImages()); - if (path.equals( "/nodes/v2/flags/")) return new FlagsResponse(nodeRepository.flags().list()); throw new NotFoundException("Nothing at path '" + path + "'"); } @@ -169,8 +165,6 @@ public class NodesApiHandler extends LoggingRequestHandler { return new MessageResponse("Added " + addedNodes + " nodes to the provisioned state"); } if (path.matches("/nodes/v2/maintenance/inactive/{job}")) return setJobActive(path.get("job"), false); - if (path.matches("/nodes/v2/flags/{flag}")) return setFlag(path.get("flag"), true, "", ""); - if (path.matches("/nodes/v2/flags/{flag}/{dimension}/{value}")) return setFlag(path.get("flag"), true, path.get("dimension"), path.get("value")); if (path.matches("/nodes/v2/upgrade/firmware")) return requestFirmwareCheckResponse(); throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'"); @@ -184,8 +178,6 @@ public class NodesApiHandler extends LoggingRequestHandler { return new MessageResponse("Removed " + removedNodes.stream().map(Node::hostname).collect(Collectors.joining(", "))); } if (path.matches("/nodes/v2/maintenance/inactive/{job}")) return setJobActive(path.get("job"), true); - if (path.matches("/nodes/v2/flags/{flag}")) return setFlag(path.get("flag"), false, "", ""); - if (path.matches("/nodes/v2/flags/{flag}/{dimension}/{value}")) return setFlag(path.get("flag"), false, path.get("dimension"), path.get("value")); if (path.matches("/nodes/v2/upgrade/firmware")) return cancelFirmwareCheckResponse(); throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'"); @@ -283,24 +275,6 @@ public class NodesApiHandler extends LoggingRequestHandler { return new MessageResponse((active ? "Re-activated" : "Deactivated" ) + " job '" + jobName + "'"); } - private HttpResponse setFlag(String flag, boolean enabled, String dimension, String value) { - FlagId flagId = FlagId.fromSerializedForm(flag); - switch (dimension) { - case "application": - nodeRepository.flags().setEnabled(flagId, ApplicationId.fromSerializedForm(value), enabled); - break; - case "node": - nodeRepository.flags().setEnabled(flagId, HostName.from(value), enabled); - break; - case "": - nodeRepository.flags().setEnabled(flagId, enabled); - break; - default: throw new IllegalArgumentException("Unknown flag dimension '" + dimension + "'"); - } - return new MessageResponse((enabled ? "Enabled" : "Disabled") + " feature " + flagId + - (!value.isEmpty() ? " for " + dimension + " '" + value + "'" : "")); - } - private MessageResponse setTargetVersions(HttpRequest request) { NodeType nodeType = NodeType.valueOf(lastElement(request.getUri().getPath()).toLowerCase()); Inspector inspector = toSlime(request.getData()).get(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilter.java index 28ead318cc0..9934c343092 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilter.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.provision.restapi.v2.filter; import com.google.inject.Inject; -import com.yahoo.config.provision.Zone; +import com.yahoo.config.provisioning.ConfigServerSecurityConfig; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.SecurityRequestFilter; @@ -31,8 +31,8 @@ public class AuthorizationFilter implements SecurityRequestFilter { private final BiConsumer<ErrorResponse, ResponseHandler> rejectAction; @Inject - public AuthorizationFilter(Zone zone, NodeRepository nodeRepository) { - this.authorizer = new Authorizer(zone.system(), nodeRepository); + public AuthorizationFilter(NodeRepository nodeRepository, ConfigServerSecurityConfig securityConfig) { + this.authorizer = new Authorizer(nodeRepository, securityConfig); this.rejectAction = AuthorizationFilter::logAndReject; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/Authorizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/Authorizer.java index afcde0949e3..062a9c32afb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/Authorizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/Authorizer.java @@ -2,9 +2,9 @@ package com.yahoo.vespa.hosted.provision.restapi.v2.filter; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provisioning.ConfigServerSecurityConfig; import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.utils.AthenzIdentities; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import org.apache.http.NameValuePair; @@ -15,7 +15,6 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; @@ -31,20 +30,23 @@ import java.util.stream.Collectors; */ public class Authorizer implements BiPredicate<NodePrincipal, URI> { private final NodeRepository nodeRepository; - private final AthenzIdentity controllerIdentity; - private final AthenzIdentity configServerIdentity = new AthenzService("vespa.vespa", "configserver"); - private final AthenzIdentity proxyIdentity = new AthenzService("vespa.vespa", "proxy"); - private final AthenzIdentity tenantIdentity = new AthenzService("vespa.vespa", "tenant-host"); + private final String athenzProviderHostname; + private final AthenzIdentity controllerHostIdentity; private final Set<AthenzIdentity> trustedIdentities; private final Set<AthenzIdentity> hostAdminIdentities; - public Authorizer(SystemName system, NodeRepository nodeRepository) { + Authorizer(NodeRepository nodeRepository, ConfigServerSecurityConfig securityConfig) { + AthenzIdentity configServerHostIdentity = AthenzIdentities.from(securityConfig.configServerHostIdentity()); + this.nodeRepository = nodeRepository; - controllerIdentity = system == SystemName.main - ? new AthenzService("vespa.vespa", "hosting") - : new AthenzService("vespa.vespa.cd", "hosting"); - this.trustedIdentities = new HashSet<>(Arrays.asList(controllerIdentity, configServerIdentity)); - this.hostAdminIdentities = new HashSet<>(Arrays.asList(controllerIdentity, configServerIdentity, proxyIdentity, tenantIdentity)); + this.athenzProviderHostname = securityConfig.athenzProviderHostname(); + this.controllerHostIdentity = AthenzIdentities.from(securityConfig.controllerHostIdentity()); + this.trustedIdentities = Set.of(controllerHostIdentity, configServerHostIdentity); + this.hostAdminIdentities = Set.of( + controllerHostIdentity, + configServerHostIdentity, + AthenzIdentities.from(securityConfig.tenantHostIdentity()), + AthenzIdentities.from(securityConfig.proxyHostIdentity())); } /** Returns whether principal is authorized to access given URI */ @@ -58,7 +60,7 @@ public class Authorizer implements BiPredicate<NodePrincipal, URI> { // Only controller can access everything else in flags if (uri.getPath().startsWith("/flags/v1/")) { - return principal.getAthenzIdentityName().get().equals(controllerIdentity); + return principal.getAthenzIdentityName().get().equals(controllerHostIdentity); } // Trusted services can access everything @@ -70,7 +72,7 @@ public class Authorizer implements BiPredicate<NodePrincipal, URI> { if (principal.getHostname().isPresent()) { String hostname = principal.getHostname().get(); if (isAthenzProviderApi(uri)) { - return hostname.equals(NodeIdentifier.ZTS_AWS_IDENTITY) || hostname.equals(NodeIdentifier.ZTS_ON_PREM_IDENTITY); + return athenzProviderHostname.equals(hostname); } // Individual nodes can only access their own resources diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifier.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifier.java index a434f021cad..ecc3f84f86c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifier.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifier.java @@ -5,14 +5,16 @@ import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; +import com.yahoo.config.provisioning.ConfigServerSecurityConfig; import com.yahoo.security.SubjectAlternativeName; import com.yahoo.security.X509CertificateUtils; +import com.yahoo.vespa.athenz.identityprovider.api.VespaUniqueInstanceId; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import java.security.cert.X509Certificate; import java.util.List; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -25,23 +27,24 @@ import static com.yahoo.security.SubjectAlternativeName.Type.DNS_NAME; */ class NodeIdentifier { - static final String TENANT_DOCKER_HOST_IDENTITY = "vespa.vespa.tenant-host"; - static final String PROXY_HOST_IDENTITY = "vespa.vespa.proxy"; - static final String CONFIGSERVER_HOST_IDENTITY = "vespa.vespa.configserver"; - static final String TENANT_DOCKER_CONTAINER_IDENTITY = "vespa.vespa.tenant"; - static final String ZTS_ON_PREM_IDENTITY = "zts.athens.yahoo.com"; - static final String ZTS_AWS_IDENTITY = "zts.athenz.ouroath.com"; private static final String INSTANCE_ID_DELIMITER = ".instanceid.athenz."; private final Zone zone; private final NodeRepository nodeRepository; - + private final String athenzProviderHostname; + private final Set<String> configServerLikeIdentities; + private final Set<String> tenantAndProxyHostIndentities; + private final String tenantIdentity; private final Supplier<List<Node>> nodeCache; - NodeIdentifier(Zone zone, NodeRepository nodeRepository) { + NodeIdentifier(Zone zone, NodeRepository nodeRepository, ConfigServerSecurityConfig securityConfig) { this.zone = zone; this.nodeRepository = nodeRepository; + this.athenzProviderHostname = securityConfig.athenzProviderHostname(); + this.configServerLikeIdentities = Set.of(securityConfig.controllerHostIdentity(), securityConfig.configServerHostIdentity()); + this.tenantAndProxyHostIndentities = Set.of(securityConfig.tenantHostIdentity(), securityConfig.proxyHostIdentity()); + this.tenantIdentity = securityConfig.tenantIdentity(); nodeCache = Suppliers.memoizeWithExpiration(nodeRepository::getNodes, 1, TimeUnit.MINUTES); } @@ -52,17 +55,17 @@ class NodeIdentifier { .orElseThrow(() -> new NodeIdentifierException("Certificate subject common name is missing!")); if (isAthenzIssued(clientCertificate)) { List<SubjectAlternativeName> sans = X509CertificateUtils.getSubjectAlternativeNames(clientCertificate); - switch (subjectCommonName) { - case TENANT_DOCKER_HOST_IDENTITY: - case PROXY_HOST_IDENTITY: - return NodePrincipal.withAthenzIdentity(subjectCommonName, getHostFromCalypsoOrAwsCertificate(sans), certificateChain); - case TENANT_DOCKER_CONTAINER_IDENTITY: - return NodePrincipal.withAthenzIdentity(subjectCommonName, getHostFromVespaCertificate(sans), certificateChain); - case CONFIGSERVER_HOST_IDENTITY: - default: - return NodePrincipal.withAthenzIdentity(subjectCommonName, certificateChain); + if (configServerLikeIdentities.contains(subjectCommonName)) { + return NodePrincipal.withAthenzIdentity(subjectCommonName, certificateChain); + } else if (tenantAndProxyHostIndentities.contains(subjectCommonName)) { + return NodePrincipal.withAthenzIdentity(subjectCommonName, getHostFromCalypsoCertificate(sans), certificateChain); + } else if (subjectCommonName.equals(tenantIdentity)) { + return NodePrincipal.withAthenzIdentity(subjectCommonName, getHostFromVespaCertificate(sans), certificateChain); } - } else if (subjectCommonName.equals(ZTS_ON_PREM_IDENTITY) || subjectCommonName.equals(ZTS_AWS_IDENTITY)) { + + throw new NodeIdentifierException(String.format( + "Subject common name (%s) does not match any expected identity", subjectCommonName)); + } else if (subjectCommonName.contains(athenzProviderHostname)) { // ZTS treated as a node principal even though its not a Vespa node return NodePrincipal.withLegacyIdentity(subjectCommonName, certificateChain); } else { @@ -79,11 +82,6 @@ class NodeIdentifier { return issuerCommonName.equals("Yahoo Athenz CA") || issuerCommonName.equals("Athenz AWS CA"); } - // NOTE: AWS instance id is currently stored as the attribute 'openstack-id' in node repository. - private String getHostFromCalypsoOrAwsCertificate(List<SubjectAlternativeName> sans) { - return getHostFromCalypsoCertificate(sans); - } - private String getHostFromCalypsoCertificate(List<SubjectAlternativeName> sans) { String openStackId = getUniqueInstanceId(sans); return nodeCache.get().stream() diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierFilter.java index 1ff8958a993..1c66c17a0bb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierFilter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierFilter.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.restapi.v2.filter; import com.google.inject.Inject; import com.yahoo.config.provision.Zone; +import com.yahoo.config.provisioning.ConfigServerSecurityConfig; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; @@ -29,8 +30,8 @@ public class NodeIdentifierFilter extends JsonSecurityRequestFilterBase { private final NodeIdentifier nodeIdentifier; @Inject - public NodeIdentifierFilter(Zone zone, NodeRepository nodeRepository) { - this.nodeIdentifier = new NodeIdentifier(zone, nodeRepository); + public NodeIdentifierFilter(Zone zone, NodeRepository nodeRepository, ConfigServerSecurityConfig securityConfig) { + this.nodeIdentifier = new NodeIdentifier(zone, nodeRepository, securityConfig); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 83f5d6bf783..b9fb88e900e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -20,7 +20,6 @@ import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.flag.FlagId; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Status; import com.yahoo.vespa.hosted.provision.provisioning.NodeRepositoryProvisioner; @@ -85,8 +84,10 @@ public class MockNodeRepository extends NodeRepository { nodes.add(createNode("node6", "host6.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.tenant)); - nodes.add(createNode("node7", "host7.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.tenant)); - // 8 and 9 are added by web service calls + Node node7 = createNode("node7", "host7.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.tenant); + nodes.add(node7); + + // 8, 9, 11 and 12 are added by web service calls Node node10 = createNode("node10", "host10.yahoo.com", ipAddresses, Optional.of("parent1.yahoo.com"), flavors.getFlavorOrThrow("default"), NodeType.tenant); Status node10newStatus = node10.status(); node10newStatus = node10newStatus @@ -95,6 +96,11 @@ public class MockNodeRepository extends NodeRepository { node10 = node10.with(node10newStatus); nodes.add(node10); + Node node13 = createNode("node13", "host13.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("large"), NodeType.tenant); + Node node14 = createNode("node14", "host14.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("large"), NodeType.tenant); + nodes.add(node13); + nodes.add(node14); + Node node55 = createNode("node55", "host55.yahoo.com", ipAddresses, Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.tenant); nodes.add(node55.with(node55.status().withWantToRetire(true).withWantToDeprovision(true))); @@ -109,18 +115,17 @@ public class MockNodeRepository extends NodeRepository { nodes.add(createNode("cfg1", "cfg1.yahoo.com", Collections.singleton("127.0.1.1"), Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.config)); nodes.add(createNode("cfg2", "cfg2.yahoo.com", Collections.singleton("127.0.1.2"), Optional.empty(), flavors.getFlavorOrThrow("default"), NodeType.config)); + // Ready all nodes, except 7 and 55 nodes = addNodes(nodes); - nodes.remove(6); - nodes.remove(7); + nodes.remove(node7); + nodes.remove(node55); nodes = setDirty(nodes, Agent.system, getClass().getSimpleName()); setReady(nodes, Agent.system, getClass().getSimpleName()); - fail("host5.yahoo.com", Agent.system, getClass().getSimpleName()); - dirtyRecursively("host55.yahoo.com", Agent.system, getClass().getSimpleName()); + fail(node5.hostname(), Agent.system, getClass().getSimpleName()); + dirtyRecursively(node55.hostname(), Agent.system, getClass().getSimpleName()); ApplicationId zoneApp = ApplicationId.from(TenantName.from("zoneapp"), ApplicationName.from("zoneapp"), InstanceName.from("zoneapp")); - // TODO: Remove this once feature flag is removed - this.flags().setEnabled(FlagId.exclusiveLoadBalancer, zoneApp, true); ClusterSpec zoneCluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin"), Version.fromString("6.42"), @@ -148,6 +153,13 @@ public class MockNodeRepository extends NodeRepository { Version.fromString("6.42"), false, Collections.emptySet()); activate(provisioner.prepare(app3, cluster3, Capacity.fromNodeCount(2, Optional.of("docker"), false, true), 1, null), app3, provisioner); + + ApplicationId app4 = ApplicationId.from(TenantName.from("tenant4"), ApplicationName.from("application4"), InstanceName.from("instance4")); + ClusterSpec cluster4 = ClusterSpec.request(ClusterSpec.Type.container, + ClusterSpec.Id.from("id4"), + Version.fromString("6.42"), + false, Collections.emptySet()); + activate(provisioner.prepare(app4, cluster4, Capacity.fromNodeCount(2, Optional.of("large"), false, true), 1, null), app4, provisioner); } private void activate(List<HostSpec> hosts, ApplicationId application, NodeRepositoryProvisioner provisioner) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/flag/FlagsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/flag/FlagsTest.java deleted file mode 100644 index 5018b18c491..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/flag/FlagsTest.java +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.flag; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; -import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors; -import com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository; -import org.junit.Test; - -import java.util.function.Supplier; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author mpolden - */ -public class FlagsTest { - - @Test - public void test_flag_toggling() { - NodeRepository nodeRepository = new MockNodeRepository(new MockCurator(), new MockNodeFlavors()); - Flags flags = nodeRepository.flags(); - Supplier<Flag> flag = () -> flags.get(FlagId.exclusiveLoadBalancer); - - // Flag is disabled by default - assertFalse(flag.get().isEnabled()); - - // Toggle flag for a node - { - HostName node1 = HostName.from("host1"); - flags.setEnabled(FlagId.exclusiveLoadBalancer, node1, true); - assertTrue(flag.get().isEnabled(node1)); - assertFalse(flag.get().isEnabled()); - flags.setEnabled(FlagId.exclusiveLoadBalancer, node1, false); - assertFalse(flag.get().isEnabled(node1)); - } - - // Toggle flag for an application - { - ApplicationId app1 = ApplicationId.from("tenant1", "application1", "default"); - flags.setEnabled(FlagId.exclusiveLoadBalancer, app1, true); - assertTrue(flag.get().isEnabled(app1)); - assertFalse(flag.get().isEnabled()); - flags.setEnabled(FlagId.exclusiveLoadBalancer, app1, false); - assertFalse(flag.get().isEnabled(app1)); - } - - // Toggle flag globally - { - flags.setEnabled(FlagId.exclusiveLoadBalancer, true); - assertTrue(flag.get().isEnabled()); - // Flag is implicitly enabled for all dimensions - assertTrue(flag.get().isEnabled(HostName.from("host1"))); - assertTrue(flag.get().isEnabled(ApplicationId.from("tenant1", "application1", "default"))); - flags.setEnabled(FlagId.exclusiveLoadBalancer, false); - assertFalse(flag.get().isEnabled()); - } - } - - -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java index 9870b94a8d5..52d297232de 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java @@ -6,7 +6,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.transaction.NestedTransaction; -import com.yahoo.vespa.hosted.provision.flag.FlagId; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -36,7 +35,6 @@ public class LoadBalancerExpirerTest { Duration.ofDays(1), new JobControl(tester.nodeRepository().database()), tester.loadBalancerService()); - tester.nodeRepository().flags().setEnabled(FlagId.exclusiveLoadBalancer, true); Supplier<Map<LoadBalancerId, LoadBalancer>> loadBalancers = () -> tester.nodeRepository().database().readLoadBalancers(); // Deploy two applications with load balancers diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/FlagSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/FlagSerializerTest.java deleted file mode 100644 index 15f2289d340..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/FlagSerializerTest.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.persistence; - -import com.google.common.collect.ImmutableSet; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.provision.flag.Flag; -import com.yahoo.vespa.hosted.provision.flag.FlagId; -import org.junit.Test; - -import java.util.Collections; - -import static org.junit.Assert.assertEquals; - -/** - * @author mpolden - */ -public class FlagSerializerTest { - - @Test - public void test_serialization() { - Flag flag = new Flag(FlagId.exclusiveLoadBalancer, true, - ImmutableSet.of("host1", "host2"), - Collections.singleton( - ApplicationId.from("tenant1", "application1", "default") - )); - Flag serialized = FlagSerializer.fromJson(FlagSerializer.toJson(flag)); - assertEquals(flag.id(), serialized.id()); - assertEquals(flag.isEnabled(), serialized.isEnabled()); - assertEquals(flag.hostnames(), serialized.hostnames()); - assertEquals(flag.applications(), serialized.applications()); - } - -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java index 46fd2183faa..a55211a112a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/AclProvisioningTest.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.flag.FlagId; import com.yahoo.vespa.hosted.provision.node.NodeAcl; import org.junit.Test; @@ -57,15 +56,9 @@ public class AclProvisioningTest { Supplier<List<NodeAcl>> nodeAcls = () -> tester.nodeRepository().getNodeAcls(node, false); // Trusted nodes is active nodes in same application, proxy nodes and config servers - assertAcls(Arrays.asList(activeNodes, proxyNodes, configServers, dockerHost), nodeAcls.get()); - - // Allocate load balancer - tester.nodeRepository().flags().setEnabled(FlagId.exclusiveLoadBalancer, application, true); - deploy(application, 2); - - // Load balancer networks are added to ACLs assertAcls(Arrays.asList(activeNodes, proxyNodes, configServers, dockerHost), - ImmutableSet.of("10.2.3.0/24", "10.4.5.0/24"), nodeAcls.get()); + ImmutableSet.of("10.2.3.0/24", "10.4.5.0/24"), + nodeAcls.get()); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index 2ed15b0a06f..4f7e09d0bd7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -10,7 +10,6 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.RotationName; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.flag.FlagId; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; import com.yahoo.vespa.hosted.provision.lb.Real; @@ -44,7 +43,6 @@ public class LoadBalancerProvisionerTest { ClusterSpec.Id containerCluster1 = ClusterSpec.Id.from("qrs1"); ClusterSpec.Id contentCluster = ClusterSpec.Id.from("content"); Set<RotationName> rotationsCluster1 = Set.of(RotationName.from("r1-1"), RotationName.from("r1-2")); - tester.nodeRepository().flags().setEnabled(FlagId.exclusiveLoadBalancer, true); tester.activate(app1, prepare(app1, clusterRequest(ClusterSpec.Type.container, containerCluster1, rotationsCluster1), clusterRequest(ClusterSpec.Type.content, contentCluster))); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java index 04664bd1d16..75995245274 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java @@ -49,7 +49,7 @@ public class RestApiTest { @After public void stopContainer() { - container.close(); + if (container != null) container.close(); } /** This test gives examples of all the requests that can be made to nodes/v2 */ @@ -76,17 +76,17 @@ public class RestApiTest { new byte[0], Request.Method.POST)); assertRestart(2, new Request("http://localhost:8080/nodes/v2/command/restart?application=tenant2.application2.instance2", new byte[0], Request.Method.POST)); - assertRestart(9, new Request("http://localhost:8080/nodes/v2/command/restart", + assertRestart(11, new Request("http://localhost:8080/nodes/v2/command/restart", new byte[0], Request.Method.POST)); assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), "\"restartGeneration\":3"); // POST reboot command - assertReboot(10, new Request("http://localhost:8080/nodes/v2/command/reboot?state=failed%20active", + assertReboot(12, new Request("http://localhost:8080/nodes/v2/command/reboot?state=failed%20active", new byte[0], Request.Method.POST)); assertReboot(2, new Request("http://localhost:8080/nodes/v2/command/reboot?application=tenant2.application2.instance2", new byte[0], Request.Method.POST)); - assertReboot(17, new Request("http://localhost:8080/nodes/v2/command/reboot", + assertReboot(19, new Request("http://localhost:8080/nodes/v2/command/reboot", new byte[0], Request.Method.POST)); assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), "\"rebootGeneration\":4"); @@ -778,49 +778,11 @@ public class RestApiTest { "{\"message\":\"Cancelled outstanding requests for firmware checks\"}"); } - @Test - public void test_flags() throws Exception { - assertFile(new Request("http://localhost:8080/nodes/v2/flags/"), "flags1.json"); - - // Enable flag for application - assertResponse(new Request("http://localhost:8080/nodes/v2/flags/exclusive-load-balancer/application/foo:bar:default", - new byte[0], Request.Method.POST), - "{\"message\":\"Enabled feature exclusiveLoadBalancer for application 'foo:bar:default'\"}"); - - // Enable flag for node - assertResponse(new Request("http://localhost:8080/nodes/v2/flags/exclusive-load-balancer/node/host1", - new byte[0], Request.Method.POST), - "{\"message\":\"Enabled feature exclusiveLoadBalancer for node 'host1'\"}"); - - assertFile(new Request("http://localhost:8080/nodes/v2/flags/"), "flags2.json"); - - // Enable flag for entire repository - assertResponse(new Request("http://localhost:8080/nodes/v2/flags/exclusive-load-balancer", - new byte[0], Request.Method.POST), - "{\"message\":\"Enabled feature exclusiveLoadBalancer\"}"); - - // Disable flag for application - assertResponse(new Request("http://localhost:8080/nodes/v2/flags/exclusive-load-balancer/application/foo:bar:default", - new byte[0], Request.Method.DELETE), - "{\"message\":\"Disabled feature exclusiveLoadBalancer for application 'foo:bar:default'\"}"); - - // Disable flag for node - assertResponse(new Request("http://localhost:8080/nodes/v2/flags/exclusive-load-balancer/node/host1", - new byte[0], Request.Method.DELETE), - "{\"message\":\"Disabled feature exclusiveLoadBalancer for node 'host1'\"}"); - - // Disable flag for entire repository - assertResponse(new Request("http://localhost:8080/nodes/v2/flags/exclusive-load-balancer", - new byte[0], Request.Method.DELETE), - "{\"message\":\"Disabled feature exclusiveLoadBalancer\"}"); - - } - /** Tests the rendering of each node separately to make it easier to find errors */ @Test public void test_single_node_rendering() throws Exception { - for (int i = 1; i <= 10; i++) { - if (i == 8 || i == 9) continue; // these nodes are added later + for (int i = 1; i <= 14; i++) { + if (i == 8 || i == 9 || i == 11 || i == 12) continue; // these nodes are added later assertFile(new Request("http://localhost:8080/nodes/v2/node/host" + i + ".yahoo.com"), "node" + i + ".json"); } } @@ -828,7 +790,7 @@ public class RestApiTest { @Test public void test_load_balancers() throws Exception { assertFile(new Request("http://localhost:8080/loadbalancers/v1/"), "load-balancers.json"); - assertFile(new Request("http://localhost:8080/loadbalancers/v1/?application=zoneapp.zoneapp.zoneapp"), "load-balancers.json"); + assertFile(new Request("http://localhost:8080/loadbalancers/v1/?application=tenant4.application4.instance4"), "load-balancers.json"); assertResponse(new Request("http://localhost:8080/loadbalancers/v1/?application=tenant.nonexistent.default"), "{\"loadBalancers\":[]}"); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilterTest.java index 2969b608d3a..3ea1570f770 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizationFilterTest.java @@ -2,15 +2,10 @@ package com.yahoo.vespa.hosted.provision.restapi.v2.filter; import com.yahoo.application.container.handler.Request.Method; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Zone; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.hosted.provision.restapi.v2.filter.FilterTester.Request; import com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors; import com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository; -import org.junit.Before; import org.junit.Test; /** @@ -18,12 +13,9 @@ import org.junit.Test; */ public class AuthorizationFilterTest { - private FilterTester tester; - - @Before - public void before() { - tester = filterTester(SystemName.main); - } + private final FilterTester tester = new FilterTester(new AuthorizationFilter( + new MockNodeRepository(new MockCurator(), new MockNodeFlavors()), + NodeIdentifierTest.SECURITY_CONFIG)); @Test public void filter() { @@ -43,11 +35,4 @@ public class AuthorizationFilterTest { tester.assertSuccess(new Request(Method.GET, "/nodes/v2/node/foo").commonName("foo")); } - private static FilterTester filterTester(SystemName system) { - Zone zone = new Zone(system, Environment.prod, RegionName.defaultName()); - return new FilterTester(new AuthorizationFilter( - zone, - new MockNodeRepository(new MockCurator(), new MockNodeFlavors()))); - } - } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizerTest.java index d696328cd7f..939de2dff25 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/AuthorizerTest.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.provision.restapi.v2.filter; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors; @@ -17,7 +16,11 @@ import java.util.List; import java.util.Optional; import java.util.Set; -import static java.util.Collections.emptyList; +import static com.yahoo.vespa.hosted.provision.restapi.v2.filter.NodeIdentifierTest.ATHENZ_PROVIDER_HOSTNAME; +import static com.yahoo.vespa.hosted.provision.restapi.v2.filter.NodeIdentifierTest.CONFIG_SERVER_IDENTITY; +import static com.yahoo.vespa.hosted.provision.restapi.v2.filter.NodeIdentifierTest.CONTROLLER_IDENTITY; +import static com.yahoo.vespa.hosted.provision.restapi.v2.filter.NodeIdentifierTest.SECURITY_CONFIG; +import static com.yahoo.vespa.hosted.provision.restapi.v2.filter.NodeIdentifierTest.TENANT_HOST_IDENTITY; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -27,13 +30,12 @@ import static org.junit.Assert.assertTrue; public class AuthorizerTest { private Authorizer authorizer; - private MockNodeRepository nodeRepository; @Before public void before() { NodeFlavors flavors = new MockNodeFlavors(); - nodeRepository = new MockNodeRepository(new MockCurator(), flavors); - authorizer = new Authorizer(SystemName.main, nodeRepository); + MockNodeRepository nodeRepository = new MockNodeRepository(new MockCurator(), flavors); + authorizer = new Authorizer(nodeRepository, SECURITY_CONFIG); Set<String> ipAddresses = Set.of("127.0.0.1", "::1"); Flavor flavor = flavors.getFlavorOrThrow("default"); @@ -98,13 +100,11 @@ public class AuthorizerTest { assertTrue(authorizedTenantHostNode("host1", "/athenz/v1/provider/identity-document/node/child1-1")); // Trusted services can access everything in their own system - assertFalse(authorizedController("vespa.vespa.cd.hosting", "/")); // Wrong system - assertTrue(new Authorizer(SystemName.cd, nodeRepository).test(NodePrincipal.withAthenzIdentity("vespa.vespa.cd.hosting", emptyList()), uri("/"))); - assertTrue(authorizedController("vespa.vespa.hosting", "/")); - assertTrue(authorizedController("vespa.vespa.configserver", "/")); - assertTrue(authorizedController("vespa.vespa.hosting", "/nodes/v2/node/")); - assertTrue(authorizedController("vespa.vespa.hosting", "/nodes/v2/node/node1")); - assertTrue(authorizedController("vespa.vespa.configserver", "/nodes/v2/node/node1")); + assertTrue(authorizedController(CONTROLLER_IDENTITY, "/")); + assertTrue(authorizedController(CONFIG_SERVER_IDENTITY, "/")); + assertTrue(authorizedController(CONTROLLER_IDENTITY, "/nodes/v2/node/")); + assertTrue(authorizedController(CONTROLLER_IDENTITY, "/nodes/v2/node/node1")); + assertTrue(authorizedController(CONFIG_SERVER_IDENTITY, "/nodes/v2/node/node1")); } @Test @@ -145,14 +145,14 @@ public class AuthorizerTest { assertTrue(authorizedTenantHostNode("proxy1-host", "/flags/v1/data")); assertFalse(authorizedTenantHostNode("proxy1-host", "/flags/v1/data/flagid")); assertFalse(authorizedTenantHostNode("proxy1-host", "/flags/v1/foo")); - assertTrue(authorizedController("vespa.vespa.configserver", "/flags/v1/data")); - assertFalse(authorizedController("vespa.vespa.configserver", "/flags/v1/data/flagid")); - assertFalse(authorizedController("vespa.vespa.configserver", "/flags/v1/foo")); + assertTrue(authorizedController(CONFIG_SERVER_IDENTITY, "/flags/v1/data")); + assertFalse(authorizedController(CONFIG_SERVER_IDENTITY, "/flags/v1/data/flagid")); + assertFalse(authorizedController(CONFIG_SERVER_IDENTITY, "/flags/v1/foo")); // Controller can access everything - assertTrue(authorizedController("vespa.vespa.hosting", "/flags/v1/data")); - assertTrue(authorizedController("vespa.vespa.hosting", "/flags/v1/data/flagid")); - assertTrue(authorizedController("vespa.vespa.hosting", "/flags/v1/foo")); + assertTrue(authorizedController(CONTROLLER_IDENTITY, "/flags/v1/data")); + assertTrue(authorizedController(CONTROLLER_IDENTITY, "/flags/v1/data/flagid")); + assertTrue(authorizedController(CONTROLLER_IDENTITY, "/flags/v1/foo")); } @Test @@ -165,24 +165,24 @@ public class AuthorizerTest { @Test public void zts_allowed_for_athenz_provider_api() { - assertTrue(authorizedLegacyNode(NodeIdentifier.ZTS_AWS_IDENTITY, "/athenz/v1/provider/refresh")); - assertTrue(authorizedLegacyNode(NodeIdentifier.ZTS_ON_PREM_IDENTITY, "/athenz/v1/provider/instance")); + assertTrue(authorizedLegacyNode(ATHENZ_PROVIDER_HOSTNAME, "/athenz/v1/provider/refresh")); + assertTrue(authorizedLegacyNode(ATHENZ_PROVIDER_HOSTNAME, "/athenz/v1/provider/instance")); } private boolean authorizedTenantNode(String hostname, String path) { - return authorized(NodePrincipal.withAthenzIdentity("vespa.vespa.tenant", hostname, emptyList()), path); + return authorized(NodePrincipal.withAthenzIdentity("vespa.vespa.tenant", hostname, List.of()), path); } private boolean authorizedTenantHostNode(String hostname, String path) { - return authorized(NodePrincipal.withAthenzIdentity("vespa.vespa.tenant-host", hostname, emptyList()), path); + return authorized(NodePrincipal.withAthenzIdentity(TENANT_HOST_IDENTITY, hostname, List.of()), path); } private boolean authorizedLegacyNode(String hostname, String path) { - return authorized(NodePrincipal.withLegacyIdentity(hostname, emptyList()), path); + return authorized(NodePrincipal.withLegacyIdentity(hostname, List.of()), path); } private boolean authorizedController(String controllerIdentity, String path) { - return authorized(NodePrincipal.withAthenzIdentity(controllerIdentity, emptyList()), path); + return authorized(NodePrincipal.withAthenzIdentity(controllerIdentity, List.of()), path); } private boolean authorized(NodePrincipal principal, String path) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierTest.java index cc23be4ddb4..2a9a1c4fa3b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierTest.java @@ -11,6 +11,7 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; +import com.yahoo.config.provisioning.ConfigServerSecurityConfig; import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.security.KeyUtils; import com.yahoo.security.Pkcs10Csr; @@ -37,11 +38,6 @@ import java.util.Optional; import static com.yahoo.security.KeyAlgorithm.EC; import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_ECDSA; import static com.yahoo.vespa.athenz.identityprovider.api.IdentityType.NODE; -import static com.yahoo.vespa.hosted.provision.restapi.v2.filter.NodeIdentifier.CONFIGSERVER_HOST_IDENTITY; -import static com.yahoo.vespa.hosted.provision.restapi.v2.filter.NodeIdentifier.PROXY_HOST_IDENTITY; -import static com.yahoo.vespa.hosted.provision.restapi.v2.filter.NodeIdentifier.TENANT_DOCKER_CONTAINER_IDENTITY; -import static com.yahoo.vespa.hosted.provision.restapi.v2.filter.NodeIdentifier.TENANT_DOCKER_HOST_IDENTITY; -import static com.yahoo.vespa.hosted.provision.restapi.v2.filter.NodeIdentifier.ZTS_AWS_IDENTITY; import static java.util.Collections.emptySet; import static java.util.Collections.singleton; import static java.util.Collections.singletonList; @@ -57,7 +53,20 @@ public class NodeIdentifierTest { @Rule public final ExpectedException expectedException = ExpectedException.none(); - private static final String CONTROLLER_IDENTITY = "vespa.vespa.hosting"; + static final String ATHENZ_PROVIDER_HOSTNAME = "zts.domain.tld"; + static final String CONTROLLER_IDENTITY = "vespa.controller"; + static final String CONFIG_SERVER_IDENTITY = "vespa.configserver"; + static final String PROXY_HOST_IDENTITY = "vespa.proxy"; + static final String TENANT_HOST_IDENTITY = "vespa.tenant-host"; + static final String TENANT_IDENTITY = "vespa.tenant"; + static final ConfigServerSecurityConfig SECURITY_CONFIG = new ConfigServerSecurityConfig.Builder() + .athenzProviderHostname(ATHENZ_PROVIDER_HOSTNAME) + .controllerHostIdentity(CONTROLLER_IDENTITY) + .configServerHostIdentity(CONFIG_SERVER_IDENTITY) + .proxyHostIdentity(PROXY_HOST_IDENTITY) + .tenantHostIdentity(TENANT_HOST_IDENTITY) + .tenantIdentity(TENANT_IDENTITY) + .build(); private static final String HOSTNAME = "myhostname"; private static final String PROXY_HOSTNAME = "myproxyhostname"; @@ -78,7 +87,7 @@ public class NodeIdentifierTest { .fromKeypair( KEYPAIR, new X500Principal("CN=" + HOSTNAME), Instant.EPOCH, Instant.EPOCH.plusSeconds(60), SHA256_WITH_ECDSA, BigInteger.ONE) .build(); - NodeIdentifier identifier = new NodeIdentifier(ZONE, nodeRepositoryDummy.nodeRepository()); + NodeIdentifier identifier = new NodeIdentifier(ZONE, nodeRepositoryDummy.nodeRepository(), SECURITY_CONFIG); expectedException.expect(NodeIdentifier.NodeIdentifierException.class); expectedException.expectMessage("(subject=myhostname, issuer=[myhostname])"); identifier.resolveNode(singletonList(certificate)); @@ -90,17 +99,17 @@ public class NodeIdentifierTest { nodeRepositoryDummy.addNode(OPENSTACK_ID, HOSTNAME, INSTANCE_ID, NodeType.host); nodeRepositoryDummy.setNodeState(HOSTNAME, Node.State.active); Pkcs10Csr csr = Pkcs10CsrBuilder - .fromKeypair(new X500Principal("CN=" + TENANT_DOCKER_HOST_IDENTITY), KEYPAIR, SHA256_WITH_ECDSA) + .fromKeypair(new X500Principal("CN=" + TENANT_HOST_IDENTITY), KEYPAIR, SHA256_WITH_ECDSA) .build(); X509Certificate certificate = X509CertificateBuilder .fromCsr(csr, ATHENZ_YAHOO_CA_CERT.getSubjectX500Principal(), Instant.EPOCH, Instant.EPOCH.plusSeconds(60), KEYPAIR.getPrivate(), SHA256_WITH_ECDSA, BigInteger.ONE) .addSubjectAlternativeName(OPENSTACK_ID + ".instanceid.athenz.provider-name.ostk.yahoo.cloud") .build(); - NodeIdentifier identifier = new NodeIdentifier(ZONE, nodeRepositoryDummy.nodeRepository()); + NodeIdentifier identifier = new NodeIdentifier(ZONE, nodeRepositoryDummy.nodeRepository(), SECURITY_CONFIG); NodePrincipal identity = identifier.resolveNode(singletonList(certificate)); assertTrue(identity.getHostname().isPresent()); assertEquals(HOSTNAME, identity.getHostname().get()); - assertEquals(TENANT_DOCKER_HOST_IDENTITY, identity.getHostIdentityName()); + assertEquals(TENANT_HOST_IDENTITY, identity.getHostIdentityName()); } @Test @@ -109,17 +118,17 @@ public class NodeIdentifierTest { nodeRepositoryDummy.addNode(AWS_INSTANCE_ID, HOSTNAME, INSTANCE_ID, NodeType.host); nodeRepositoryDummy.setNodeState(HOSTNAME, Node.State.active); Pkcs10Csr csr = Pkcs10CsrBuilder - .fromKeypair(new X500Principal("CN=" + TENANT_DOCKER_HOST_IDENTITY), KEYPAIR, SHA256_WITH_ECDSA) + .fromKeypair(new X500Principal("CN=" + TENANT_HOST_IDENTITY), KEYPAIR, SHA256_WITH_ECDSA) .build(); X509Certificate certificate = X509CertificateBuilder .fromCsr(csr, ATHENZ_AWS_CA_CERT.getSubjectX500Principal(), Instant.EPOCH, Instant.EPOCH.plusSeconds(60), KEYPAIR.getPrivate(), SHA256_WITH_ECDSA, BigInteger.ONE) .addSubjectAlternativeName(AWS_INSTANCE_ID + ".instanceid.athenz.aws.oath.cloud") .build(); - NodeIdentifier identifier = new NodeIdentifier(ZONE, nodeRepositoryDummy.nodeRepository()); + NodeIdentifier identifier = new NodeIdentifier(ZONE, nodeRepositoryDummy.nodeRepository(), SECURITY_CONFIG); NodePrincipal identity = identifier.resolveNode(singletonList(certificate)); assertTrue(identity.getHostname().isPresent()); assertEquals(HOSTNAME, identity.getHostname().get()); - assertEquals(TENANT_DOCKER_HOST_IDENTITY, identity.getHostIdentityName()); + assertEquals(TENANT_HOST_IDENTITY, identity.getHostIdentityName()); } @Test @@ -134,7 +143,7 @@ public class NodeIdentifierTest { .fromCsr(csr, ATHENZ_AWS_CA_CERT.getSubjectX500Principal(), Instant.EPOCH, Instant.EPOCH.plusSeconds(60), KEYPAIR.getPrivate(), SHA256_WITH_ECDSA, BigInteger.ONE) .addSubjectAlternativeName(AWS_INSTANCE_ID + ".instanceid.athenz.aws.oath.cloud") .build(); - NodeIdentifier identifier = new NodeIdentifier(ZONE, nodeRepositoryDummy.nodeRepository()); + NodeIdentifier identifier = new NodeIdentifier(ZONE, nodeRepositoryDummy.nodeRepository(), SECURITY_CONFIG); NodePrincipal identity = identifier.resolveNode(singletonList(certificate)); assertTrue(identity.getHostname().isPresent()); assertEquals(PROXY_HOSTNAME, identity.getHostname().get()); @@ -145,25 +154,25 @@ public class NodeIdentifierTest { public void accepts_aws_configserver_host_certificate() { NodeRepositoryTester nodeRepositoryDummy = new NodeRepositoryTester(); Pkcs10Csr csr = Pkcs10CsrBuilder - .fromKeypair(new X500Principal("CN=" + CONFIGSERVER_HOST_IDENTITY), KEYPAIR, SHA256_WITH_ECDSA) + .fromKeypair(new X500Principal("CN=" + CONFIG_SERVER_IDENTITY), KEYPAIR, SHA256_WITH_ECDSA) .build(); X509Certificate certificate = X509CertificateBuilder .fromCsr(csr, ATHENZ_AWS_CA_CERT.getSubjectX500Principal(), Instant.EPOCH, Instant.EPOCH.plusSeconds(60), KEYPAIR.getPrivate(), SHA256_WITH_ECDSA, BigInteger.ONE) .addSubjectAlternativeName(AWS_INSTANCE_ID + ".instanceid.athenz.aws.oath.cloud") .build(); - NodeIdentifier identifier = new NodeIdentifier(ZONE, nodeRepositoryDummy.nodeRepository()); + NodeIdentifier identifier = new NodeIdentifier(ZONE, nodeRepositoryDummy.nodeRepository(), SECURITY_CONFIG); NodePrincipal identity = identifier.resolveNode(singletonList(certificate)); - assertEquals(CONFIGSERVER_HOST_IDENTITY, identity.getHostIdentityName()); + assertEquals(CONFIG_SERVER_IDENTITY, identity.getHostIdentityName()); } @Test public void accepts_zts_certificate() { X509Certificate certificate = X509CertificateBuilder - .fromKeypair(KEYPAIR, new X500Principal("CN=" + ZTS_AWS_IDENTITY), Instant.EPOCH, Instant.EPOCH.plusSeconds(60), SHA256_WITH_ECDSA, BigInteger.ONE) + .fromKeypair(KEYPAIR, new X500Principal("CN=" + ATHENZ_PROVIDER_HOSTNAME), Instant.EPOCH, Instant.EPOCH.plusSeconds(60), SHA256_WITH_ECDSA, BigInteger.ONE) .build(); - NodeIdentifier identifier = new NodeIdentifier(ZONE, new NodeRepositoryTester().nodeRepository()); + NodeIdentifier identifier = new NodeIdentifier(ZONE, new NodeRepositoryTester().nodeRepository(), SECURITY_CONFIG); NodePrincipal identity = identifier.resolveNode(singletonList(certificate)); - assertEquals(ZTS_AWS_IDENTITY, identity.getHostIdentityName()); + assertEquals(ATHENZ_PROVIDER_HOSTNAME, identity.getHostIdentityName()); assertEquals(NodePrincipal.Type.LEGACY, identity.getType()); } @@ -179,18 +188,18 @@ public class NodeIdentifierTest { Node node = createNode(clusterId, clusterIndex, tenant, application); nodeRepositoryDummy.nodeRepository().addDockerNodes(singletonList(node), nodeRepositoryDummy.nodeRepository().lockAllocation()); Pkcs10Csr csr = Pkcs10CsrBuilder - .fromKeypair(new X500Principal("CN=" + TENANT_DOCKER_CONTAINER_IDENTITY), KEYPAIR, SHA256_WITH_ECDSA) + .fromKeypair(new X500Principal("CN=" + TENANT_IDENTITY), KEYPAIR, SHA256_WITH_ECDSA) .build(); VespaUniqueInstanceId vespaUniqueInstanceId = new VespaUniqueInstanceId(clusterIndex, clusterId, INSTANCE_ID, application, tenant, region, environment, NODE); X509Certificate certificate = X509CertificateBuilder .fromCsr(csr, ATHENZ_YAHOO_CA_CERT.getSubjectX500Principal(), Instant.EPOCH, Instant.EPOCH.plusSeconds(60), KEYPAIR.getPrivate(), SHA256_WITH_ECDSA, BigInteger.ONE) .addSubjectAlternativeName(vespaUniqueInstanceId.asDottedString() + ".instanceid.athenz.provider-name.vespa.yahoo.cloud") .build(); - NodeIdentifier identifier = new NodeIdentifier(ZONE, nodeRepositoryDummy.nodeRepository()); + NodeIdentifier identifier = new NodeIdentifier(ZONE, nodeRepositoryDummy.nodeRepository(), SECURITY_CONFIG); NodePrincipal identity = identifier.resolveNode(singletonList(certificate)); assertTrue(identity.getHostname().isPresent()); assertEquals(HOSTNAME, identity.getHostname().get()); - assertEquals(TENANT_DOCKER_CONTAINER_IDENTITY, identity.getHostIdentityName()); + assertEquals(TENANT_IDENTITY, identity.getHostIdentityName()); } @Test @@ -202,7 +211,7 @@ public class NodeIdentifierTest { X509Certificate certificate = X509CertificateBuilder .fromCsr(csr, ATHENZ_YAHOO_CA_CERT.getSubjectX500Principal(), Instant.EPOCH, Instant.EPOCH.plusSeconds(60), KEYPAIR.getPrivate(), SHA256_WITH_ECDSA, BigInteger.ONE) .build(); - NodeIdentifier identifier = new NodeIdentifier(ZONE, nodeRepositoryDummy.nodeRepository()); + NodeIdentifier identifier = new NodeIdentifier(ZONE, nodeRepositoryDummy.nodeRepository(), SECURITY_CONFIG); NodePrincipal identity = identifier.resolveNode(singletonList(certificate)); assertFalse(identity.getHostname().isPresent()); assertEquals(CONTROLLER_IDENTITY, identity.getHostIdentityName()); @@ -214,17 +223,17 @@ public class NodeIdentifierTest { nodeRepositoryDummy.addNode(OPENSTACK_ID, HOSTNAME, INSTANCE_ID, NodeType.tenant); nodeRepositoryDummy.setNodeState(HOSTNAME, Node.State.active); Pkcs10Csr csr = Pkcs10CsrBuilder - .fromKeypair(new X500Principal("CN=" + TENANT_DOCKER_CONTAINER_IDENTITY), KEYPAIR, SHA256_WITH_ECDSA) + .fromKeypair(new X500Principal("CN=" + TENANT_IDENTITY), KEYPAIR, SHA256_WITH_ECDSA) .build(); X509Certificate certificate = X509CertificateBuilder .fromCsr(csr, ATHENZ_YAHOO_CA_CERT.getSubjectX500Principal(), Instant.EPOCH, Instant.EPOCH.plusSeconds(60), KEYPAIR.getPrivate(), SHA256_WITH_ECDSA, BigInteger.ONE) .addSubjectAlternativeName(OPENSTACK_ID + ".instanceid.athenz.ostk.yahoo.cloud") .build(); - NodeIdentifier identifier = new NodeIdentifier(ZONE, nodeRepositoryDummy.nodeRepository()); + NodeIdentifier identifier = new NodeIdentifier(ZONE, nodeRepositoryDummy.nodeRepository(), SECURITY_CONFIG); NodePrincipal identity = identifier.resolveNode(singletonList(certificate)); assertTrue(identity.getHostname().isPresent()); assertEquals(HOSTNAME, identity.getHostname().get()); - assertEquals(TENANT_DOCKER_CONTAINER_IDENTITY, identity.getHostIdentityName()); + assertEquals(TENANT_IDENTITY, identity.getHostIdentityName()); } private static Node createNode(String clusterId, int clusterIndex, String tenant, String application) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-config-server.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-config-server.json index 92b5885cde4..05fa3e8ad72 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-config-server.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-config-server.json @@ -97,6 +97,30 @@ "trustedBy": "cfg1.yahoo.com" }, { + "hostname": "host13.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.0.1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host13.yahoo.com", + "type": "tenant", + "ipAddress": "::1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host14.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.0.1", + "trustedBy": "cfg1.yahoo.com" + }, + { + "hostname": "host14.yahoo.com", + "type": "tenant", + "ipAddress": "::1", + "trustedBy": "cfg1.yahoo.com" + }, + { "hostname": "host2.yahoo.com", "type": "tenant", "ipAddress": "127.0.0.1", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-tenant-node.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-tenant-node.json index 66d92ecc9fe..75c9aaa2b5c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-tenant-node.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-tenant-node.json @@ -43,6 +43,30 @@ "trustedBy": "foo.yahoo.com" }, { + "hostname": "host13.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.0.1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host13.yahoo.com", + "type": "tenant", + "ipAddress": "::1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host14.yahoo.com", + "type": "tenant", + "ipAddress": "127.0.0.1", + "trustedBy": "foo.yahoo.com" + }, + { + "hostname": "host14.yahoo.com", + "type": "tenant", + "ipAddress": "::1", + "trustedBy": "foo.yahoo.com" + }, + { "hostname": "host2.yahoo.com", "type": "tenant", "ipAddress": "127.0.0.1", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json index 9c462bb5520..a5d8148c1e4 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json @@ -1,7 +1,9 @@ { "nodes": [ @include(node6.json), + @include(node13.json), @include(node2.json), + @include(node14.json), @include(docker-container1.json), @include(node4.json), @include(docker-node4.json), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers.json index e3baa069907..c8738ab838c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers.json @@ -1,12 +1,12 @@ { "loadBalancers": [ { - "id": "zoneapp:zoneapp:zoneapp:node-admin", - "application": "zoneapp", - "tenant": "zoneapp", - "instance": "zoneapp", - "cluster": "node-admin", - "hostname": "lb-zoneapp.zoneapp.zoneapp-node-admin", + "id": "tenant4:application4:instance4:id4", + "application": "application4", + "tenant": "tenant4", + "instance": "instance4", + "cluster": "id4", + "hostname": "lb-tenant4.application4.instance4-id4", "dnsZone": "zone-id-1", "networks": [ "10.2.3.0/24", @@ -17,36 +17,17 @@ ], "reals": [ { - "hostname": "dockerhost1.yahoo.com", + "hostname": "host13.yahoo.com", "ipAddress": "127.0.0.1", "port": 4080 }, { - "hostname": "dockerhost2.yahoo.com", + "hostname": "host14.yahoo.com", "ipAddress": "127.0.0.1", "port": 4080 - }, - { - "hostname": "dockerhost3.yahoo.com", - "ipAddress": "127.0.0.1", - "port": 4080 - }, - { - "hostname": "dockerhost4.yahoo.com", - "ipAddress": "127.0.0.1", - "port": 4080 - }, - { - "hostname": "dockerhost5.yahoo.com", - "ipAddress": "127.0.0.1", - "port": 4080 - } - ], - "rotations": [ - { - "name": "us-cluster" } ], + "rotations": [], "inactive": false } ] diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node13.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node13.json new file mode 100644 index 00000000000..1584231f208 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node13.json @@ -0,0 +1,67 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host13.yahoo.com", + "id": "host13.yahoo.com", + "state": "active", + "type": "tenant", + "hostname": "host13.yahoo.com", + "openStackId": "node13", + "flavor": "large", + "canonicalFlavor": "large", + "minDiskAvailableGb": 1600.0, + "minMainMemoryAvailableGb": 32.0, + "description": "Flavor-name-is-large", + "minCpuCores": 4.0, + "fastDisk": true, + "bandwidth": 0.0, + "environment": "BARE_METAL", + "owner": { + "tenant": "tenant4", + "application": "application4", + "instance": "instance4" + }, + "membership": { + "clustertype": "container", + "clusterid": "id4", + "group": "0", + "index": 0, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "127.0.0.1", + "::1" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node14.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node14.json new file mode 100644 index 00000000000..462f6e2433f --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node14.json @@ -0,0 +1,67 @@ +{ + "url": "http://localhost:8080/nodes/v2/node/host14.yahoo.com", + "id": "host14.yahoo.com", + "state": "active", + "type": "tenant", + "hostname": "host14.yahoo.com", + "openStackId": "node14", + "flavor": "large", + "canonicalFlavor": "large", + "minDiskAvailableGb": 1600.0, + "minMainMemoryAvailableGb": 32.0, + "description": "Flavor-name-is-large", + "minCpuCores": 4.0, + "fastDisk": true, + "bandwidth": 0.0, + "environment": "BARE_METAL", + "owner": { + "tenant": "tenant4", + "application": "application4", + "instance": "instance4" + }, + "membership": { + "clustertype": "container", + "clusterid": "id4", + "group": "0", + "index": 1, + "retired": false + }, + "restartGeneration": 0, + "currentRestartGeneration": 0, + "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", + "wantedVespaVersion": "6.42.0", + "allowedToBeDown": false, + "rebootGeneration": 1, + "currentRebootGeneration": 0, + "failCount": 0, + "hardwareFailure": false, + "wantToRetire": false, + "wantToDeprovision": false, + "history": [ + { + "event": "provisioned", + "at": 123, + "agent": "system" + }, + { + "event": "readied", + "at": 123, + "agent": "system" + }, + { + "event": "reserved", + "at": 123, + "agent": "application" + }, + { + "event": "activated", + "at": 123, + "agent": "application" + } + ], + "ipAddresses": [ + "127.0.0.1", + "::1" + ], + "additionalIpAddresses": [] +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json index 57aee72e9f8..5f51f53fb2b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json @@ -10,9 +10,11 @@ @include(node6.json), @include(docker-node5.json), @include(docker-node2.json), + @include(node13.json), @include(node2.json), @include(docker-node1.json), @include(docker-node3.json), + @include(node14.json), @include(docker-container1.json), @include(node4.json), @include(node55.json), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json index 1c069951750..8f29a774fb4 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json @@ -31,6 +31,9 @@ "url": "http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com" }, { + "url": "http://localhost:8080/nodes/v2/node/host13.yahoo.com" + }, + { "url": "http://localhost:8080/nodes/v2/node/host2.yahoo.com" }, { @@ -40,6 +43,9 @@ "url": "http://localhost:8080/nodes/v2/node/dockerhost3.yahoo.com" }, { + "url": "http://localhost:8080/nodes/v2/node/host14.yahoo.com" + }, + { "url": "http://localhost:8080/nodes/v2/node/test-container-1" }, { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json index e60ebe37246..5d90346eec7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json @@ -25,7 +25,9 @@ "url": "http://localhost:8080/nodes/v2/state/active", "nodes": [ @include(node6.json), + @include(node13.json), @include(node2.json), + @include(node14.json), @include(docker-container1.json), @include(node4.json), @include(docker-node4.json), diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 7b27ba2fa3a..2f952f0957c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -450,17 +450,19 @@ Proton::~Proton() if (_fs4Server) { _fs4Server->shutDown(); } - // size_t numCores = _protonConfigurer.getActiveConfigSnapshot()->getBootstrapConfig()->getHwInfo().cpu().cores(); - size_t numCores = 4; - const std::shared_ptr<proton::ProtonConfigSnapshot> pcsp = _protonConfigurer.getActiveConfigSnapshot(); - if (pcsp) { - const std::shared_ptr<proton::BootstrapConfig> bcp = pcsp->getBootstrapConfig(); - if (bcp) { - numCores = bcp->getHwInfo().cpu().cores(); + if (_documentDBMap.size() > 0) { + size_t numCores = 4; + const std::shared_ptr<proton::ProtonConfigSnapshot> pcsp = _protonConfigurer.getActiveConfigSnapshot(); + if (pcsp) { + const std::shared_ptr<proton::BootstrapConfig> bcp = pcsp->getBootstrapConfig(); + if (bcp) { + numCores = std::max(bcp->getHwInfo().cpu().cores(), 1u); + } } + + vespalib::ThreadStackExecutor closePool(std::min(_documentDBMap.size(), numCores), 0x20000, close_executor); + closeDocumentDBs(closePool); } - vespalib::ThreadStackExecutor closePool(std::min(_documentDBMap.size(), numCores), 0x20000, close_executor); - closeDocumentDBs(closePool); _documentDBMap.clear(); _persistenceEngine.reset(); _tls.reset(); diff --git a/searchlib/src/tests/memoryindex/field_index/CMakeLists.txt b/searchlib/src/tests/memoryindex/field_index/CMakeLists.txt index 767097b99db..a09d6baf1a5 100644 --- a/searchlib/src/tests/memoryindex/field_index/CMakeLists.txt +++ b/searchlib/src/tests/memoryindex/field_index/CMakeLists.txt @@ -5,5 +5,15 @@ vespa_add_executable(searchlib_field_index_test_app TEST DEPENDS searchlib searchlib_test + gtest ) vespa_add_test(NAME searchlib_field_index_test_app COMMAND searchlib_field_index_test_app) + +vespa_add_executable(searchlib_field_index_iterator_test_app TEST + SOURCES + field_index_iterator_test.cpp + DEPENDS + searchlib + searchlib_test +) +vespa_add_test(NAME searchlib_field_index_iterator_test_app COMMAND searchlib_field_index_iterator_test_app) diff --git a/searchlib/src/tests/memoryindex/field_index/field_index_iterator_test.cpp b/searchlib/src/tests/memoryindex/field_index/field_index_iterator_test.cpp new file mode 100644 index 00000000000..df7f80e8601 --- /dev/null +++ b/searchlib/src/tests/memoryindex/field_index/field_index_iterator_test.cpp @@ -0,0 +1,73 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchcommon/common/schema.h> +#include <vespa/searchlib/memoryindex/field_index.h> +#include <vespa/searchlib/memoryindex/posting_iterator.h> +#include <vespa/searchlib/test/memoryindex/wrap_inserter.h> +#include <vespa/searchlib/test/searchiteratorverifier.h> +#include <vespa/vespalib/testkit/testapp.h> + +#include <vespa/log/log.h> +LOG_SETUP("field_index_iterator_test"); + +using namespace search::fef; +using namespace search::index; +using namespace search::memoryindex::test; +using namespace search::memoryindex; + +using search::index::schema::DataType; +using search::test::SearchIteratorVerifier; + +class Verifier : public SearchIteratorVerifier { +private: + mutable TermFieldMatchData _tfmd; + FieldIndex _field_index; + +public: + Verifier(const Schema& schema) + : _tfmd(), + _field_index(schema, 0) + { + WrapInserter inserter(_field_index); + inserter.word("a"); + for (uint32_t docId : getExpectedDocIds()) { + inserter.add(docId); + } + inserter.flush(); + } + ~Verifier() {} + + SearchIterator::UP create(bool strict) const override { + (void) strict; + TermFieldMatchDataArray match_data; + match_data.add(&_tfmd); + return std::make_unique<PostingIterator>(_field_index.find("a"), + _field_index.getFeatureStore(), 0, match_data); + } +}; + +Schema +get_schema() +{ + Schema result; + result.addIndexField(Schema::IndexField("f0", DataType::STRING)); + return result; +} + +struct Fixture { + Schema schema; + Verifier verifier; + Fixture() + : schema(get_schema()), + verifier(schema) + { + } +}; + +TEST_F("require that posting iterator conforms", Fixture) +{ + f.verifier.verify(); +} + +TEST_MAIN() { TEST_RUN_ALL(); } + diff --git a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp index 3a635756ec7..a63746548f0 100644 --- a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp +++ b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp @@ -15,11 +15,11 @@ #include <vespa/searchlib/memoryindex/field_inverter.h> #include <vespa/searchlib/memoryindex/ordered_field_index_inserter.h> #include <vespa/searchlib/memoryindex/posting_iterator.h> -#include <vespa/searchlib/test/searchiteratorverifier.h> -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/searchlib/test/memoryindex/wrap_inserter.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> -LOG_SETUP("dictionary_test"); +LOG_SETUP("field_index_test"); namespace search { @@ -32,13 +32,13 @@ using document::Document; using queryeval::SearchIterator; using search::index::schema::CollectionType; using search::index::schema::DataType; -using test::SearchIteratorVerifier; using vespalib::GenerationHandler; namespace memoryindex { -typedef FieldIndex::PostingList PostingList; -typedef PostingList::ConstIterator PostingConstItr; +using test::WrapInserter; +using PostingList = FieldIndex::PostingList; +using PostingConstItr = PostingList::ConstIterator; class MyBuilder : public IndexBuilder { private: @@ -52,8 +52,8 @@ private: bool _firstDoc; bool _firstElem; bool _firstPos; -public: +public: MyBuilder(const Schema &schema) : IndexBuilder(schema), _ss(), @@ -68,9 +68,7 @@ public: _firstPos(true) {} - virtual void - startWord(vespalib::stringref word) override - { + virtual void startWord(vespalib::stringref word) override { assert(_insideField); assert(!_insideWord); if (!_firstWord) @@ -80,9 +78,7 @@ public: _insideWord = true; } - virtual void - endWord() override - { + virtual void endWord() override { assert(_insideWord); assert(!_insideDoc); _ss << "]"; @@ -90,9 +86,7 @@ public: _insideWord = false; } - virtual void - startField(uint32_t fieldId) override - { + virtual void startField(uint32_t fieldId) override { assert(!_insideField); if (!_firstField) _ss << ","; _ss << "f=" << fieldId << "["; @@ -100,9 +94,7 @@ public: _insideField = true; } - virtual void - endField() override - { + virtual void endField() override { assert(_insideField); assert(!_insideWord); _ss << "]"; @@ -110,9 +102,7 @@ public: _insideField = false; } - virtual void - startDocument(uint32_t docId) override - { + virtual void startDocument(uint32_t docId) override { assert(_insideWord); assert(!_insideDoc); if (!_firstDoc) _ss << ","; @@ -121,9 +111,7 @@ public: _insideDoc = true; } - virtual void - endDocument() override - { + virtual void endDocument() override { assert(_insideDoc); assert(!_insideElem); _ss << "]"; @@ -131,11 +119,9 @@ public: _insideDoc = false; } - virtual void - startElement(uint32_t elementId, - int32_t weight, - uint32_t elementLen) override - { + virtual void startElement(uint32_t elementId, + int32_t weight, + uint32_t elementLen) override { assert(_insideDoc); assert(!_insideElem); if (!_firstElem) @@ -146,27 +132,21 @@ public: _insideElem = true; } - virtual void - endElement() override - { + virtual void endElement() override { assert(_insideElem); _ss << "]"; _firstElem = false; _insideElem = false; } - virtual void - addOcc(const WordDocElementWordPosFeatures &features) override - { + virtual void addOcc(const WordDocElementWordPosFeatures &features) override { assert(_insideElem); if (!_firstPos) _ss << ","; _ss << features.getWordPos(); _firstPos = false; } - std::string - toStr() const - { + std::string toStr() const { return _ss.str(); } }; @@ -186,8 +166,9 @@ toString(FieldPositionsIterator posItr, first = false; if (hasElements) { ss << "[e=" << posItr.getElementId(); - if (hasWeights) + if (hasWeights) { ss << ",w=" << posItr.getElementWeight(); + } ss << ",l=" << posItr.getElementLen() << "]"; } } @@ -198,10 +179,10 @@ toString(FieldPositionsIterator posItr, bool assertPostingList(const std::string &exp, PostingConstItr itr, - const FeatureStore *store = NULL) + const FeatureStore *store = nullptr) { std::stringstream ss; - FeatureStore::DecodeContextCooked decoder(NULL); + FeatureStore::DecodeContextCooked decoder(nullptr); TermFieldMatchData tfmd; TermFieldMatchDataArray matchData; matchData.add(&tfmd); @@ -210,7 +191,7 @@ assertPostingList(const std::string &exp, if (i > 0) ss << ","; uint32_t docId = itr.getKey(); ss << docId; - if (store != NULL) { // consider features as well + if (store != nullptr) { // consider features as well EntryRef ref(itr.getData()); store->setupForField(0, decoder); store->setupForUnpackFeatures(ref, decoder); @@ -219,7 +200,9 @@ assertPostingList(const std::string &exp, } } ss << "]"; - return EXPECT_EQUAL(exp, ss.str()); + bool result = (exp == ss.str()); + EXPECT_EQ(exp, ss.str()); + return result; } bool @@ -236,15 +219,13 @@ assertPostingList(std::vector<uint32_t> &exp, PostingConstItr itr) } -namespace -{ +namespace { /** * A simple mockup of a memory field index, used to verify * that we get correct posting lists from real memory field index. */ -class MockFieldIndex -{ +class MockFieldIndex { std::map<std::pair<vespalib::string, uint32_t>, std::set<uint32_t>> _dict; vespalib::string _word; uint32_t _fieldId; @@ -252,32 +233,23 @@ class MockFieldIndex public: ~MockFieldIndex(); void - setNextWord(const vespalib::string &word) - { + setNextWord(const vespalib::string &word) { _word = word; } - void - setNextField(uint32_t fieldId) - { + void setNextField(uint32_t fieldId) { _fieldId = fieldId; } - void - add(uint32_t docId) - { + void add(uint32_t docId) { _dict[std::make_pair(_word, _fieldId)].insert(docId); } - void - remove(uint32_t docId) - { + void remove(uint32_t docId) { _dict[std::make_pair(_word, _fieldId)].erase(docId); } - std::vector<uint32_t> - find(const vespalib::string &word, uint32_t fieldId) - { + std::vector<uint32_t> find(const vespalib::string &word, uint32_t fieldId) { std::vector<uint32_t> res; for (auto docId : _dict[std::make_pair(word, fieldId)] ) { res.push_back(docId); @@ -285,13 +257,11 @@ public: return res; } - auto begin() - { + auto begin() { return _dict.begin(); } - auto end() - { + auto end() { return _dict.end(); } }; @@ -303,8 +273,7 @@ MockFieldIndex::~MockFieldIndex() = default; * still stored safely in memory, to satisfy OrderedFieldIndexInserter * needs. */ -class MockWordStoreScan -{ +class MockWordStoreScan { vespalib::string _word0; vespalib::string _word1; vespalib::string *_prevWord; @@ -319,15 +288,11 @@ public: { } ~MockWordStoreScan(); - const vespalib::string & - getWord() const - { + const vespalib::string &getWord() const { return *_word; } - const vespalib::string & - setWord(const vespalib::string &word) - { + const vespalib::string &setWord(const vespalib::string &word) { std::swap(_prevWord, _word); *_word = word; return *_word; @@ -341,8 +306,7 @@ MockWordStoreScan::~MockWordStoreScan() = default; * and a real memory index. Mockup version is used to calculate expected * answers. */ -class MyInserter -{ +class MyInserter { MockWordStoreScan _wordStoreScan; MockFieldIndex _mock; FieldIndexCollection _fieldIndexes; @@ -361,17 +325,13 @@ public: } ~MyInserter(); - void - setNextWord(const vespalib::string &word) - { + void setNextWord(const vespalib::string &word) { const vespalib::string &w = _wordStoreScan.setWord(word); _inserter->setNextWord(w); _mock.setNextWord(w); } - void - setNextField(uint32_t fieldId) - { + void setNextField(uint32_t fieldId) { if (_inserter != nullptr) { _inserter->flush(); } @@ -380,32 +340,26 @@ public: _mock.setNextField(fieldId); } - void - add(uint32_t docId) - { + void add(uint32_t docId) { _inserter->add(docId, _features); _mock.add(docId); } - void - remove(uint32_t docId) - { + void remove(uint32_t docId) { _inserter->remove(docId); _mock.remove(docId); } - bool - assertPosting(const vespalib::string &word, - uint32_t fieldId) - { + bool assertPosting(const vespalib::string &word, + uint32_t fieldId) { std::vector<uint32_t> exp = _mock.find(word, fieldId); PostingConstItr itr = _fieldIndexes.find(word, fieldId); - return EXPECT_TRUE(assertPostingList(exp, itr)); + bool result = assertPostingList(exp, itr); + EXPECT_TRUE(result); + return result; } - bool - assertPostings() - { + bool assertPostings() { if (_inserter != nullptr) { _inserter->flush(); } @@ -413,25 +367,23 @@ public: auto &wf = wfp.first; auto &word = wf.first; auto fieldId = wf.second; - if (!EXPECT_TRUE(assertPosting(word, fieldId))) { + bool result = assertPosting(word, fieldId); + EXPECT_TRUE(result); + if (!result) { return false; } } return true; } - void - rewind() - { + void rewind() { if (_inserter != nullptr) { _inserter->flush(); _inserter = nullptr; } } - uint32_t - getNumUniqueWords() - { + uint32_t getNumUniqueWords() { return _fieldIndexes.getNumUniqueWords(); } @@ -439,6 +391,7 @@ public: }; MyInserter::~MyInserter() = default; + void myremove(uint32_t docId, DocumentInverter &inv, FieldIndexCollection &fieldIndexes, ISequencedTaskExecutor &invertThreads) @@ -448,63 +401,7 @@ myremove(uint32_t docId, DocumentInverter &inv, FieldIndexCollection &fieldIndex inv.pushDocuments(fieldIndexes, std::shared_ptr<IDestructorCallback>()); } - -class WrapInserter -{ - OrderedFieldIndexInserter &_inserter; -public: - WrapInserter(FieldIndexCollection &fieldIndexes, uint32_t fieldId) - : _inserter(fieldIndexes.getFieldIndex(fieldId)->getInserter()) - { - } - - WrapInserter &word(vespalib::stringref word_) - { - _inserter.setNextWord(word_); - return *this; - } - - WrapInserter &add(uint32_t docId, const index::DocIdAndFeatures &features) - { - _inserter.add(docId, features); - return *this; - } - - WrapInserter &add(uint32_t docId) - { - DocIdAndPosOccFeatures features; - features.addNextOcc(0, 0, 1, 1); - return add(docId, features); - } - - WrapInserter &remove(uint32_t docId) - { - _inserter.remove(docId); - return *this; - } - - WrapInserter &flush() - { - _inserter.flush(); - return *this; - } - - WrapInserter &rewind() - { - _inserter.rewind(); - return *this; - } - - datastore::EntryRef - getWordRef() - { - return _inserter.getWordRef(); - } -}; - - -class MyDrainRemoves : IFieldIndexRemoveListener -{ +class MyDrainRemoves : IFieldIndexRemoveListener { FieldIndexRemover &_remover; public: virtual void remove(const vespalib::stringref, uint32_t) override { } @@ -514,8 +411,12 @@ public: { } - void drain(uint32_t docId) + MyDrainRemoves(FieldIndex& field_index) + : _remover(field_index.getDocumentRemover()) { + } + + void drain(uint32_t docId) { _remover.remove(docId, *this); } }; @@ -526,7 +427,6 @@ myPushDocument(DocumentInverter &inv, FieldIndexCollection &fieldIndexes) inv.pushDocuments(fieldIndexes, std::shared_ptr<IDestructorCallback>()); } - const FeatureStore * featureStorePtr(const FieldIndexCollection &fieldIndexes, uint32_t fieldId) { @@ -539,7 +439,6 @@ featureStoreRef(const FieldIndexCollection &fieldIndexes, uint32_t fieldId) return fieldIndexes.getFieldIndex(fieldId)->getFeatureStore(); } - DataStoreBase::MemStats getFeatureStoreMemStats(const FieldIndexCollection &fieldIndexes) { @@ -553,8 +452,8 @@ getFeatureStoreMemStats(const FieldIndexCollection &fieldIndexes) return res; } - -void myCommit(FieldIndexCollection &fieldIndexes, ISequencedTaskExecutor &pushThreads) +void +myCommit(FieldIndexCollection &fieldIndexes, ISequencedTaskExecutor &pushThreads) { uint32_t fieldId = 0; for (auto &fieldIndex : fieldIndexes.getFieldIndexes()) { @@ -566,7 +465,6 @@ void myCommit(FieldIndexCollection &fieldIndexes, ISequencedTaskExecutor &pushTh pushThreads.sync(); } - void myCompactFeatures(FieldIndexCollection &fieldIndexes, ISequencedTaskExecutor &pushThreads) { @@ -581,57 +479,77 @@ myCompactFeatures(FieldIndexCollection &fieldIndexes, ISequencedTaskExecutor &pu } - -struct Fixture +Schema +make_single_field_schema() { - Schema _schema; - Fixture() : _schema() { - _schema.addIndexField(Schema::IndexField("f0", DataType::STRING)); - _schema.addIndexField(Schema::IndexField("f1", DataType::STRING)); - _schema.addIndexField(Schema::IndexField("f2", DataType::STRING, CollectionType::ARRAY)); - _schema.addIndexField(Schema::IndexField("f3", DataType::STRING, CollectionType::WEIGHTEDSET)); + Schema result; + result.addIndexField(Schema::IndexField("f0", DataType::STRING)); + return result; +} + +struct FieldIndexTest : public ::testing::Test { + Schema schema; + FieldIndex idx; + FieldIndexTest() + : schema(make_single_field_schema()), + idx(schema, 0) + { } - const Schema & getSchema() const { return _schema; } }; -// TODO: Rewrite most tests to use FieldIndex directly instead of going via FieldIndexCollection. +Schema +make_multi_field_schema() +{ + Schema result; + result.addIndexField(Schema::IndexField("f0", DataType::STRING)); + result.addIndexField(Schema::IndexField("f1", DataType::STRING)); + result.addIndexField(Schema::IndexField("f2", DataType::STRING, CollectionType::ARRAY)); + result.addIndexField(Schema::IndexField("f3", DataType::STRING, CollectionType::WEIGHTEDSET)); + return result; +} -TEST_F("requireThatFreshInsertWorks", Fixture) +struct FieldIndexCollectionTest : public ::testing::Test { + Schema schema; + FieldIndexCollection fic; + FieldIndexCollectionTest() + : schema(make_multi_field_schema()), + fic(schema) + { + } + ~FieldIndexCollectionTest() {} +}; + +TEST_F(FieldIndexTest, require_that_fresh_insert_works) { - FieldIndexCollection fic(f.getSchema()); - SequencedTaskExecutor pushThreads(2); - EXPECT_TRUE(assertPostingList("[]", fic.find("a", 0))); - EXPECT_TRUE(assertPostingList("[]", fic.findFrozen("a", 0))); - EXPECT_EQUAL(0u, fic.getNumUniqueWords()); - WrapInserter(fic, 0).word("a").add(10).flush(); - EXPECT_TRUE(assertPostingList("[10]", fic.find("a", 0))); - EXPECT_TRUE(assertPostingList("[]", fic.findFrozen("a", 0))); - myCommit(fic, pushThreads); - EXPECT_TRUE(assertPostingList("[10]", fic.findFrozen("a", 0))); - EXPECT_EQUAL(1u, fic.getNumUniqueWords()); + EXPECT_TRUE(assertPostingList("[]", idx.find("a"))); + EXPECT_TRUE(assertPostingList("[]", idx.findFrozen("a"))); + EXPECT_EQ(0u, idx.getNumUniqueWords()); + WrapInserter(idx).word("a").add(10).flush(); + EXPECT_TRUE(assertPostingList("[10]", idx.find("a"))); + EXPECT_TRUE(assertPostingList("[]", idx.findFrozen("a"))); + idx.commit(); + EXPECT_TRUE(assertPostingList("[10]", idx.findFrozen("a"))); + EXPECT_EQ(1u, idx.getNumUniqueWords()); } -TEST_F("requireThatAppendInsertWorks", Fixture) +TEST_F(FieldIndexTest, require_that_append_insert_works) { - FieldIndexCollection fic(f.getSchema()); - SequencedTaskExecutor pushThreads(2); - WrapInserter(fic, 0).word("a").add(10).flush().rewind(). - word("a").add(5).flush(); - EXPECT_TRUE(assertPostingList("[5,10]", fic.find("a", 0))); - EXPECT_TRUE(assertPostingList("[]", fic.findFrozen("a", 0))); - WrapInserter(fic, 0).rewind().word("a").add(20).flush(); - EXPECT_TRUE(assertPostingList("[5,10,20]", fic.find("a", 0))); - EXPECT_TRUE(assertPostingList("[]", fic.findFrozen("a", 0))); - myCommit(fic, pushThreads); - EXPECT_TRUE(assertPostingList("[5,10,20]", fic.findFrozen("a", 0))); + WrapInserter(idx).word("a").add(10).flush().rewind(). + word("a").add(5).flush(); + EXPECT_TRUE(assertPostingList("[5,10]", idx.find("a"))); + EXPECT_TRUE(assertPostingList("[]", idx.findFrozen("a"))); + WrapInserter(idx).rewind().word("a").add(20).flush(); + EXPECT_TRUE(assertPostingList("[5,10,20]", idx.find("a"))); + EXPECT_TRUE(assertPostingList("[]", idx.findFrozen("a"))); + idx.commit(); + EXPECT_TRUE(assertPostingList("[5,10,20]", idx.findFrozen("a"))); } -TEST_F("requireThatMultiplePostingListsCanExist", Fixture) +TEST_F(FieldIndexCollectionTest, require_that_multiple_posting_lists_across_multiple_fields_can_exist) { - FieldIndexCollection fic(f.getSchema()); WrapInserter(fic, 0).word("a").add(10).word("b").add(11).add(15).flush(); WrapInserter(fic, 1).word("a").add(5).word("b").add(12).flush(); - EXPECT_EQUAL(4u, fic.getNumUniqueWords()); + EXPECT_EQ(4u, fic.getNumUniqueWords()); EXPECT_TRUE(assertPostingList("[10]", fic.find("a", 0))); EXPECT_TRUE(assertPostingList("[5]", fic.find("a", 1))); EXPECT_TRUE(assertPostingList("[11,15]", fic.find("b", 0))); @@ -640,28 +558,27 @@ TEST_F("requireThatMultiplePostingListsCanExist", Fixture) EXPECT_TRUE(assertPostingList("[]", fic.find("c", 0))); } -TEST_F("requireThatRemoveWorks", Fixture) +TEST_F(FieldIndexTest, require_that_remove_works) { - FieldIndexCollection fic(f.getSchema()); - WrapInserter(fic, 0).word("a").remove(10).flush(); - EXPECT_TRUE(assertPostingList("[]", fic.find("a", 0))); - WrapInserter(fic, 0).add(10).add(20).add(30).flush(); - EXPECT_TRUE(assertPostingList("[10,20,30]", fic.find("a", 0))); - WrapInserter(fic, 0).rewind().word("a").remove(10).flush(); - EXPECT_TRUE(assertPostingList("[20,30]", fic.find("a", 0))); - WrapInserter(fic, 0).remove(20).flush(); - EXPECT_TRUE(assertPostingList("[30]", fic.find("a", 0))); - WrapInserter(fic, 0).remove(30).flush(); - EXPECT_TRUE(assertPostingList("[]", fic.find("a", 0))); - EXPECT_EQUAL(1u, fic.getNumUniqueWords()); - MyDrainRemoves(fic, 0).drain(10); - WrapInserter(fic, 0).rewind().word("a").add(10).flush(); - EXPECT_TRUE(assertPostingList("[10]", fic.find("a", 0))); + WrapInserter(idx).word("a").remove(10).flush(); + EXPECT_TRUE(assertPostingList("[]", idx.find("a"))); + WrapInserter(idx).add(10).add(20).add(30).flush(); + EXPECT_TRUE(assertPostingList("[10,20,30]", idx.find("a"))); + WrapInserter(idx).rewind().word("a").remove(10).flush(); + EXPECT_TRUE(assertPostingList("[20,30]", idx.find("a"))); + WrapInserter(idx).remove(20).flush(); + EXPECT_TRUE(assertPostingList("[30]", idx.find("a"))); + WrapInserter(idx).remove(30).flush(); + EXPECT_TRUE(assertPostingList("[]", idx.find("a"))); + EXPECT_EQ(1u, idx.getNumUniqueWords()); + MyDrainRemoves(idx).drain(10); + WrapInserter(idx).rewind().word("a").add(10).flush(); + EXPECT_TRUE(assertPostingList("[10]", idx.find("a"))); } -TEST_F("requireThatMultipleInsertAndRemoveWorks", Fixture) +TEST_F(FieldIndexCollectionTest, require_that_multiple_insert_and_remove_works) { - MyInserter inserter(f.getSchema()); + MyInserter inserter(schema); uint32_t numFields = 4; for (uint32_t fi = 0; fi < numFields; ++fi) { inserter.setNextField(fi); @@ -671,8 +588,8 @@ TEST_F("requireThatMultipleInsertAndRemoveWorks", Fixture) for (uint32_t di = 0; di < (uint32_t) w; ++di) { // insert inserter.add(di * 3); } - EXPECT_EQUAL((w - 'a' + 1u) + ('z' - 'a' +1u) * fi, - inserter.getNumUniqueWords()); + EXPECT_EQ((w - 'a' + 1u) + ('z' - 'a' +1u) * fi, + inserter.getNumUniqueWords()); } } EXPECT_TRUE(inserter.assertPostings()); @@ -724,9 +641,8 @@ getFeatures(uint32_t elemLen, uint32_t numOccs, int32_t weight = 1) return f; } -TEST_F("requireThatFeaturesAreInPostingLists", Fixture) +TEST_F(FieldIndexCollectionTest, require_that_features_are_in_posting_lists) { - FieldIndexCollection fic(f.getSchema()); WrapInserter(fic, 0).word("a").add(1, getFeatures(4, 2)).flush(); EXPECT_TRUE(assertPostingList("[1{4:0,1}]", fic.find("a", 0), @@ -742,47 +658,9 @@ TEST_F("requireThatFeaturesAreInPostingLists", Fixture) featureStorePtr(fic, 1))); } -class Verifier : public SearchIteratorVerifier { -public: - Verifier(const Schema & schema); - ~Verifier(); - - SearchIterator::UP create(bool strict) const override { - (void) strict; - TermFieldMatchDataArray matchData; - matchData.add(&_tfmd); - return std::make_unique<PostingIterator>(_fieldIndexes.find("a", 0), featureStoreRef(_fieldIndexes, 0), 0, matchData); - } - -private: - mutable TermFieldMatchData _tfmd; - FieldIndexCollection _fieldIndexes; -}; - - -Verifier::Verifier(const Schema & schema) - : _tfmd(), - _fieldIndexes(schema) -{ - WrapInserter inserter(_fieldIndexes, 0); - inserter.word("a"); - for (uint32_t docId : getExpectedDocIds()) { - inserter.add(docId); - } - inserter.flush(); -} -Verifier::~Verifier() {} - -TEST_F("require that postingiterator conforms", Fixture) { - Verifier verifier(f.getSchema()); - verifier.verify(); - -} - -TEST_F("requireThatPostingIteratorIsWorking", Fixture) +TEST_F(FieldIndexTest, require_that_posting_iterator_is_working) { - FieldIndexCollection fic(f.getSchema()); - WrapInserter(fic, 0).word("a").add(10, getFeatures(4, 1)). + WrapInserter(idx).word("a").add(10, getFeatures(4, 1)). add(20, getFeatures(5, 2)). add(30, getFeatures(6, 1)). add(40, getFeatures(7, 2)).flush(); @@ -790,166 +668,167 @@ TEST_F("requireThatPostingIteratorIsWorking", Fixture) TermFieldMatchDataArray matchData; matchData.add(&tfmd); { - PostingIterator itr(fic.find("not", 0), - featureStoreRef(fic, 0), + PostingIterator itr(idx.find("not"), + idx.getFeatureStore(), 0, matchData); itr.initFullRange(); EXPECT_TRUE(itr.isAtEnd()); } { - PostingIterator itr(fic.find("a", 0), - featureStoreRef(fic, 0), + PostingIterator itr(idx.find("a"), + idx.getFeatureStore(), 0, matchData); itr.initFullRange(); - EXPECT_EQUAL(10u, itr.getDocId()); + EXPECT_EQ(10u, itr.getDocId()); itr.unpack(10); - EXPECT_EQUAL("{4:0}", toString(tfmd.getIterator())); + EXPECT_EQ("{4:0}", toString(tfmd.getIterator())); EXPECT_TRUE(!itr.seek(25)); - EXPECT_EQUAL(30u, itr.getDocId()); + EXPECT_EQ(30u, itr.getDocId()); itr.unpack(30); - EXPECT_EQUAL("{6:0}", toString(tfmd.getIterator())); + EXPECT_EQ("{6:0}", toString(tfmd.getIterator())); EXPECT_TRUE(itr.seek(40)); - EXPECT_EQUAL(40u, itr.getDocId()); + EXPECT_EQ(40u, itr.getDocId()); itr.unpack(40); - EXPECT_EQUAL("{7:0,1}", toString(tfmd.getIterator())); + EXPECT_EQ("{7:0,1}", toString(tfmd.getIterator())); EXPECT_TRUE(!itr.seek(41)); EXPECT_TRUE(itr.isAtEnd()); } } -TEST_F("requireThatDumpingToIndexBuilderIsWorking", Fixture) +TEST_F(FieldIndexCollectionTest, require_that_basic_dumping_to_index_builder_is_working) { - { - MyBuilder b(f.getSchema()); - WordDocElementWordPosFeatures wpf; - b.startField(4); - b.startWord("a"); - b.startDocument(2); - b.startElement(0, 10, 20); - wpf.setWordPos(1); - b.addOcc(wpf); - wpf.setWordPos(3); - b.addOcc(wpf); - b.endElement(); - b.endDocument(); - b.endWord(); - b.endField(); - EXPECT_EQUAL("f=4[w=a[d=2[e=0,w=10,l=20[1,3]]]]", b.toStr()); - } - { - FieldIndexCollection fic(f.getSchema()); - MyBuilder b(f.getSchema()); - DocIdAndFeatures df; - WrapInserter(fic, 1).word("a").add(5, getFeatures(2, 1)). + MyBuilder b(schema); + WordDocElementWordPosFeatures wpf; + b.startField(4); + b.startWord("a"); + b.startDocument(2); + b.startElement(0, 10, 20); + wpf.setWordPos(1); + b.addOcc(wpf); + wpf.setWordPos(3); + b.addOcc(wpf); + b.endElement(); + b.endDocument(); + b.endWord(); + b.endField(); + EXPECT_EQ("f=4[w=a[d=2[e=0,w=10,l=20[1,3]]]]", b.toStr()); +} + +TEST_F(FieldIndexCollectionTest, require_that_dumping_of_multiple_fields_to_index_builder_is_working) +{ + MyBuilder b(schema); + DocIdAndFeatures df; + WrapInserter(fic, 1).word("a").add(5, getFeatures(2, 1)). add(7, getFeatures(3, 2)). word("b").add(5, getFeatures(12, 2)).flush(); - df = getFeatures(4, 1); - addElement(df, 5, 2); - WrapInserter(fic, 2).word("a").add(5, df); - df = getFeatures(6, 1); - addElement(df, 7, 2); - WrapInserter(fic, 2).add(7, df).flush(); - - df = getFeatures(8, 1, 12); - addElement(df, 9, 2, 13); - WrapInserter(fic, 3).word("a").add(5, df); - df = getFeatures(10, 1, 14); - addElement(df, 11, 2, 15); - WrapInserter(fic, 3).add(7, df).flush(); - - fic.dump(b); + df = getFeatures(4, 1); + addElement(df, 5, 2); + WrapInserter(fic, 2).word("a").add(5, df); + df = getFeatures(6, 1); + addElement(df, 7, 2); + WrapInserter(fic, 2).add(7, df).flush(); + + df = getFeatures(8, 1, 12); + addElement(df, 9, 2, 13); + WrapInserter(fic, 3).word("a").add(5, df); + df = getFeatures(10, 1, 14); + addElement(df, 11, 2, 15); + WrapInserter(fic, 3).add(7, df).flush(); + + fic.dump(b); + + EXPECT_EQ("f=0[]," + "f=1[w=a[d=5[e=0,w=1,l=2[0]],d=7[e=0,w=1,l=3[0,1]]]," + "w=b[d=5[e=0,w=1,l=12[0,1]]]]," + "f=2[w=a[d=5[e=0,w=1,l=4[0],e=1,w=1,l=5[0,1]]," + "d=7[e=0,w=1,l=6[0],e=1,w=1,l=7[0,1]]]]," + "f=3[w=a[d=5[e=0,w=12,l=8[0],e=1,w=13,l=9[0,1]]," + "d=7[e=0,w=14,l=10[0],e=1,w=15,l=11[0,1]]]]", + b.toStr()); +} - EXPECT_EQUAL("f=0[]," - "f=1[w=a[d=5[e=0,w=1,l=2[0]],d=7[e=0,w=1,l=3[0,1]]]," - "w=b[d=5[e=0,w=1,l=12[0,1]]]]," - "f=2[w=a[d=5[e=0,w=1,l=4[0],e=1,w=1,l=5[0,1]]," - "d=7[e=0,w=1,l=6[0],e=1,w=1,l=7[0,1]]]]," - "f=3[w=a[d=5[e=0,w=12,l=8[0],e=1,w=13,l=9[0,1]]," - "d=7[e=0,w=14,l=10[0],e=1,w=15,l=11[0,1]]]]", - b.toStr()); - } - { // test word with no docs - FieldIndexCollection fic(f.getSchema()); - WrapInserter(fic, 0).word("a").add(2, getFeatures(2, 1)). +TEST_F(FieldIndexCollectionTest, require_that_dumping_words_with_no_docs_to_index_builder_is_working) +{ + WrapInserter(fic, 0).word("a").add(2, getFeatures(2, 1)). word("b").add(4, getFeatures(4, 1)).flush().rewind(). word("a").remove(2).flush(); - { - MyBuilder b(f.getSchema()); - fic.dump(b); - EXPECT_EQUAL("f=0[w=b[d=4[e=0,w=1,l=4[0]]]],f=1[],f=2[],f=3[]", - b.toStr()); - } - { - search::diskindex::IndexBuilder b(f.getSchema()); - b.setPrefix("dump"); - TuneFileIndexing tuneFileIndexing; - DummyFileHeaderContext fileHeaderContext; - b.open(5, 2, tuneFileIndexing, fileHeaderContext); - fic.dump(b); - b.close(); - } + { + MyBuilder b(schema); + fic.dump(b); + EXPECT_EQ("f=0[w=b[d=4[e=0,w=1,l=4[0]]]],f=1[],f=2[],f=3[]", + b.toStr()); + } + { + search::diskindex::IndexBuilder b(schema); + b.setPrefix("dump"); + TuneFileIndexing tuneFileIndexing; + DummyFileHeaderContext fileHeaderContext; + b.open(5, 2, tuneFileIndexing, fileHeaderContext); + fic.dump(b); + b.close(); } } - -template <typename FixtureBase> -class FieldIndexFixture : public FixtureBase -{ +class InverterTest : public ::testing::Test { public: - using FixtureBase::getSchema; + Schema _schema; FieldIndexCollection _fic; DocBuilder _b; SequencedTaskExecutor _invertThreads; SequencedTaskExecutor _pushThreads; DocumentInverter _inv; - FieldIndexFixture() - : FixtureBase(), - _fic(getSchema()), - _b(getSchema()), + InverterTest(const Schema& schema) + : _schema(schema), + _fic(_schema), + _b(_schema), _invertThreads(2), _pushThreads(2), - _inv(getSchema(), _invertThreads, _pushThreads) + _inv(_schema, _invertThreads, _pushThreads) { } }; +class BasicInverterTest : public InverterTest { +public: + BasicInverterTest() : InverterTest(make_multi_field_schema()) {} +}; -TEST_F("requireThatInversionIsWorking", FieldIndexFixture<Fixture>) +TEST_F(BasicInverterTest, require_that_inversion_is_working) { Document::UP doc; - f._b.startDocument("doc::10"); - f._b.startIndexField("f0"). + _b.startDocument("doc::10"); + _b.startIndexField("f0"). addStr("a").addStr("b").addStr("c").addStr("d"). endField(); - doc = f._b.endDocument(); - f._inv.invertDocument(10, *doc); - f._invertThreads.sync(); - myPushDocument(f._inv, f._fic); - f._pushThreads.sync(); - - f._b.startDocument("doc::20"); - f._b.startIndexField("f0"). + doc = _b.endDocument(); + _inv.invertDocument(10, *doc); + _invertThreads.sync(); + myPushDocument(_inv, _fic); + _pushThreads.sync(); + + _b.startDocument("doc::20"); + _b.startIndexField("f0"). addStr("a").addStr("a").addStr("b").addStr("c").addStr("d"). endField(); - doc = f._b.endDocument(); - f._inv.invertDocument(20, *doc); - f._invertThreads.sync(); - myPushDocument(f._inv, f._fic); - f._pushThreads.sync(); - - f._b.startDocument("doc::30"); - f._b.startIndexField("f0"). + doc = _b.endDocument(); + _inv.invertDocument(20, *doc); + _invertThreads.sync(); + myPushDocument(_inv, _fic); + _pushThreads.sync(); + + _b.startDocument("doc::30"); + _b.startIndexField("f0"). addStr("a").addStr("b").addStr("c").addStr("d"). addStr("e").addStr("f"). endField(); - f._b.startIndexField("f1"). + _b.startIndexField("f1"). addStr("\nw2").addStr("w").addStr("x"). addStr("\nw3").addStr("y").addStr("z"). endField(); - f._b.startIndexField("f2"). + _b.startIndexField("f2"). startElement(4). addStr("w").addStr("x"). endElement(). @@ -957,7 +836,7 @@ TEST_F("requireThatInversionIsWorking", FieldIndexFixture<Fixture>) addStr("y").addStr("z"). endElement(). endField(); - f._b.startIndexField("f3"). + _b.startIndexField("f3"). startElement(6). addStr("w").addStr("x"). endElement(). @@ -965,56 +844,56 @@ TEST_F("requireThatInversionIsWorking", FieldIndexFixture<Fixture>) addStr("y").addStr("z"). endElement(). endField(); - doc = f._b.endDocument(); - f._inv.invertDocument(30, *doc); - f._invertThreads.sync(); - myPushDocument(f._inv, f._fic); - f._pushThreads.sync(); - - f._b.startDocument("doc::40"); - f._b.startIndexField("f0"). + doc = _b.endDocument(); + _inv.invertDocument(30, *doc); + _invertThreads.sync(); + myPushDocument(_inv, _fic); + _pushThreads.sync(); + + _b.startDocument("doc::40"); + _b.startIndexField("f0"). addStr("a").addStr("a").addStr("b").addStr("c").addStr("a"). addStr("e").addStr("f"). endField(); - doc = f._b.endDocument(); - f._inv.invertDocument(40, *doc); - f._invertThreads.sync(); - myPushDocument(f._inv, f._fic); - f._pushThreads.sync(); - - f._b.startDocument("doc::999"); - f._b.startIndexField("f0"). + doc = _b.endDocument(); + _inv.invertDocument(40, *doc); + _invertThreads.sync(); + myPushDocument(_inv, _fic); + _pushThreads.sync(); + + _b.startDocument("doc::999"); + _b.startIndexField("f0"). addStr("this").addStr("is").addStr("_a_").addStr("test"). addStr("for").addStr("insertion").addStr("speed").addStr("with"). addStr("more").addStr("than").addStr("just").addStr("__a__"). addStr("few").addStr("words").addStr("present").addStr("in"). addStr("some").addStr("of").addStr("the").addStr("fields"). endField(); - f._b.startIndexField("f1"). + _b.startIndexField("f1"). addStr("the").addStr("other").addStr("field").addStr("also"). addStr("has").addStr("some").addStr("content"). endField(); - f._b.startIndexField("f2"). + _b.startIndexField("f2"). startElement(1). addStr("strange").addStr("things").addStr("here"). addStr("has").addStr("some").addStr("content"). endElement(). endField(); - f._b.startIndexField("f3"). + _b.startIndexField("f3"). startElement(3). addStr("not").addStr("a").addStr("weighty").addStr("argument"). endElement(). endField(); - doc = f._b.endDocument(); + doc = _b.endDocument(); for (uint32_t docId = 10000; docId < 20000; ++docId) { - f._inv.invertDocument(docId, *doc); - f._invertThreads.sync(); - myPushDocument(f._inv, f._fic); - f._pushThreads.sync(); + _inv.invertDocument(docId, *doc); + _invertThreads.sync(); + myPushDocument(_inv, _fic); + _pushThreads.sync(); } - f._pushThreads.sync(); - DataStoreBase::MemStats beforeStats = getFeatureStoreMemStats(f._fic); + _pushThreads.sync(); + DataStoreBase::MemStats beforeStats = getFeatureStoreMemStats(_fic); LOG(info, "Before feature compaction: allocElems=%zu, usedElems=%zu" ", deadElems=%zu, holdElems=%zu" @@ -1027,14 +906,14 @@ TEST_F("requireThatInversionIsWorking", FieldIndexFixture<Fixture>) beforeStats._freeBuffers, beforeStats._activeBuffers, beforeStats._holdBuffers); - myCompactFeatures(f._fic, f._pushThreads); + myCompactFeatures(_fic, _pushThreads); std::vector<std::unique_ptr<GenerationHandler::Guard>> guards; - for (auto &fieldIndex : f._fic.getFieldIndexes()) { + for (auto &fieldIndex : _fic.getFieldIndexes()) { guards.push_back(std::make_unique<GenerationHandler::Guard> (fieldIndex->takeGenerationGuard())); } - myCommit(f._fic, f._pushThreads); - DataStoreBase::MemStats duringStats = getFeatureStoreMemStats(f._fic); + myCommit(_fic, _pushThreads); + DataStoreBase::MemStats duringStats = getFeatureStoreMemStats(_fic); LOG(info, "During feature compaction: allocElems=%zu, usedElems=%zu" ", deadElems=%zu, holdElems=%zu" @@ -1048,8 +927,8 @@ TEST_F("requireThatInversionIsWorking", FieldIndexFixture<Fixture>) duringStats._activeBuffers, duringStats._holdBuffers); guards.clear(); - myCommit(f._fic, f._pushThreads); - DataStoreBase::MemStats afterStats = getFeatureStoreMemStats(f._fic); + myCommit(_fic, _pushThreads); + DataStoreBase::MemStats afterStats = getFeatureStoreMemStats(_fic); LOG(info, "After feature compaction: allocElems=%zu, usedElems=%zu" ", deadElems=%zu, holdElems=%zu" @@ -1067,116 +946,115 @@ TEST_F("requireThatInversionIsWorking", FieldIndexFixture<Fixture>) TermFieldMatchDataArray matchData; matchData.add(&tfmd); { - PostingIterator itr(f._fic.findFrozen("not", 0), featureStoreRef(f._fic, 0), 0, matchData); + PostingIterator itr(_fic.findFrozen("not", 0), featureStoreRef(_fic, 0), 0, matchData); itr.initFullRange(); EXPECT_TRUE(itr.isAtEnd()); } { - PostingIterator itr(f._fic.findFrozen("a", 0), featureStoreRef(f._fic, 0), 0, matchData); + PostingIterator itr(_fic.findFrozen("a", 0), featureStoreRef(_fic, 0), 0, matchData); itr.initFullRange(); - EXPECT_EQUAL(10u, itr.getDocId()); + EXPECT_EQ(10u, itr.getDocId()); itr.unpack(10); - EXPECT_EQUAL("{4:0}", toString(tfmd.getIterator())); + EXPECT_EQ("{4:0}", toString(tfmd.getIterator())); EXPECT_TRUE(!itr.seek(25)); - EXPECT_EQUAL(30u, itr.getDocId()); + EXPECT_EQ(30u, itr.getDocId()); itr.unpack(30); - EXPECT_EQUAL("{6:0}", toString(tfmd.getIterator())); + EXPECT_EQ("{6:0}", toString(tfmd.getIterator())); EXPECT_TRUE(itr.seek(40)); - EXPECT_EQUAL(40u, itr.getDocId()); + EXPECT_EQ(40u, itr.getDocId()); itr.unpack(40); - EXPECT_EQUAL("{7:0,1,4}", toString(tfmd.getIterator())); + EXPECT_EQ("{7:0,1,4}", toString(tfmd.getIterator())); EXPECT_TRUE(!itr.seek(41)); EXPECT_TRUE(itr.isAtEnd()); } { - PostingIterator itr(f._fic.findFrozen("x", 0), featureStoreRef(f._fic, 0), 0, matchData); + PostingIterator itr(_fic.findFrozen("x", 0), featureStoreRef(_fic, 0), 0, matchData); itr.initFullRange(); EXPECT_TRUE(itr.isAtEnd()); } { - PostingIterator itr(f._fic.findFrozen("x", 1), featureStoreRef(f._fic, 1), 1, matchData); + PostingIterator itr(_fic.findFrozen("x", 1), featureStoreRef(_fic, 1), 1, matchData); itr.initFullRange(); - EXPECT_EQUAL(30u, itr.getDocId()); + EXPECT_EQ(30u, itr.getDocId()); itr.unpack(30); - EXPECT_EQUAL("{6:2[e=0,w=1,l=6]}", toString(tfmd.getIterator(), true, true)); + EXPECT_EQ("{6:2[e=0,w=1,l=6]}", toString(tfmd.getIterator(), true, true)); } { - PostingIterator itr(f._fic.findFrozen("x", 2), featureStoreRef(f._fic, 2), 2, matchData); + PostingIterator itr(_fic.findFrozen("x", 2), featureStoreRef(_fic, 2), 2, matchData); itr.initFullRange(); - EXPECT_EQUAL(30u, itr.getDocId()); + EXPECT_EQ(30u, itr.getDocId()); itr.unpack(30); // weight is hardcoded to 1 for new style il doc array field - EXPECT_EQUAL("{2:1[e=0,w=1,l=2]}", toString(tfmd.getIterator(), true, true)); + EXPECT_EQ("{2:1[e=0,w=1,l=2]}", toString(tfmd.getIterator(), true, true)); } { - PostingIterator itr(f._fic.findFrozen("x", 3), featureStoreRef(f._fic, 3), 3, matchData); + PostingIterator itr(_fic.findFrozen("x", 3), featureStoreRef(_fic, 3), 3, matchData); itr.initFullRange(); - EXPECT_EQUAL(30u, itr.getDocId()); + EXPECT_EQ(30u, itr.getDocId()); itr.unpack(30); - EXPECT_EQUAL("{2:1[e=0,w=6,l=2]}", - toString(tfmd.getIterator(), true, true)); + EXPECT_EQ("{2:1[e=0,w=6,l=2]}", + toString(tfmd.getIterator(), true, true)); } } -TEST_F("requireThatInverterHandlesRemoveViaDocumentRemover", - FieldIndexFixture<Fixture>) +TEST_F(BasicInverterTest, require_that_inverter_handles_remove_via_document_remover) { Document::UP doc; - f._b.startDocument("doc::1"); - f._b.startIndexField("f0").addStr("a").addStr("b").endField(); - f._b.startIndexField("f1").addStr("a").addStr("c").endField(); - Document::UP doc1 = f._b.endDocument(); - f._inv.invertDocument(1, *doc1.get()); - f._invertThreads.sync(); - myPushDocument(f._inv, f._fic); - f._pushThreads.sync(); - - f._b.startDocument("doc::2"); - f._b.startIndexField("f0").addStr("b").addStr("c").endField(); - Document::UP doc2 = f._b.endDocument(); - f._inv.invertDocument(2, *doc2.get()); - f._invertThreads.sync(); - myPushDocument(f._inv, f._fic); - f._pushThreads.sync(); - - EXPECT_TRUE(assertPostingList("[1]", f._fic.find("a", 0))); - EXPECT_TRUE(assertPostingList("[1,2]", f._fic.find("b", 0))); - EXPECT_TRUE(assertPostingList("[2]", f._fic.find("c", 0))); - EXPECT_TRUE(assertPostingList("[1]", f._fic.find("a", 1))); - EXPECT_TRUE(assertPostingList("[1]", f._fic.find("c", 1))); - - myremove(1, f._inv, f._fic, f._invertThreads); - f._pushThreads.sync(); - - EXPECT_TRUE(assertPostingList("[]", f._fic.find("a", 0))); - EXPECT_TRUE(assertPostingList("[2]", f._fic.find("b", 0))); - EXPECT_TRUE(assertPostingList("[2]", f._fic.find("c", 0))); - EXPECT_TRUE(assertPostingList("[]", f._fic.find("a", 1))); - EXPECT_TRUE(assertPostingList("[]", f._fic.find("c", 1))); + _b.startDocument("doc::1"); + _b.startIndexField("f0").addStr("a").addStr("b").endField(); + _b.startIndexField("f1").addStr("a").addStr("c").endField(); + Document::UP doc1 = _b.endDocument(); + _inv.invertDocument(1, *doc1.get()); + _invertThreads.sync(); + myPushDocument(_inv, _fic); + _pushThreads.sync(); + + _b.startDocument("doc::2"); + _b.startIndexField("f0").addStr("b").addStr("c").endField(); + Document::UP doc2 = _b.endDocument(); + _inv.invertDocument(2, *doc2.get()); + _invertThreads.sync(); + myPushDocument(_inv, _fic); + _pushThreads.sync(); + + EXPECT_TRUE(assertPostingList("[1]", _fic.find("a", 0))); + EXPECT_TRUE(assertPostingList("[1,2]", _fic.find("b", 0))); + EXPECT_TRUE(assertPostingList("[2]", _fic.find("c", 0))); + EXPECT_TRUE(assertPostingList("[1]", _fic.find("a", 1))); + EXPECT_TRUE(assertPostingList("[1]", _fic.find("c", 1))); + + myremove(1, _inv, _fic, _invertThreads); + _pushThreads.sync(); + + EXPECT_TRUE(assertPostingList("[]", _fic.find("a", 0))); + EXPECT_TRUE(assertPostingList("[2]", _fic.find("b", 0))); + EXPECT_TRUE(assertPostingList("[2]", _fic.find("c", 0))); + EXPECT_TRUE(assertPostingList("[]", _fic.find("a", 1))); + EXPECT_TRUE(assertPostingList("[]", _fic.find("c", 1))); } -class UriFixture +Schema +make_uri_schema() { + Schema result; + result.addUriIndexFields(Schema::IndexField("iu", DataType::STRING)); + result.addUriIndexFields(Schema::IndexField("iau", DataType::STRING, CollectionType::ARRAY)); + result.addUriIndexFields(Schema::IndexField("iwu", DataType::STRING, CollectionType::WEIGHTEDSET)); + return result; +} + +class UriInverterTest : public InverterTest { public: - Schema _schema; - UriFixture() - : _schema() - { - _schema.addUriIndexFields(Schema::IndexField("iu", DataType::STRING)); - _schema.addUriIndexFields(Schema::IndexField("iau", DataType::STRING, CollectionType::ARRAY)); - _schema.addUriIndexFields(Schema::IndexField("iwu", DataType::STRING, CollectionType::WEIGHTEDSET)); - } - const Schema & getSchema() const { return _schema; } + UriInverterTest() : InverterTest(make_uri_schema()) {} }; - -TEST_F("requireThatUriIndexingIsWorking", FieldIndexFixture<UriFixture>) +TEST_F(UriInverterTest, require_that_uri_indexing_is_working) { Document::UP doc; - f._b.startDocument("doc::10"); - f._b.startIndexField("iu"). + _b.startDocument("doc::10"); + _b.startIndexField("iu"). startSubField("all"). addUrlTokenizedString("http://www.example.com:81/fluke?ab=2#4"). endSubField(). @@ -1199,7 +1077,7 @@ TEST_F("requireThatUriIndexingIsWorking", FieldIndexFixture<UriFixture>) addUrlTokenizedString("4"). endSubField(). endField(); - f._b.startIndexField("iau"). + _b.startIndexField("iau"). startElement(1). startSubField("all"). addUrlTokenizedString("http://www.example.com:82/fluke?ab=2#8"). @@ -1247,7 +1125,7 @@ TEST_F("requireThatUriIndexingIsWorking", FieldIndexFixture<UriFixture>) endSubField(). endElement(). endField(); - f._b.startIndexField("iwu"). + _b.startIndexField("iwu"). startElement(4). startSubField("all"). addUrlTokenizedString("http://www.example.com:83/fluke?ab=2#12"). @@ -1295,141 +1173,131 @@ TEST_F("requireThatUriIndexingIsWorking", FieldIndexFixture<UriFixture>) endSubField(). endElement(). endField(); - doc = f._b.endDocument(); - f._inv.invertDocument(10, *doc); - f._invertThreads.sync(); - myPushDocument(f._inv, f._fic); + doc = _b.endDocument(); + _inv.invertDocument(10, *doc); + _invertThreads.sync(); + myPushDocument(_inv, _fic); - f._pushThreads.sync(); + _pushThreads.sync(); TermFieldMatchData tfmd; TermFieldMatchDataArray matchData; matchData.add(&tfmd); { - uint32_t fieldId = f.getSchema().getIndexFieldId("iu"); - PostingIterator itr(f._fic.findFrozen("not", fieldId), - featureStoreRef(f._fic, fieldId), + uint32_t fieldId = _schema.getIndexFieldId("iu"); + PostingIterator itr(_fic.findFrozen("not", fieldId), + featureStoreRef(_fic, fieldId), fieldId, matchData); itr.initFullRange(); EXPECT_TRUE(itr.isAtEnd()); } { - uint32_t fieldId = f.getSchema().getIndexFieldId("iu"); - PostingIterator itr(f._fic.findFrozen("example", fieldId), - featureStoreRef(f._fic, fieldId), + uint32_t fieldId = _schema.getIndexFieldId("iu"); + PostingIterator itr(_fic.findFrozen("example", fieldId), + featureStoreRef(_fic, fieldId), fieldId, matchData); itr.initFullRange(); - EXPECT_EQUAL(10u, itr.getDocId()); + EXPECT_EQ(10u, itr.getDocId()); itr.unpack(10); - EXPECT_EQUAL("{9:2}", toString(tfmd.getIterator())); + EXPECT_EQ("{9:2}", toString(tfmd.getIterator())); EXPECT_TRUE(!itr.seek(25)); EXPECT_TRUE(itr.isAtEnd()); } { - uint32_t fieldId = f.getSchema().getIndexFieldId("iau"); - PostingIterator itr(f._fic.findFrozen("example", fieldId), - featureStoreRef(f._fic, fieldId), + uint32_t fieldId = _schema.getIndexFieldId("iau"); + PostingIterator itr(_fic.findFrozen("example", fieldId), + featureStoreRef(_fic, fieldId), fieldId, matchData); itr.initFullRange(); - EXPECT_EQUAL(10u, itr.getDocId()); + EXPECT_EQ(10u, itr.getDocId()); itr.unpack(10); - EXPECT_EQUAL("{9:2[e=0,l=9]}", - toString(tfmd.getIterator(), true, false)); + EXPECT_EQ("{9:2[e=0,l=9]}", + toString(tfmd.getIterator(), true, false)); EXPECT_TRUE(!itr.seek(25)); EXPECT_TRUE(itr.isAtEnd()); } { - uint32_t fieldId = f.getSchema().getIndexFieldId("iwu"); - PostingIterator itr(f._fic.findFrozen("example", fieldId), - featureStoreRef(f._fic, fieldId), + uint32_t fieldId = _schema.getIndexFieldId("iwu"); + PostingIterator itr(_fic.findFrozen("example", fieldId), + featureStoreRef(_fic, fieldId), fieldId, matchData); itr.initFullRange(); - EXPECT_EQUAL(10u, itr.getDocId()); + EXPECT_EQ(10u, itr.getDocId()); itr.unpack(10); - EXPECT_EQUAL("{9:2[e=0,w=4,l=9]}", - toString(tfmd.getIterator(), true, true)); + EXPECT_EQ("{9:2[e=0,w=4,l=9]}", + toString(tfmd.getIterator(), true, true)); EXPECT_TRUE(!itr.seek(25)); EXPECT_TRUE(itr.isAtEnd()); } { - search::diskindex::IndexBuilder dib(f.getSchema()); + search::diskindex::IndexBuilder dib(_schema); dib.setPrefix("urldump"); TuneFileIndexing tuneFileIndexing; DummyFileHeaderContext fileHeaderContext; - dib.open(11, f._fic.getNumUniqueWords(), tuneFileIndexing, + dib.open(11, _fic.getNumUniqueWords(), tuneFileIndexing, fileHeaderContext); - f._fic.dump(dib); + _fic.dump(dib); dib.close(); } } - -class SingleFieldFixture -{ +class CjkInverterTest : public InverterTest { public: - Schema _schema; - SingleFieldFixture() - : _schema() - { - _schema.addIndexField(Schema::IndexField("i", DataType::STRING)); - } - const Schema & getSchema() const { return _schema; } + CjkInverterTest() : InverterTest(make_single_field_schema()) {} }; -TEST_F("requireThatCjkIndexingIsWorking", FieldIndexFixture<SingleFieldFixture>) +TEST_F(CjkInverterTest, require_that_cjk_indexing_is_working) { Document::UP doc; - f._b.startDocument("doc::10"); - f._b.startIndexField("i"). + _b.startDocument("doc::10"); + _b.startIndexField("f0"). addStr("我就是那个"). setAutoSpace(false). addStr("大灰狼"). setAutoSpace(true). endField(); - doc = f._b.endDocument(); - f._inv.invertDocument(10, *doc); - f._invertThreads.sync(); - myPushDocument(f._inv, f._fic); + doc = _b.endDocument(); + _inv.invertDocument(10, *doc); + _invertThreads.sync(); + myPushDocument(_inv, _fic); - f._pushThreads.sync(); + _pushThreads.sync(); TermFieldMatchData tfmd; TermFieldMatchDataArray matchData; matchData.add(&tfmd); + uint32_t fieldId = _schema.getIndexFieldId("f0"); { - uint32_t fieldId = f.getSchema().getIndexFieldId("i"); - PostingIterator itr(f._fic.findFrozen("not", fieldId), - featureStoreRef(f._fic, fieldId), + PostingIterator itr(_fic.findFrozen("not", fieldId), + featureStoreRef(_fic, fieldId), fieldId, matchData); itr.initFullRange(); EXPECT_TRUE(itr.isAtEnd()); } { - uint32_t fieldId = f.getSchema().getIndexFieldId("i"); - PostingIterator itr(f._fic.findFrozen("我就" + PostingIterator itr(_fic.findFrozen("我就" "是那个", fieldId), - featureStoreRef(f._fic, fieldId), + featureStoreRef(_fic, fieldId), fieldId, matchData); itr.initFullRange(); - EXPECT_EQUAL(10u, itr.getDocId()); + EXPECT_EQ(10u, itr.getDocId()); itr.unpack(10); - EXPECT_EQUAL("{2:0}", toString(tfmd.getIterator())); + EXPECT_EQ("{2:0}", toString(tfmd.getIterator())); EXPECT_TRUE(!itr.seek(25)); EXPECT_TRUE(itr.isAtEnd()); } { - uint32_t fieldId = f.getSchema().getIndexFieldId("i"); - PostingIterator itr(f._fic.findFrozen("大灰" + PostingIterator itr(_fic.findFrozen("大灰" "狼", fieldId), - featureStoreRef(f._fic, fieldId), + featureStoreRef(_fic, fieldId), fieldId, matchData); itr.initFullRange(); - EXPECT_EQUAL(10u, itr.getDocId()); + EXPECT_EQ(10u, itr.getDocId()); itr.unpack(10); - EXPECT_EQUAL("{2:1}", toString(tfmd.getIterator())); + EXPECT_EQ("{2:1}", toString(tfmd.getIterator())); EXPECT_TRUE(!itr.seek(25)); EXPECT_TRUE(itr.isAtEnd()); } @@ -1441,80 +1309,74 @@ insertAndAssertTuple(const vespalib::string &word, uint32_t fieldId, uint32_t do { EntryRef wordRef = WrapInserter(dict, fieldId).rewind().word(word). add(docId).flush().getWordRef(); - EXPECT_EQUAL(word, - dict.getFieldIndex(fieldId)->getWordStore().getWord(wordRef)); + EXPECT_EQ(word, dict.getFieldIndex(fieldId)->getWordStore().getWord(wordRef)); MyDrainRemoves(dict, fieldId).drain(docId); } -TEST_F("require that insert tells which word ref that was inserted", Fixture) +TEST_F(FieldIndexCollectionTest, require_that_insert_tells_which_word_ref_that_was_inserted) { - FieldIndexCollection d(f.getSchema()); - insertAndAssertTuple("a", 1, 11, d); - insertAndAssertTuple("b", 1, 11, d); - insertAndAssertTuple("a", 2, 11, d); - - insertAndAssertTuple("a", 1, 22, d); - insertAndAssertTuple("b", 2, 22, d); - insertAndAssertTuple("c", 2, 22, d); + insertAndAssertTuple("a", 1, 11, fic); + insertAndAssertTuple("b", 1, 11, fic); + insertAndAssertTuple("a", 2, 11, fic); + + insertAndAssertTuple("a", 1, 22, fic); + insertAndAssertTuple("b", 2, 22, fic); + insertAndAssertTuple("c", 2, 22, fic); } -struct RemoverFixture : public Fixture -{ - FieldIndexCollection _fic; +struct RemoverTest : public FieldIndexCollectionTest { SequencedTaskExecutor _invertThreads; SequencedTaskExecutor _pushThreads; - RemoverFixture() - : - Fixture(), - _fic(getSchema()), - _invertThreads(2), - _pushThreads(2) + RemoverTest() + : FieldIndexCollectionTest(), + _invertThreads(2), + _pushThreads(2) { } void assertPostingLists(const vespalib::string &e1, const vespalib::string &e2, const vespalib::string &e3) { - EXPECT_TRUE(assertPostingList(e1, _fic.find("a", 1))); - EXPECT_TRUE(assertPostingList(e2, _fic.find("a", 2))); - EXPECT_TRUE(assertPostingList(e3, _fic.find("b", 1))); + EXPECT_TRUE(assertPostingList(e1, fic.find("a", 1))); + EXPECT_TRUE(assertPostingList(e2, fic.find("a", 2))); + EXPECT_TRUE(assertPostingList(e3, fic.find("b", 1))); } void remove(uint32_t docId) { - DocumentInverter inv(getSchema(), _invertThreads, _pushThreads); - myremove(docId, inv, _fic, _invertThreads); + DocumentInverter inv(schema, _invertThreads, _pushThreads); + myremove(docId, inv, fic, _invertThreads); _pushThreads.sync(); - EXPECT_FALSE(_fic.getFieldIndex(0u)->getDocumentRemover(). + EXPECT_FALSE(fic.getFieldIndex(0u)->getDocumentRemover(). getStore().get(docId).valid()); } }; -TEST_F("require that document remover can remove several documents", RemoverFixture) +TEST_F(RemoverTest, require_that_document_remover_can_remove_several_documents) { - WrapInserter(f._fic, 1).word("a").add(11).add(13).add(15). - word("b").add(11).add(15).flush(); - WrapInserter(f._fic, 2).word("a").add(11).add(13).flush(); - f.assertPostingLists("[11,13,15]", "[11,13]", "[11,15]"); + WrapInserter(fic, 1).word("a").add(11).add(13).add(15). + word("b").add(11).add(15).flush(); + WrapInserter(fic, 2).word("a").add(11).add(13).flush(); + assertPostingLists("[11,13,15]", "[11,13]", "[11,15]"); - f.remove(13); - f.assertPostingLists("[11,15]", "[11]", "[11,15]"); + remove(13); + assertPostingLists("[11,15]", "[11]", "[11,15]"); - f.remove(11); - f.assertPostingLists("[15]", "[]", "[15]"); + remove(11); + assertPostingLists("[15]", "[]", "[15]"); - f.remove(15); - f.assertPostingLists("[]", "[]", "[]"); + remove(15); + assertPostingLists("[]", "[]", "[]"); } -TEST_F("require that removal of non-existing document does not do anything", RemoverFixture) +TEST_F(RemoverTest, require_that_removal_of_non_existing_document_does_not_do_anything) { - WrapInserter(f._fic, 1).word("a").add(11).word("b").add(11).flush(); - WrapInserter(f._fic, 2).word("a").add(11).flush(); - f.assertPostingLists("[11]", "[11]", "[11]"); - f.remove(13); - f.assertPostingLists("[11]", "[11]", "[11]"); + WrapInserter(fic, 1).word("a").add(11).word("b").add(11).flush(); + WrapInserter(fic, 2).word("a").add(11).flush(); + assertPostingLists("[11]", "[11]", "[11]"); + remove(13); + assertPostingLists("[11]", "[11]", "[11]"); } -} // namespace memoryindex -} // namespace search +} +} -TEST_MAIN() { TEST_RUN_ALL(); } +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchlib/diskindex/diskindex.cpp b/searchlib/src/vespa/searchlib/diskindex/diskindex.cpp index d71ddc2c2d6..64a54187254 100644 --- a/searchlib/src/vespa/searchlib/diskindex/diskindex.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/diskindex.cpp @@ -39,7 +39,9 @@ DiskIndex::Key::Key() = default; DiskIndex::Key::Key(const IndexList & indexes, vespalib::stringref word) : _word(word), _indexes(indexes) -{ } +{ +} + DiskIndex::Key::~Key() = default; DiskIndex::DiskIndex(const vespalib::string &indexDir, size_t cacheSize) @@ -73,7 +75,6 @@ DiskIndex::loadSchema() return true; } - bool DiskIndex::openDictionaries(const TuneFileSearch &tuneFileSearch) { @@ -91,7 +92,6 @@ DiskIndex::openDictionaries(const TuneFileSearch &tuneFileSearch) return true; } - bool DiskIndex::openField(const vespalib::string &fieldDir, const TuneFileSearch &tuneFileSearch) @@ -147,7 +147,6 @@ DiskIndex::openField(const vespalib::string &fieldDir, return true; } - bool DiskIndex::setup(const TuneFileSearch &tuneFileSearch) { @@ -165,7 +164,6 @@ DiskIndex::setup(const TuneFileSearch &tuneFileSearch) return true; } - bool DiskIndex::setup(const TuneFileSearch &tuneFileSearch, const DiskIndex &old) @@ -315,7 +313,6 @@ DiskIndex::readPostingList(const LookupResult &lookupRes) const return handle; } - BitVector::UP DiskIndex::readBitVector(const LookupResult &lookupRes) const { @@ -327,7 +324,6 @@ DiskIndex::readBitVector(const LookupResult &lookupRes) const return dict->lookup(lookupRes.wordNum); } - void DiskIndex::calculateSize() { @@ -335,19 +331,18 @@ DiskIndex::calculateSize() _size = dirt.GetTreeSize(); } - namespace { DiskIndex::LookupResult _G_nothing; -class LookupCache -{ +class LookupCache { public: LookupCache(DiskIndex & diskIndex, const std::vector<uint32_t> & fieldIds) : _diskIndex(diskIndex), _fieldIds(fieldIds), _cache() - { } + { + } const DiskIndex::LookupResult & lookup(const vespalib::string & word, uint32_t fieldId) { Cache::const_iterator it = _cache.find(word); @@ -363,14 +358,14 @@ public: return _G_nothing; } private: + typedef vespalib::hash_map<vespalib::string, DiskIndex::LookupResultVector> Cache; DiskIndex & _diskIndex; const std::vector<uint32_t> & _fieldIds; Cache _cache; }; -class CreateBlueprintVisitor : public CreateBlueprintVisitorHelper -{ +class CreateBlueprintVisitor : public CreateBlueprintVisitorHelper { private: LookupCache &_cache; DiskIndex &_diskIndex; @@ -391,8 +386,7 @@ public: } template <class TermNode> - void visitTerm(TermNode &n) - { + void visitTerm(TermNode &n) { const vespalib::string termStr = termAsString(n); const DiskIndex::LookupResult & lookupRes = _cache.lookup(termStr, _fieldId); if (lookupRes.valid()) { @@ -418,7 +412,6 @@ public: void visit(PredicateQuery &) override { } }; - Blueprint::UP createBlueprintHelper(LookupCache & cache, DiskIndex & diskIndex, const IRequestContext & requestContext, const FieldSpec &field, uint32_t fieldId, const Node &term) @@ -442,7 +435,6 @@ DiskIndex::createBlueprint(const IRequestContext & requestContext, const FieldSp return createBlueprintHelper(cache, *this, requestContext, field, fieldIds[0], term); } - Blueprint::UP DiskIndex::createBlueprint(const IRequestContext & requestContext, const FieldSpecList &fields, const Node &term) { diff --git a/searchlib/src/vespa/searchlib/diskindex/diskindex.h b/searchlib/src/vespa/searchlib/diskindex/diskindex.h index 4bef53a3030..d83b2f56d7c 100644 --- a/searchlib/src/vespa/searchlib/diskindex/diskindex.h +++ b/searchlib/src/vespa/searchlib/diskindex/diskindex.h @@ -12,14 +12,13 @@ namespace search::diskindex { /** - * This class represents a disk index with a common dictionary, and - * posting list files and bit vector files for each field. - * Parts of the disk dictionary and all bit vector - * dictionaries are loaded into memory during setup. All other files - * are just opened, ready for later access. - **/ -class DiskIndex : public queryeval::Searchable -{ + * This class represents a disk index that contains a set of field indexes that are independent of each other. + * + * Each field index has a dictionary, posting list files and bit vector files. + * Parts of the disk dictionary and all bit vector dictionaries are loaded into memory during setup. + * All other files are just opened, ready for later access. + */ +class DiskIndex : public queryeval::Searchable { public: /** * The result after performing a disk dictionary lookup. @@ -60,11 +59,12 @@ public: vespalib::string _word; IndexList _indexes; }; + private: - typedef index::PostingListFileRandRead DiskPostingFile; - typedef Zc4PosOccRandRead DiskPostingFileReal; - typedef ZcPosOccRandRead DiskPostingFileDynamicKReal; - typedef vespalib::cache<vespalib::CacheParam<vespalib::LruParam<Key, LookupResultVector>, DiskIndex>> Cache; + using DiskPostingFile = index::PostingListFileRandRead; + using DiskPostingFileReal = Zc4PosOccRandRead; + using DiskPostingFileDynamicKReal = ZcPosOccRandRead; + using Cache = vespalib::cache<vespalib::CacheParam<vespalib::LruParam<Key, LookupResultVector>, DiskIndex>>; vespalib::string _indexDir; size_t _cacheSize; @@ -83,11 +83,11 @@ private: public: /** - * Create a view of the disk index located in the given directory - * described by the given schema. + * Create a view of the disk index located in the given directory. * * @param indexDir the directory where the disk index is located. - **/ + * @param cacheSize optional size (in bytes) of the disk dictionary lookup cache. + */ DiskIndex(const vespalib::string &indexDir, size_t cacheSize=0); ~DiskIndex(); @@ -95,29 +95,27 @@ public: * Setup this instance by opening and loading relevant index files. * * @return true if this instance was successfully setup. - **/ + */ bool setup(const TuneFileSearch &tuneFileSearch); bool setup(const TuneFileSearch &tuneFileSearch, const DiskIndex &old); /** - * Perform a dictionary lookup for the given word in the given - * field. + * Perform a dictionary lookup for the given word in the given field. * - * @param indexId the id of the field to - * perform lookup for. + * @param indexId the id of the field to perform lookup for. * @param word the word to lookup. * @return the lookup result or nullptr if the word is not found. - **/ + */ LookupResult::UP lookup(uint32_t indexId, vespalib::stringref word); - LookupResultVector lookup(const std::vector<uint32_t> & indexes, vespalib::stringref word); + LookupResultVector lookup(const std::vector<uint32_t> & indexes, vespalib::stringref word); /** * Read the posting list corresponding to the given lookup result. * * @param lookupRes the result of the previous dictionary lookup. * @return a handle for the posting list in memory. - **/ + */ index::PostingListHandle::UP readPostingList(const LookupResult &lookupRes) const; /** @@ -126,22 +124,19 @@ public: * @param lookupRes the result of the previous dictionary lookup. * @return the bit vector or nullptr if no bit vector exists for the * word in the lookup result. - **/ + */ BitVector::UP readBitVector(const LookupResult &lookupRes) const; - queryeval::Blueprint::UP - createBlueprint(const queryeval::IRequestContext & requestContext, - const queryeval::FieldSpec &field, - const query::Node &term) override; + queryeval::Blueprint::UP createBlueprint(const queryeval::IRequestContext & requestContext, + const queryeval::FieldSpec &field, + const query::Node &term) override; - queryeval::Blueprint::UP - createBlueprint(const queryeval::IRequestContext & requestContext, - const queryeval::FieldSpecList &fields, - const query::Node &term) override; + queryeval::Blueprint::UP createBlueprint(const queryeval::IRequestContext & requestContext, + const queryeval::FieldSpecList &fields, + const query::Node &term) override; /** * Get the size on disk of this index. - * @return the size of the index. */ uint64_t getSize() const { return _size; } diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp index 6454c0851a7..8c2b33a933e 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp @@ -98,7 +98,6 @@ FieldWriter::open(const vespalib::string &prefix, return true; } - void FieldWriter::flush() { @@ -120,7 +119,6 @@ FieldWriter::flush() } } - void FieldWriter::newWord(uint64_t wordNum, vespalib::stringref word) { @@ -134,14 +132,12 @@ FieldWriter::newWord(uint64_t wordNum, vespalib::stringref word) _prevDocId = 0; } - void FieldWriter::newWord(vespalib::stringref word) { newWord(_wordNum + 1, word); } - bool FieldWriter::close() { @@ -183,7 +179,6 @@ FieldWriter::getFeatureParams(PostingListParams ¶ms) _posoccfile->getFeatureParams(params); } - static const char *termOccNames[] = { "boolocc.bdat", @@ -199,7 +194,6 @@ static const char *termOccNames[] = nullptr, }; - void FieldWriter::remove(const vespalib::string &prefix) { diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h index 9a6edf90243..1e9afb717e8 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h +++ b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h @@ -10,15 +10,13 @@ namespace search::diskindex { -/* - * FieldWriter is used to write a dictionary and posting list file - * together. +/** + * FieldWriter is used to write a dictionary and posting list file together. * * It is used by the fusion code to write the merged output for a field, * and by the memory index dump code to write a field to disk. */ -class FieldWriter -{ +class FieldWriter { private: uint64_t _wordNum; uint32_t _prevDocId; @@ -28,14 +26,15 @@ public: using DictionaryFileSeqWrite = index::DictionaryFileSeqWrite; - typedef index::PostingListFileSeqWrite PostingListFileSeqWrite; - typedef index::DocIdAndFeatures DocIdAndFeatures; - typedef index::Schema Schema; - typedef index::PostingListCounts PostingListCounts; - typedef index::PostingListParams PostingListParams; + using PostingListFileSeqWrite = index::PostingListFileSeqWrite; + using DocIdAndFeatures = index::DocIdAndFeatures; + using Schema = index::Schema; + using PostingListCounts = index::PostingListCounts; + using PostingListParams = index::PostingListParams; std::unique_ptr<DictionaryFileSeqWrite> _dictFile; std::unique_ptr<PostingListFileSeqWrite> _posoccfile; + private: BitVectorCandidate _bvc; BitVectorFileWrite _bmapfile; diff --git a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp index a3c37cb91f6..964f37eb5cf 100644 --- a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp @@ -33,8 +33,7 @@ noWordPos() } -class FileHandle -{ +class FileHandle { public: FieldWriter *_fieldWriter; DocIdAndFeatures _docIdAndFeatures; @@ -43,22 +42,18 @@ public: ~FileHandle(); - void - open(vespalib::stringref dir, - const SchemaUtil::IndexIterator &index, - uint32_t docIdLimit, uint64_t numWordIds, - const TuneFileSeqWrite &tuneFileWrite, - const FileHeaderContext &fileHeaderContext); + void open(vespalib::stringref dir, + const SchemaUtil::IndexIterator &index, + uint32_t docIdLimit, uint64_t numWordIds, + const TuneFileSeqWrite &tuneFileWrite, + const FileHeaderContext &fileHeaderContext); - void - close(); + void close(); }; - } -class IndexBuilder::FieldHandle -{ +class IndexBuilder::FieldHandle { public: FieldHandle(const Schema &schema, uint32_t fieldId, @@ -66,20 +61,15 @@ public: ~FieldHandle(); - static uint32_t - noDocRef() - { + static uint32_t noDocRef() { return std::numeric_limits<uint32_t>::max(); } - static uint32_t - noElRef() - { + static uint32_t noElRef() { return std::numeric_limits<uint32_t>::max(); } - class FHWordDocFieldFeatures - { + class FHWordDocFieldFeatures { public: uint32_t _docId; uint32_t _numElements; @@ -90,28 +80,12 @@ public: { } - uint32_t - getDocId() const - { - return _docId; - } - - uint32_t - getNumElements() const - { - return _numElements; - } - - void - incNumElements() - { - ++_numElements; - } + uint32_t getDocId() const { return _docId; } + uint32_t getNumElements() const { return _numElements; } + void incNumElements() { ++_numElements; } }; - class FHWordDocElementFeatures - : public WordDocElementFeatures - { + class FHWordDocElementFeatures : public WordDocElementFeatures { public: uint32_t _docRef; @@ -127,24 +101,21 @@ public: } }; - class FHWordDocElementWordPosFeatures - : public WordDocElementWordPosFeatures - { + class FHWordDocElementWordPosFeatures : public WordDocElementWordPosFeatures { public: uint32_t _elementRef; - FHWordDocElementWordPosFeatures( - const WordDocElementWordPosFeatures &features, - uint32_t elementRef) + FHWordDocElementWordPosFeatures(const WordDocElementWordPosFeatures &features, + uint32_t elementRef) : WordDocElementWordPosFeatures(features), _elementRef(elementRef) { } }; - typedef vespalib::Array<FHWordDocFieldFeatures> FHWordDocFieldFeaturesVector; - typedef vespalib::Array<FHWordDocElementFeatures> FHWordDocElementFeaturesVector; - typedef vespalib::Array<FHWordDocElementWordPosFeatures> FHWordDocElementWordPosFeaturesVector; + using FHWordDocFieldFeaturesVector = vespalib::Array<FHWordDocFieldFeatures>; + using FHWordDocElementFeaturesVector = vespalib::Array<FHWordDocElementFeatures>; + using FHWordDocElementWordPosFeaturesVector = vespalib::Array<FHWordDocElementWordPosFeatures>; FHWordDocFieldFeaturesVector _wdff; FHWordDocElementFeaturesVector _wdfef; @@ -162,72 +133,35 @@ public: FileHandle _files; - void - startWord(vespalib::stringref word); - - void - endWord(); - - void - startDocument(uint32_t docId); - - void - endDocument(); - - void - startElement(uint32_t elementId, - int32_t weight, - uint32_t elementLen); - - void - endElement(); - - void - addOcc(const WordDocElementWordPosFeatures &features); - - void - setValid() - { - _valid = true; - } - - bool - getValid() const - { - return _valid; - } - - const Schema::IndexField & - getSchemaField(); - - const vespalib::string & - getName(); - - vespalib::string - getDir(); - - void - open(uint32_t docIdLimit, uint64_t numWordIds, - const TuneFileSeqWrite &tuneFileWrite, - const FileHeaderContext &fileHeaderContext); - - void - close(); - - uint32_t - getIndexId() const - { - return _fieldId; - } + void startWord(vespalib::stringref word); + void endWord(); + void startDocument(uint32_t docId); + void endDocument(); + void startElement(uint32_t elementId, + int32_t weight, + uint32_t elementLen); + void endElement(); + void addOcc(const WordDocElementWordPosFeatures &features); + + const Schema::IndexField &getSchemaField(); + const vespalib::string &getName(); + vespalib::string getDir(); + void open(uint32_t docIdLimit, uint64_t numWordIds, + const TuneFileSeqWrite &tuneFileWrite, + const FileHeaderContext &fileHeaderContext); + void close(); + + void setValid() { _valid = true; } + bool getValid() const { return _valid; } + uint32_t getIndexId() const { return _fieldId; } }; namespace { -class SingleIterator -{ +class SingleIterator { public: - typedef IndexBuilder::FieldHandle FH; + using FH = IndexBuilder::FieldHandle; FH::FHWordDocFieldFeaturesVector::const_iterator _dFeatures; FH::FHWordDocFieldFeaturesVector::const_iterator _dFeaturesE; FH::FHWordDocElementFeaturesVector::const_iterator _elFeatures; @@ -237,18 +171,13 @@ public: SingleIterator(FH &fieldHandle, uint32_t localFieldId); - void - appendFeatures(DocIdAndFeatures &features); + void appendFeatures(DocIdAndFeatures &features); - bool - isValid() const - { + bool isValid() const { return _dFeatures != _dFeaturesE; } - bool - operator<(const SingleIterator &rhs) const - { + bool operator<(const SingleIterator &rhs) const { if (_docId != rhs._docId) { return _docId < rhs._docId; } @@ -256,23 +185,19 @@ public: } }; - } - FileHandle::FileHandle() : _fieldWriter(nullptr), _docIdAndFeatures() { } - FileHandle::~FileHandle() { delete _fieldWriter; } - void FileHandle::open(vespalib::stringref dir, const SchemaUtil::IndexIterator &index, @@ -293,7 +218,6 @@ FileHandle::open(vespalib::stringref dir, } } - void FileHandle::close() { @@ -312,7 +236,6 @@ FileHandle::close() (void) ret; } - IndexBuilder::FieldHandle::FieldHandle(const Schema &schema, uint32_t fieldId, IndexBuilder *ib) @@ -331,10 +254,8 @@ IndexBuilder::FieldHandle::FieldHandle(const Schema &schema, { } - IndexBuilder::FieldHandle::~FieldHandle() = default; - void IndexBuilder::FieldHandle::startWord(vespalib::stringref word) { @@ -342,7 +263,6 @@ IndexBuilder::FieldHandle::startWord(vespalib::stringref word) _files._fieldWriter->newWord(word); } - void IndexBuilder::FieldHandle::endWord() { @@ -362,7 +282,6 @@ IndexBuilder::FieldHandle::endWord() _elRef = noElRef(); } - void IndexBuilder::FieldHandle::startDocument(uint32_t docId) { @@ -373,7 +292,6 @@ IndexBuilder::FieldHandle::startDocument(uint32_t docId) _lowestOKElementId = 0u; } - void IndexBuilder::FieldHandle::endDocument() { @@ -385,12 +303,10 @@ IndexBuilder::FieldHandle::endDocument() _docRef = noDocRef(); } - void -IndexBuilder::FieldHandle:: -startElement(uint32_t elementId, - int32_t weight, - uint32_t elementLen) +IndexBuilder::FieldHandle::startElement(uint32_t elementId, + int32_t weight, + uint32_t elementLen) { assert(_docRef != noDocRef()); assert(_elRef == noElRef()); @@ -407,7 +323,6 @@ startElement(uint32_t elementId, _lowestOKWordPos = 0u; } - void IndexBuilder::FieldHandle::endElement() { @@ -418,10 +333,8 @@ IndexBuilder::FieldHandle::endElement() _lowestOKElementId = ef.getElementId() + 1; } - void -IndexBuilder::FieldHandle:: -addOcc(const WordDocElementWordPosFeatures &features) +IndexBuilder::FieldHandle::addOcc(const WordDocElementWordPosFeatures &features) { assert(_elRef != noElRef()); FHWordDocElementFeatures &ef = _wdfef[_elRef]; @@ -435,29 +348,24 @@ addOcc(const WordDocElementWordPosFeatures &features) ef.incNumOccs(); } - const Schema::IndexField & IndexBuilder::FieldHandle::getSchemaField() { return _schema->getIndexField(_fieldId); } - const vespalib::string & IndexBuilder::FieldHandle::getName() { return getSchemaField().getName(); - } - vespalib::string IndexBuilder::FieldHandle::getDir() { return _ib->appendToPrefix(getName()); } - void IndexBuilder::FieldHandle::open(uint32_t docIdLimit, uint64_t numWordIds, const TuneFileSeqWrite &tuneFileWrite, @@ -468,14 +376,12 @@ IndexBuilder::FieldHandle::open(uint32_t docIdLimit, uint64_t numWordIds, docIdLimit, numWordIds, tuneFileWrite, fileHeaderContext); } - void IndexBuilder::FieldHandle::close() { _files.close(); } - SingleIterator::SingleIterator(FH &fieldHandle, uint32_t localFieldId) : _dFeatures(fieldHandle._wdff.begin()), _dFeaturesE(fieldHandle._wdff.end()), @@ -486,7 +392,6 @@ SingleIterator::SingleIterator(FH &fieldHandle, uint32_t localFieldId) { } - void SingleIterator::appendFeatures(DocIdAndFeatures &features) { @@ -511,7 +416,6 @@ SingleIterator::appendFeatures(DocIdAndFeatures &features) } } - IndexBuilder::IndexBuilder(const Schema &schema) : index::IndexBuilder(schema), _currentField(nullptr), @@ -541,6 +445,27 @@ IndexBuilder::IndexBuilder(const Schema &schema) IndexBuilder::~IndexBuilder() = default; void +IndexBuilder::startField(uint32_t fieldId) +{ + assert(_curDocId == noDocId()); + assert(_currentField == nullptr); + assert(fieldId < _fields.size()); + assert(fieldId >= _lowestOKFieldId); + _currentField = &_fields[fieldId]; + assert(_currentField != nullptr); +} + +void +IndexBuilder::endField() +{ + assert(_curDocId == noDocId()); + assert(!_inWord); + assert(_currentField != nullptr); + _lowestOKFieldId = _currentField->_fieldId + 1; + _currentField = nullptr; +} + +void IndexBuilder::startWord(vespalib::stringref word) { assert(_currentField != nullptr); @@ -551,7 +476,6 @@ IndexBuilder::startWord(vespalib::stringref word) _currentField->startWord(word); } - void IndexBuilder::endWord() { @@ -562,7 +486,6 @@ IndexBuilder::endWord() _lowestOKDocId = 1u; } - void IndexBuilder::startDocument(uint32_t docId) { @@ -575,7 +498,6 @@ IndexBuilder::startDocument(uint32_t docId) _currentField->startDocument(docId); } - void IndexBuilder::endDocument() { @@ -586,30 +508,6 @@ IndexBuilder::endDocument() _curDocId = noDocId(); } - -void -IndexBuilder::startField(uint32_t fieldId) -{ - assert(_curDocId == noDocId()); - assert(_currentField == nullptr); - assert(fieldId < _fields.size()); - assert(fieldId >= _lowestOKFieldId); - _currentField = &_fields[fieldId]; - assert(_currentField != nullptr); -} - - -void -IndexBuilder::endField() -{ - assert(_curDocId == noDocId()); - assert(!_inWord); - assert(_currentField != nullptr); - _lowestOKFieldId = _currentField->_fieldId + 1; - _currentField = nullptr; -} - - void IndexBuilder::startElement(uint32_t elementId, int32_t weight, @@ -619,7 +517,6 @@ IndexBuilder::startElement(uint32_t elementId, _currentField->startElement(elementId, weight, elementLen); } - void IndexBuilder::endElement() { @@ -627,7 +524,6 @@ IndexBuilder::endElement() _currentField->endElement(); } - void IndexBuilder::addOcc(const WordDocElementWordPosFeatures &features) { @@ -635,14 +531,12 @@ IndexBuilder::addOcc(const WordDocElementWordPosFeatures &features) _currentField->addOcc(features); } - void IndexBuilder::setPrefix(vespalib::stringref prefix) { _prefix = prefix; } - vespalib::string IndexBuilder::appendToPrefix(vespalib::stringref name) { @@ -652,7 +546,6 @@ IndexBuilder::appendToPrefix(vespalib::stringref name) return _prefix + "/" + name; } - void IndexBuilder::open(uint32_t docIdLimit, uint64_t numWordIds, const TuneFileIndexing &tuneFileIndexing, @@ -682,7 +575,6 @@ IndexBuilder::open(uint32_t docIdLimit, uint64_t numWordIds, } } - void IndexBuilder::close() { diff --git a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.h b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.h index fa818bf08e6..43ac49a0a72 100644 --- a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.h +++ b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.h @@ -13,12 +13,16 @@ namespace search::diskindex { class BitVectorCandidate; -class IndexBuilder : public index::IndexBuilder -{ +/** + * Class used to build a disk index for the set of index fields specified in a schema. + * + * The resulting disk index consists of field indexes that are independent of each other. + */ +class IndexBuilder : public index::IndexBuilder { public: class FieldHandle; - typedef index::Schema Schema; + using Schema = index::Schema; private: // Text fields FieldHandle *_currentField; @@ -49,19 +53,16 @@ public: IndexBuilder(const Schema &schema); ~IndexBuilder() override; + void startField(uint32_t fieldId) override; + void endField() override; void startWord(vespalib::stringref word) override; void endWord() override; void startDocument(uint32_t docId) override; void endDocument() override; - void startField(uint32_t fieldId) override; - void endField() override; void startElement(uint32_t elementId, int32_t weight, uint32_t elementLen) override; void endElement() override; void addOcc(const WordDocElementWordPosFeatures &features) override; - // TODO: methods for attribute vectors. - - // TODO: methods for document summary. void setPrefix(vespalib::stringref prefix); vespalib::string appendToPrefix(vespalib::stringref name); diff --git a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.cpp b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.cpp index 51f7a2ea151..5ab37cecc3d 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.cpp @@ -8,6 +8,189 @@ using search::index::PostingListParams; namespace search::diskindex { +namespace { + +class DocIdEncoder { +protected: + uint32_t _doc_id; + uint32_t _doc_id_pos; + uint32_t _feature_pos; + using DocIdAndFeatureSize = std::pair<uint32_t, uint32_t>; + +public: + DocIdEncoder() + : _doc_id(0u), + _doc_id_pos(0u), + _feature_pos(0u) + { + } + + void write(ZcBuf &zc_buf, const DocIdAndFeatureSize &doc_id_and_feature_size); + void set_doc_id(uint32_t doc_id) { _doc_id = doc_id; } + uint32_t get_doc_id() const { return _doc_id; } + uint32_t get_doc_id_pos() const { return _doc_id_pos; } + uint32_t get_feature_pos() const { return _feature_pos; } +}; + +class L1SkipEncoder : public DocIdEncoder { +protected: + uint32_t _stride_check; + uint32_t _l1_skip_pos; + const bool _encode_features; + +public: + L1SkipEncoder(bool encode_features) + : DocIdEncoder(), + _stride_check(0u), + _l1_skip_pos(0u), + _encode_features(encode_features) + { + } + + void encode_skip(ZcBuf &zc_buf, const DocIdEncoder &doc_id_encoder); + void write_skip(ZcBuf &zc_buf, const DocIdEncoder &doc_id_encoder); + bool should_write_skip(uint32_t stride) { return ++_stride_check >= stride; } + void dec_stride_check() { --_stride_check; } + void write_partial_skip(ZcBuf &zc_buf, uint32_t doc_id); + uint32_t get_l1_skip_pos() const { return _l1_skip_pos; } +}; + +struct L2SkipEncoder : public L1SkipEncoder { +protected: + uint32_t _l2_skip_pos; + +public: + L2SkipEncoder(bool encode_features) + : L1SkipEncoder(encode_features), + _l2_skip_pos(0u) + { + } + + void encode_skip(ZcBuf &zc_buf, const L1SkipEncoder &l1_skip); + void write_skip(ZcBuf &zc_buf, const L1SkipEncoder &l1_skip); + uint32_t get_l2_skip_pos() const { return _l2_skip_pos; } +}; + +class L3SkipEncoder : public L2SkipEncoder { +protected: + uint32_t _l3_skip_pos; + +public: + L3SkipEncoder(bool encode_features) + : L2SkipEncoder(encode_features), + _l3_skip_pos(0u) + { + } + + void encode_skip(ZcBuf &zc_buf, const L2SkipEncoder &l2_skip); + void write_skip(ZcBuf &zc_buf, const L2SkipEncoder &l2_skip); + uint32_t get_l3_skip_pos() const { return _l3_skip_pos; } +}; + +class L4SkipEncoder : public L3SkipEncoder { + +public: + L4SkipEncoder(bool encode_features) + : L3SkipEncoder(encode_features) + { + } + + void encode_skip(ZcBuf &zc_buf, const L3SkipEncoder &l3_skip); + void write_skip(ZcBuf &zc_buf, const L3SkipEncoder &l3_skip); +}; + +void +DocIdEncoder::write(ZcBuf &zc_buf, const DocIdAndFeatureSize &doc_id_and_feature_size) +{ + _feature_pos += doc_id_and_feature_size.second; + zc_buf.encode(doc_id_and_feature_size.first - _doc_id - 1); + _doc_id = doc_id_and_feature_size.first; + _doc_id_pos = zc_buf.size(); +} + +void +L1SkipEncoder::encode_skip(ZcBuf &zc_buf, const DocIdEncoder &doc_id_encoder) +{ + _stride_check = 0; + // doc id + uint32_t doc_id_delta = doc_id_encoder.get_doc_id() - _doc_id; + assert(static_cast<int32_t>(doc_id_delta) > 0); + zc_buf.encode(doc_id_delta - 1); + _doc_id = doc_id_encoder.get_doc_id(); + // doc id pos + zc_buf.encode(doc_id_encoder.get_doc_id_pos() - _doc_id_pos - 1); + _doc_id_pos = doc_id_encoder.get_doc_id_pos(); + if (_encode_features) { + // features pos + zc_buf.encode(doc_id_encoder.get_feature_pos() - _feature_pos - 1); + _feature_pos = doc_id_encoder.get_feature_pos(); + } +} + +void +L1SkipEncoder::write_skip(ZcBuf &zc_buf, const DocIdEncoder &doc_id_encoder) +{ + encode_skip(zc_buf, doc_id_encoder); + _l1_skip_pos = zc_buf.size(); +} + +void +L1SkipEncoder::write_partial_skip(ZcBuf &zc_buf, uint32_t doc_id) +{ + if (zc_buf.size() > 0) { + zc_buf.encode(doc_id - _doc_id - 1); + } +} + +void +L2SkipEncoder::encode_skip(ZcBuf &zc_buf, const L1SkipEncoder &l1_skip) +{ + L1SkipEncoder::encode_skip(zc_buf, l1_skip); + // L1 skip pos + zc_buf.encode(l1_skip.get_l1_skip_pos() - _l1_skip_pos - 1); + _l1_skip_pos = l1_skip.get_l1_skip_pos(); +} + +void +L2SkipEncoder::write_skip(ZcBuf &zc_buf, const L1SkipEncoder &l1_skip) +{ + encode_skip(zc_buf, l1_skip); + _l2_skip_pos = zc_buf.size(); +} + +void +L3SkipEncoder::encode_skip(ZcBuf &zc_buf, const L2SkipEncoder &l2_skip) +{ + L2SkipEncoder::encode_skip(zc_buf, l2_skip); + // L2 skip pos + zc_buf.encode(l2_skip.get_l2_skip_pos() - _l2_skip_pos - 1); + _l2_skip_pos = l2_skip.get_l2_skip_pos(); +} + +void +L3SkipEncoder::write_skip(ZcBuf &zc_buf, const L2SkipEncoder &l2_skip) +{ + encode_skip(zc_buf, l2_skip); + _l3_skip_pos = zc_buf.size(); +} + +void +L4SkipEncoder::encode_skip(ZcBuf &zc_buf, const L3SkipEncoder &l3_skip) +{ + L3SkipEncoder::encode_skip(zc_buf, l3_skip); + // L3 skip pos + zc_buf.encode(l3_skip.get_l3_skip_pos() - _l3_skip_pos - 1); + _l3_skip_pos = l3_skip.get_l3_skip_pos(); +} + +void +L4SkipEncoder::write_skip(ZcBuf &zc_buf, const L3SkipEncoder &l3_skip) +{ + encode_skip(zc_buf, l3_skip); +} + +} + Zc4PostingWriterBase::Zc4PostingWriterBase(PostingListCounts &counts) : _minChunkDocs(1 << 30), _minSkipDocs(64), @@ -45,159 +228,42 @@ Zc4PostingWriterBase::~Zc4PostingWriterBase() #define L4SKIPSTRIDE 8 void -Zc4PostingWriterBase::calc_skip_info(bool encodeFeatures) +Zc4PostingWriterBase::calc_skip_info(bool encode_features) { - uint32_t lastDocId = 0u; - uint32_t lastL1SkipDocId = 0u; - uint32_t lastL1SkipDocIdPos = 0; - uint32_t lastL1SkipFeaturePos = 0; - uint32_t lastL2SkipDocId = 0u; - uint32_t lastL2SkipDocIdPos = 0; - uint32_t lastL2SkipFeaturePos = 0; - uint32_t lastL2SkipL1SkipPos = 0; - uint32_t lastL3SkipDocId = 0u; - uint32_t lastL3SkipDocIdPos = 0; - uint32_t lastL3SkipFeaturePos = 0; - uint32_t lastL3SkipL1SkipPos = 0; - uint32_t lastL3SkipL2SkipPos = 0; - uint32_t lastL4SkipDocId = 0u; - uint32_t lastL4SkipDocIdPos = 0; - uint32_t lastL4SkipFeaturePos = 0; - uint32_t lastL4SkipL1SkipPos = 0; - uint32_t lastL4SkipL2SkipPos = 0; - uint32_t lastL4SkipL3SkipPos = 0; - unsigned int l1SkipCnt = 0; - unsigned int l2SkipCnt = 0; - unsigned int l3SkipCnt = 0; - unsigned int l4SkipCnt = 0; - uint64_t featurePos = 0; - - std::vector<DocIdAndFeatureSize>::const_iterator dit = _docIds.begin(); - std::vector<DocIdAndFeatureSize>::const_iterator dite = _docIds.end(); - + DocIdEncoder doc_id_encoder; + L1SkipEncoder l1_skip_encoder(encode_features); + L2SkipEncoder l2_skip_encoder(encode_features); + L3SkipEncoder l3_skip_encoder(encode_features); + L4SkipEncoder l4_skip_encoder(encode_features); + l1_skip_encoder.dec_stride_check(); if (!_counts._segments.empty()) { - lastDocId = _counts._segments.back()._lastDoc; - lastL1SkipDocId = lastDocId; - lastL2SkipDocId = lastDocId; - lastL3SkipDocId = lastDocId; - lastL4SkipDocId = lastDocId; + uint32_t doc_id = _counts._segments.back()._lastDoc; + doc_id_encoder.set_doc_id(doc_id); + l1_skip_encoder.set_doc_id(doc_id); + l2_skip_encoder.set_doc_id(doc_id); + l3_skip_encoder.set_doc_id(doc_id); + l4_skip_encoder.set_doc_id(doc_id); } - - for (; dit != dite; ++dit) { - if (l1SkipCnt >= L1SKIPSTRIDE) { - // L1 docid delta - uint32_t docIdDelta = lastDocId - lastL1SkipDocId; - assert(static_cast<int32_t>(docIdDelta) > 0); - _l1Skip.encode(docIdDelta - 1); - lastL1SkipDocId = lastDocId; - // L1 docid pos - uint64_t docIdPos = _zcDocIds.size(); - _l1Skip.encode(docIdPos - lastL1SkipDocIdPos - 1); - lastL1SkipDocIdPos = docIdPos; - if (encodeFeatures) { - // L1 features pos - _l1Skip.encode(featurePos - lastL1SkipFeaturePos - 1); - lastL1SkipFeaturePos = featurePos; - } - l1SkipCnt = 0; - ++l2SkipCnt; - if (l2SkipCnt >= L2SKIPSTRIDE) { - // L2 docid delta - docIdDelta = lastDocId - lastL2SkipDocId; - assert(static_cast<int32_t>(docIdDelta) > 0); - _l2Skip.encode(docIdDelta - 1); - lastL2SkipDocId = lastDocId; - // L2 docid pos - docIdPos = _zcDocIds.size(); - _l2Skip.encode(docIdPos - lastL2SkipDocIdPos - 1); - lastL2SkipDocIdPos = docIdPos; - if (encodeFeatures) { - // L2 features pos - _l2Skip.encode(featurePos - lastL2SkipFeaturePos - 1); - lastL2SkipFeaturePos = featurePos; - } - // L2 L1Skip pos - uint64_t l1SkipPos = _l1Skip.size(); - _l2Skip.encode(l1SkipPos - lastL2SkipL1SkipPos - 1); - lastL2SkipL1SkipPos = l1SkipPos; - l2SkipCnt = 0; - ++l3SkipCnt; - if (l3SkipCnt >= L3SKIPSTRIDE) { - // L3 docid delta - docIdDelta = lastDocId - lastL3SkipDocId; - assert(static_cast<int32_t>(docIdDelta) > 0); - _l3Skip.encode(docIdDelta - 1); - lastL3SkipDocId = lastDocId; - // L3 docid pos - docIdPos = _zcDocIds.size(); - _l3Skip.encode(docIdPos - lastL3SkipDocIdPos - 1); - lastL3SkipDocIdPos = docIdPos; - if (encodeFeatures) { - // L3 features pos - _l3Skip.encode(featurePos - lastL3SkipFeaturePos - 1); - lastL3SkipFeaturePos = featurePos; - } - // L3 L1Skip pos - l1SkipPos = _l1Skip.size(); - _l3Skip.encode(l1SkipPos - lastL3SkipL1SkipPos - 1); - lastL3SkipL1SkipPos = l1SkipPos; - // L3 L2Skip pos - uint64_t l2SkipPos = _l2Skip.size(); - _l3Skip.encode(l2SkipPos - lastL3SkipL2SkipPos - 1); - lastL3SkipL2SkipPos = l2SkipPos; - l3SkipCnt = 0; - ++l4SkipCnt; - if (l4SkipCnt >= L4SKIPSTRIDE) { - // L4 docid delta - docIdDelta = lastDocId - lastL4SkipDocId; - assert(static_cast<int32_t>(docIdDelta) > 0); - _l4Skip.encode(docIdDelta - 1); - lastL4SkipDocId = lastDocId; - // L4 docid pos - docIdPos = _zcDocIds.size(); - _l4Skip.encode(docIdPos - lastL4SkipDocIdPos - 1); - lastL4SkipDocIdPos = docIdPos; - if (encodeFeatures) { - // L4 features pos - _l4Skip.encode(featurePos - lastL4SkipFeaturePos - 1); - lastL4SkipFeaturePos = featurePos; - } - // L4 L1Skip pos - l1SkipPos = _l1Skip.size(); - _l4Skip.encode(l1SkipPos - lastL4SkipL1SkipPos - 1); - lastL4SkipL1SkipPos = l1SkipPos; - // L4 L2Skip pos - l2SkipPos = _l2Skip.size(); - _l4Skip.encode(l2SkipPos - lastL4SkipL2SkipPos - 1); - lastL4SkipL2SkipPos = l2SkipPos; - // L4 L3Skip pos - uint64_t l3SkipPos = _l3Skip.size(); - _l4Skip.encode(l3SkipPos - lastL4SkipL3SkipPos - 1); - lastL4SkipL3SkipPos = l3SkipPos; - l4SkipCnt = 0; + for (const auto &doc_id_and_feature_size : _docIds) { + if (l1_skip_encoder.should_write_skip(L1SKIPSTRIDE)) { + l1_skip_encoder.write_skip(_l1Skip, doc_id_encoder); + if (l2_skip_encoder.should_write_skip(L2SKIPSTRIDE)) { + l2_skip_encoder.write_skip(_l2Skip, l1_skip_encoder); + if (l3_skip_encoder.should_write_skip(L3SKIPSTRIDE)) { + l3_skip_encoder.write_skip(_l3Skip, l2_skip_encoder); + if (l4_skip_encoder.should_write_skip(L4SKIPSTRIDE)) { + l4_skip_encoder.write_skip(_l4Skip, l3_skip_encoder); } } } } - uint32_t docId = dit->first; - featurePos += dit->second; - _zcDocIds.encode(docId - lastDocId - 1); - lastDocId = docId; - ++l1SkipCnt; + doc_id_encoder.write(_zcDocIds, doc_id_and_feature_size); } // Extra partial entries for skip tables to simplify iterator during search - if (_l1Skip.size() > 0) { - _l1Skip.encode(lastDocId - lastL1SkipDocId - 1); - } - if (_l2Skip.size() > 0) { - _l2Skip.encode(lastDocId - lastL2SkipDocId - 1); - } - if (_l3Skip.size() > 0) { - _l3Skip.encode(lastDocId - lastL3SkipDocId - 1); - } - if (_l4Skip.size() > 0) { - _l4Skip.encode(lastDocId - lastL4SkipDocId - 1); - } + l1_skip_encoder.write_partial_skip(_l1Skip, doc_id_encoder.get_doc_id()); + l2_skip_encoder.write_partial_skip(_l2Skip, doc_id_encoder.get_doc_id()); + l3_skip_encoder.write_partial_skip(_l3Skip, doc_id_encoder.get_doc_id()); + l4_skip_encoder.write_partial_skip(_l4Skip, doc_id_encoder.get_doc_id()); } void diff --git a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.h b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.h index e803fc692c3..6da59028803 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.h +++ b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.h @@ -47,7 +47,7 @@ protected: Zc4PostingWriterBase &operator=(Zc4PostingWriterBase &&) = delete; Zc4PostingWriterBase(index::PostingListCounts &counts); ~Zc4PostingWriterBase(); - void calc_skip_info(bool encodeFeatures); + void calc_skip_info(bool encode_features); void clear_skip_info(); public: diff --git a/searchlib/src/vespa/searchlib/index/indexbuilder.cpp b/searchlib/src/vespa/searchlib/index/indexbuilder.cpp index 6b88c51e6cc..d585238107a 100644 --- a/searchlib/src/vespa/searchlib/index/indexbuilder.cpp +++ b/searchlib/src/vespa/searchlib/index/indexbuilder.cpp @@ -6,7 +6,8 @@ namespace search::index { IndexBuilder::IndexBuilder(const Schema &schema) : _schema(schema) -{ } +{ +} IndexBuilder::~IndexBuilder() = default; diff --git a/searchlib/src/vespa/searchlib/index/indexbuilder.h b/searchlib/src/vespa/searchlib/index/indexbuilder.h index 66ca740a20c..0496809336b 100644 --- a/searchlib/src/vespa/searchlib/index/indexbuilder.h +++ b/searchlib/src/vespa/searchlib/index/indexbuilder.h @@ -8,6 +8,15 @@ namespace search::index { class Schema; class WordDocElementWordPosFeatures; +/** + * Interface used to build an index for the set of index fields specified in a schema. + * + * + * The index should be built as follows: + * For each field add the set of unique words in sorted order. + * For each word add the set of document ids in sorted order. + * For each document id add the position information for that document. + */ class IndexBuilder { protected: const Schema &_schema; @@ -15,39 +24,16 @@ protected: public: IndexBuilder(const Schema &schema); - virtual - ~IndexBuilder(); - - virtual void - startWord(vespalib::stringref word) = 0; - - virtual void - endWord() = 0; - - virtual void - startDocument(uint32_t docId) = 0; - - virtual void - endDocument() = 0; - - virtual void - startField(uint32_t fieldId) = 0; - - virtual void - endField() = 0; - - virtual void - startElement(uint32_t elementId, int32_t weight, uint32_t elementLen) = 0; - - virtual void - endElement() = 0; - - virtual void - addOcc(const WordDocElementWordPosFeatures &features) = 0; - - // TODO: methods for attribute vectors. - - // TODO: methods for document summary. + virtual ~IndexBuilder(); + virtual void startField(uint32_t fieldId) = 0; + virtual void endField() = 0; + virtual void startWord(vespalib::stringref word) = 0; + virtual void endWord() = 0; + virtual void startDocument(uint32_t docId) = 0; + virtual void endDocument() = 0; + virtual void startElement(uint32_t elementId, int32_t weight, uint32_t elementLen) = 0; + virtual void endElement() = 0; + virtual void addOcc(const WordDocElementWordPosFeatures &features) = 0; }; } diff --git a/searchlib/src/vespa/searchlib/test/memoryindex/wrap_inserter.h b/searchlib/src/vespa/searchlib/test/memoryindex/wrap_inserter.h new file mode 100644 index 00000000000..eeb09898aa2 --- /dev/null +++ b/searchlib/src/vespa/searchlib/test/memoryindex/wrap_inserter.h @@ -0,0 +1,64 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/memoryindex/field_index_collection.h> +#include <vespa/searchlib/memoryindex/ordered_field_index_inserter.h> + +namespace search::memoryindex::test { + +/** + * Test class used to populate a FieldIndex. + */ +class WrapInserter { +private: + OrderedFieldIndexInserter& _inserter; + +public: + WrapInserter(FieldIndexCollection& field_indexes, uint32_t field_id) + : _inserter(field_indexes.getFieldIndex(field_id)->getInserter()) + { + } + + WrapInserter(FieldIndex& field_index) + : _inserter(field_index.getInserter()) + { + } + + WrapInserter& word(vespalib::stringref word_) { + _inserter.setNextWord(word_); + return *this; + } + + WrapInserter& add(uint32_t doc_id, const index::DocIdAndFeatures& features) { + _inserter.add(doc_id, features); + return *this; + } + + WrapInserter& add(uint32_t doc_id) { + index::DocIdAndPosOccFeatures features; + features.addNextOcc(0, 0, 1, 1); + return add(doc_id, features); + } + + WrapInserter& remove(uint32_t doc_id) { + _inserter.remove(doc_id); + return *this; + } + + WrapInserter& flush() { + _inserter.flush(); + return *this; + } + + WrapInserter& rewind() { + _inserter.rewind(); + return *this; + } + + datastore::EntryRef getWordRef() { + return _inserter.getWordRef(); + } +}; + +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitor.java b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitor.java index 482969c6625..1a402419ac0 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitor.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitor.java @@ -67,7 +67,7 @@ public class SlobrokMonitor implements AutoCloseable { } List<Mirror.Entry> lookup(String pattern) { - return Arrays.asList(mirror.lookup(pattern)); + return mirror.lookup(pattern); } @Override @@ -76,6 +76,6 @@ public class SlobrokMonitor implements AutoCloseable { } boolean registeredInSlobrok(String slobrokServiceName) { - return mirror.lookup(slobrokServiceName).length > 0; + return !mirror.lookup(slobrokServiceName).isEmpty(); } } diff --git a/statistics/pom.xml b/statistics/pom.xml index 8206d97ad34..491d339ec2a 100644 --- a/statistics/pom.xml +++ b/statistics/pom.xml @@ -34,11 +34,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-core</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>vespalog</artifactId> <version>${project.version}</version> diff --git a/storageapi/src/tests/CMakeLists.txt b/storageapi/src/tests/CMakeLists.txt index ebbf3b8357a..ddc43c70004 100644 --- a/storageapi/src/tests/CMakeLists.txt +++ b/storageapi/src/tests/CMakeLists.txt @@ -7,6 +7,7 @@ vespa_add_executable(storageapi_gtest_runner_app TEST gtest_runner.cpp DEPENDS storageapi_testbuckets + storageapi_testmbusprot storageapi gtest ) @@ -22,8 +23,8 @@ vespa_add_executable(storageapi_testrunner_app TEST testrunner.cpp DEPENDS storageapi_testmessageapi - storageapi_testmbusprot storageapi + vdstestlib ) vespa_add_test( diff --git a/storageapi/src/tests/mbusprot/CMakeLists.txt b/storageapi/src/tests/mbusprot/CMakeLists.txt index 16ced76155c..2801c9a91dd 100644 --- a/storageapi/src/tests/mbusprot/CMakeLists.txt +++ b/storageapi/src/tests/mbusprot/CMakeLists.txt @@ -4,5 +4,5 @@ vespa_add_library(storageapi_testmbusprot storageprotocoltest.cpp DEPENDS storageapi - vdstestlib + gtest ) diff --git a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp index f634667afd5..b1a754bbbab 100644 --- a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp +++ b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp @@ -14,12 +14,16 @@ #include <vespa/document/update/fieldpathupdates.h> #include <vespa/document/test/make_document_bucket.h> #include <vespa/document/test/make_bucket_space.h> -#include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/vespalib/util/growablebytebuffer.h> #include <vespa/vespalib/objects/nbostream.h> + #include <iomanip> #include <sstream> +#include <gtest/gtest.h> + +using namespace ::testing; + using std::shared_ptr; using document::BucketSpace; using document::ByteBuffer; @@ -32,183 +36,103 @@ using document::test::makeBucketSpace; using storage::lib::ClusterState; using vespalib::string; -namespace storage { -namespace api { +namespace vespalib { + +// Needed for GTest to properly understand how to print Version values. +// If not present, it will print the byte values of the presumed memory area +// (which will be overrun for some reason, causing Valgrind to scream at us). +void PrintTo(const vespalib::Version& v, std::ostream* os) { + *os << v.toString(); +} + +} + +namespace storage::api { -struct StorageProtocolTest : public CppUnit::TestFixture { +struct StorageProtocolTest : TestWithParam<vespalib::Version> { document::TestDocMan _docMan; document::Document::SP _testDoc; document::DocumentId _testDocId; + document::BucketId _bucket_id; document::Bucket _bucket; - vespalib::Version _version5_0{5, 0, 12}; - vespalib::Version _version5_1{5, 1, 0}; - vespalib::Version _version5_2{5, 93, 30}; - vespalib::Version _version6_0{6, 240, 0}; + document::BucketId _dummy_remap_bucket{17, 12345}; + BucketInfo _dummy_bucket_info{1,2,3,4,5, true, false, 48}; documentapi::LoadTypeSet _loadTypes; mbusprot::StorageProtocol _protocol; - static std::vector<std::string> _nonVerboseMessageStrings; - static std::vector<std::string> _verboseMessageStrings; - static std::vector<char> _serialization50; static auto constexpr CONDITION_STRING = "There's just one condition"; StorageProtocolTest() : _docMan(), _testDoc(_docMan.createDocument()), _testDocId(_testDoc->getId()), - _bucket(makeDocumentBucket(document::BucketId(16, 0x51))), + _bucket_id(16, 0x51), + _bucket(makeDocumentBucket(_bucket_id)), _protocol(_docMan.getTypeRepoSP(), _loadTypes) { _loadTypes.addLoadType(34, "foo", documentapi::Priority::PRI_NORMAL_2); } + ~StorageProtocolTest(); + + void set_dummy_bucket_info_reply_fields(BucketInfoReply& reply) { + reply.setBucketInfo(_dummy_bucket_info); + reply.remapBucketId(_dummy_remap_bucket); + } + + void assert_bucket_info_reply_fields_propagated(const BucketInfoReply& reply) { + EXPECT_EQ(_dummy_bucket_info, reply.getBucketInfo()); + EXPECT_TRUE(reply.hasBeenRemapped()); + EXPECT_EQ(_dummy_remap_bucket, reply.getBucketId()); + EXPECT_EQ(_bucket_id, reply.getOriginalBucketId()); + } template<typename Command> - std::shared_ptr<Command> copyCommand(const std::shared_ptr<Command>&, vespalib::Version); + std::shared_ptr<Command> copyCommand(const std::shared_ptr<Command>&); template<typename Reply> std::shared_ptr<Reply> copyReply(const std::shared_ptr<Reply>&); - void recordOutput(const api::StorageMessage& msg); - - void recordSerialization50(); - - void testWriteSerialization50(); - void testAddress50(); - void testStringOutputs(); - - void testPut51(); - void testUpdate51(); - void testGet51(); - void testRemove51(); - void testRevert51(); - void testRequestBucketInfo51(); - void testNotifyBucketChange51(); - void testCreateBucket51(); - void testDeleteBucket51(); - void testMergeBucket51(); - void testGetBucketDiff51(); - void testApplyBucketDiff51(); - void testSplitBucket51(); - void testSplitBucketChain51(); - void testJoinBuckets51(); - void testCreateVisitor51(); - void testDestroyVisitor51(); - void testRemoveLocation51(); - void testInternalMessage(); - void testSetBucketState51(); - - void testPutCommand52(); - void testUpdateCommand52(); - void testRemoveCommand52(); - - void testPutCommandWithBucketSpace6_0(); - void testCreateVisitorWithBucketSpace6_0(); - void testRequestBucketInfoWithBucketSpace6_0(); - - void serialized_size_is_used_to_set_approx_size_of_storage_message(); - - CPPUNIT_TEST_SUITE(StorageProtocolTest); - - // Enable to see string outputs of messages - // CPPUNIT_TEST_DISABLED(testStringOutputs); - - // Enable this to write 5.0 serialization to disk - // CPPUNIT_TEST_DISABLED(testWriteSerialization50); - // CPPUNIT_TEST_DISABLED(testAddress50); - - // 5.1 tests - CPPUNIT_TEST(testPut51); - CPPUNIT_TEST(testUpdate51); - CPPUNIT_TEST(testGet51); - CPPUNIT_TEST(testRemove51); - CPPUNIT_TEST(testRevert51); - CPPUNIT_TEST(testRequestBucketInfo51); - CPPUNIT_TEST(testNotifyBucketChange51); - CPPUNIT_TEST(testCreateBucket51); - CPPUNIT_TEST(testDeleteBucket51); - CPPUNIT_TEST(testMergeBucket51); - CPPUNIT_TEST(testGetBucketDiff51); - CPPUNIT_TEST(testApplyBucketDiff51); - CPPUNIT_TEST(testSplitBucket51); - CPPUNIT_TEST(testJoinBuckets51); - CPPUNIT_TEST(testCreateVisitor51); - CPPUNIT_TEST(testDestroyVisitor51); - CPPUNIT_TEST(testRemoveLocation51); - CPPUNIT_TEST(testInternalMessage); - CPPUNIT_TEST(testSetBucketState51); - - // 5.2 tests - CPPUNIT_TEST(testPutCommand52); - CPPUNIT_TEST(testUpdateCommand52); - CPPUNIT_TEST(testRemoveCommand52); - - // 6.0 tests - CPPUNIT_TEST(testPutCommandWithBucketSpace6_0); - CPPUNIT_TEST(testCreateVisitorWithBucketSpace6_0); - CPPUNIT_TEST(testRequestBucketInfoWithBucketSpace6_0); - - CPPUNIT_TEST(serialized_size_is_used_to_set_approx_size_of_storage_message); - - CPPUNIT_TEST_SUITE_END(); }; -CPPUNIT_TEST_SUITE_REGISTRATION(StorageProtocolTest); - -std::vector<std::string> StorageProtocolTest::_nonVerboseMessageStrings; -std::vector<std::string> StorageProtocolTest::_verboseMessageStrings; -std::vector<char> StorageProtocolTest::_serialization50; - -void -StorageProtocolTest::recordOutput(const api::StorageMessage& msg) -{ - std::ostringstream ost; - ost << " "; - msg.print(ost, false, " "); - _nonVerboseMessageStrings.push_back(ost.str()); - ost.str(""); - ost << " "; - msg.print(ost, true, " "); - _verboseMessageStrings.push_back(ost.str()); -} +StorageProtocolTest::~StorageProtocolTest() = default; namespace { - bool debug = false; - - struct ScopedName { - std::string _name; +std::string version_as_gtest_string(TestParamInfo<vespalib::Version> info) { + std::ostringstream ss; + auto& p = info.param; + // Dots are not allowed in test names, so convert to underscores. + ss << p.getMajor() << '_' << p.getMinor() << '_' << p.getMicro(); + return ss.str(); +} - ScopedName(const std::string& s) : _name(s) { - if (debug) std::cerr << "Starting test " << _name << "\n"; - } - ~ScopedName() { - if (debug) std::cerr << "Finished test " << _name << "\n"; - } - }; +} -} // Anonymous namespace +// TODO replace with INSTANTIATE_TEST_SUITE_P on newer gtest versions +INSTANTIATE_TEST_CASE_P(MultiVersionTest, StorageProtocolTest, + Values(vespalib::Version(6, 240, 0), + vespalib::Version(7, 41, 19)), + version_as_gtest_string); namespace { mbus::Message::UP lastCommand; mbus::Reply::UP lastReply; } -void -StorageProtocolTest::testAddress50() -{ +TEST_F(StorageProtocolTest, testAddress50) { StorageMessageAddress address("foo", lib::NodeType::STORAGE, 3); - CPPUNIT_ASSERT_EQUAL(vespalib::string("storage/cluster.foo/storage/3/default"), + EXPECT_EQ(vespalib::string("storage/cluster.foo/storage/3/default"), address.getRoute().toString()); } template<typename Command> std::shared_ptr<Command> -StorageProtocolTest::copyCommand(const std::shared_ptr<Command>& m, vespalib::Version version) +StorageProtocolTest::copyCommand(const std::shared_ptr<Command>& m) { - mbus::Message::UP mbusMessage(new mbusprot::StorageCommand(m)); + auto mbusMessage = std::make_unique<mbusprot::StorageCommand>(m); + auto version = GetParam(); mbus::Blob blob = _protocol.encode(version, *mbusMessage); mbus::Routable::UP copy(_protocol.decode(version, blob)); + assert(copy.get()); - CPPUNIT_ASSERT(copy.get()); - - mbusprot::StorageCommand* copy2(dynamic_cast<mbusprot::StorageCommand*>(copy.get())); - CPPUNIT_ASSERT(copy2 != 0); + auto* copy2 = dynamic_cast<mbusprot::StorageCommand*>(copy.get()); + assert(copy2 != nullptr); StorageCommand::SP internalMessage(copy2->getCommand()); lastCommand = std::move(mbusMessage); @@ -219,80 +143,112 @@ StorageProtocolTest::copyCommand(const std::shared_ptr<Command>& m, vespalib::Ve template<typename Reply> std::shared_ptr<Reply> StorageProtocolTest::copyReply(const std::shared_ptr<Reply>& m) { - mbus::Reply::UP mbusMessage(new mbusprot::StorageReply(m)); - mbus::Blob blob = _protocol.encode(_version5_1, *mbusMessage); - mbus::Routable::UP copy(_protocol.decode(_version5_1, blob)); - CPPUNIT_ASSERT(copy.get()); - mbusprot::StorageReply* copy2( - dynamic_cast<mbusprot::StorageReply*>(copy.get())); - CPPUNIT_ASSERT(copy2 != 0); + auto mbusMessage = std::make_unique<mbusprot::StorageReply>(m); + auto version = GetParam(); + mbus::Blob blob = _protocol.encode(version, *mbusMessage); + mbus::Routable::UP copy(_protocol.decode(version, blob)); + assert(copy.get()); + + auto* copy2 = dynamic_cast<mbusprot::StorageReply*>(copy.get()); + assert(copy2 != nullptr); + copy2->setMessage(std::move(lastCommand)); - StorageReply::SP internalMessage(copy2->getReply()); + auto internalMessage = copy2->getReply(); lastReply = std::move(mbusMessage); lastCommand = copy2->getMessage(); return std::dynamic_pointer_cast<Reply>(internalMessage); } -void -StorageProtocolTest::recordSerialization50() -{ - assert(lastCommand.get()); - assert(lastReply.get()); - for (uint32_t j=0; j<2; ++j) { - mbusprot::StorageMessage& msg(j == 0 - ? dynamic_cast<mbusprot::StorageMessage&>(*lastCommand) - : dynamic_cast<mbusprot::StorageMessage&>(*lastReply)); - msg.getInternalMessage()->forceMsgId(0); - mbus::Routable& routable(j == 0 - ? dynamic_cast<mbus::Routable&>(*lastCommand) - : dynamic_cast<mbus::Routable&>(*lastReply)); - mbus::Blob blob = _protocol.encode(_version5_0, routable); - _serialization50.push_back('\n'); - std::string type(msg.getInternalMessage()->getType().toString()); - for (uint32_t i=0, n=type.size(); i<n; ++i) { - _serialization50.push_back(type[i]); - } - _serialization50.push_back('\n'); - - for (uint32_t i=0, n=blob.size(); i<n; ++i) { - _serialization50.push_back(blob.data()[i]); - } - } -} - -void -StorageProtocolTest::testPut51() -{ - ScopedName test("testPut51"); - PutCommand::SP cmd(new PutCommand(_bucket, _testDoc, 14)); +TEST_P(StorageProtocolTest, put) { + auto cmd = std::make_shared<PutCommand>(_bucket, _testDoc, 14); cmd->setUpdateTimestamp(Timestamp(13)); cmd->setLoadType(_loadTypes["foo"]); - PutCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(*_testDoc, *cmd2->getDocument()); - CPPUNIT_ASSERT_EQUAL(vespalib::string("foo"), cmd2->getLoadType().getName()); - CPPUNIT_ASSERT_EQUAL(Timestamp(14), cmd2->getTimestamp()); - CPPUNIT_ASSERT_EQUAL(Timestamp(13), cmd2->getUpdateTimestamp()); - - PutReply::SP reply(new PutReply(*cmd2)); - CPPUNIT_ASSERT(reply->hasDocument()); - CPPUNIT_ASSERT_EQUAL(*_testDoc, *reply->getDocument()); - PutReply::SP reply2(copyReply(reply)); - CPPUNIT_ASSERT(reply2->hasDocument()); - CPPUNIT_ASSERT_EQUAL(*_testDoc, *reply->getDocument()); - CPPUNIT_ASSERT_EQUAL(_testDoc->getId(), reply2->getDocumentId()); - CPPUNIT_ASSERT_EQUAL(Timestamp(14), reply2->getTimestamp()); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); -} - -void -StorageProtocolTest::testUpdate51() -{ - ScopedName test("testUpdate51"); - document::DocumentUpdate::SP update(new document::DocumentUpdate(_docMan.getTypeRepo(), *_testDoc->getDataType(), _testDoc->getId())); - std::shared_ptr<document::AssignValueUpdate> assignUpdate(new document::AssignValueUpdate(document::IntFieldValue(17))); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); + EXPECT_EQ(*_testDoc, *cmd2->getDocument()); + EXPECT_EQ(vespalib::string("foo"), cmd2->getLoadType().getName()); + EXPECT_EQ(Timestamp(14), cmd2->getTimestamp()); + EXPECT_EQ(Timestamp(13), cmd2->getUpdateTimestamp()); + + auto reply = std::make_shared<PutReply>(*cmd2); + ASSERT_TRUE(reply->hasDocument()); + EXPECT_EQ(*_testDoc, *reply->getDocument()); + set_dummy_bucket_info_reply_fields(*reply); + auto reply2 = copyReply(reply); + ASSERT_TRUE(reply2->hasDocument()); + EXPECT_EQ(*_testDoc, *reply->getDocument()); + EXPECT_EQ(_testDoc->getId(), reply2->getDocumentId()); + EXPECT_EQ(Timestamp(14), reply2->getTimestamp()); + EXPECT_NO_FATAL_FAILURE(assert_bucket_info_reply_fields_propagated(*reply2)); +} + +TEST_P(StorageProtocolTest, response_without_remapped_bucket_preserves_original_bucket) { + auto cmd = std::make_shared<PutCommand>(_bucket, _testDoc, 14); + auto cmd2 = copyCommand(cmd); + auto reply = std::make_shared<PutReply>(*cmd2); + auto reply2 = copyReply(reply); + + EXPECT_FALSE(reply2->hasBeenRemapped()); + EXPECT_EQ(_bucket_id, reply2->getBucketId()); + EXPECT_EQ(document::BucketId(), reply2->getOriginalBucketId()); +} + +TEST_P(StorageProtocolTest, invalid_bucket_info_is_propagated) { + auto cmd = std::make_shared<PutCommand>(_bucket, _testDoc, 14); + auto cmd2 = copyCommand(cmd); + auto reply = std::make_shared<PutReply>(*cmd2); + BucketInfo invalid_info; + EXPECT_FALSE(invalid_info.valid()); + reply->setBucketInfo(invalid_info); + auto reply2 = copyReply(reply); + + EXPECT_EQ(invalid_info, reply2->getBucketInfo()); + EXPECT_FALSE(reply2->getBucketInfo().valid()); +} + +TEST_P(StorageProtocolTest, all_zero_bucket_info_is_propagated) { + auto cmd = std::make_shared<PutCommand>(_bucket, _testDoc, 14); + auto cmd2 = copyCommand(cmd); + auto reply = std::make_shared<PutReply>(*cmd2); + BucketInfo zero_info(0, 0, 0, 0, 0, false, false, 0); + reply->setBucketInfo(zero_info); + auto reply2 = copyReply(reply); + + EXPECT_EQ(zero_info, reply2->getBucketInfo()); +} + +TEST_P(StorageProtocolTest, request_metadata_is_propagated) { + auto cmd = std::make_shared<PutCommand>(_bucket, _testDoc, 14); + cmd->forceMsgId(12345); + cmd->setPriority(50); + cmd->setLoadType(_loadTypes["foo"]); + cmd->setSourceIndex(321); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(12345, cmd2->getMsgId()); + EXPECT_EQ(50, cmd2->getPriority()); + EXPECT_EQ(_loadTypes["foo"].getId(), cmd2->getLoadType().getId()); + EXPECT_EQ(321, cmd2->getSourceIndex()); +} + +TEST_P(StorageProtocolTest, response_metadata_is_propagated) { + auto cmd = std::make_shared<PutCommand>(_bucket, _testDoc, 14); + auto cmd2 = copyCommand(cmd); + auto reply = std::make_shared<PutReply>(*cmd2); + reply->forceMsgId(1234); + reply->setPriority(101); + ReturnCode result(ReturnCode::TEST_AND_SET_CONDITION_FAILED, "foo is not bar"); + reply->setResult(result); + + auto reply2 = copyReply(reply); + EXPECT_EQ(result, reply2->getResult()); + EXPECT_EQ(1234, reply->getMsgId()); + EXPECT_EQ(101, reply->getPriority()); +} + +TEST_P(StorageProtocolTest, update) { + auto update = std::make_shared<document::DocumentUpdate>( + _docMan.getTypeRepo(), *_testDoc->getDataType(), _testDoc->getId()); + auto assignUpdate = std::make_shared<document::AssignValueUpdate>(document::IntFieldValue(17)); document::FieldUpdate fieldUpdate(_testDoc->getField("headerval")); fieldUpdate.addUpdate(*assignUpdate); update->addUpdate(fieldUpdate); @@ -300,217 +256,157 @@ StorageProtocolTest::testUpdate51() update->addFieldPathUpdate(document::FieldPathUpdate::CP( new document::RemoveFieldPathUpdate("headerval", "testdoctype1.headerval > 0"))); - UpdateCommand::SP cmd(new UpdateCommand(_bucket, update, 14)); - CPPUNIT_ASSERT_EQUAL(Timestamp(0), cmd->getOldTimestamp()); + auto cmd = std::make_shared<UpdateCommand>(_bucket, update, 14); + EXPECT_EQ(Timestamp(0), cmd->getOldTimestamp()); cmd->setOldTimestamp(10); - UpdateCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(_testDocId, cmd2->getDocumentId()); - CPPUNIT_ASSERT_EQUAL(Timestamp(14), cmd2->getTimestamp()); - CPPUNIT_ASSERT_EQUAL(Timestamp(10), cmd2->getOldTimestamp()); - CPPUNIT_ASSERT_EQUAL(*update, *cmd2->getUpdate()); - - UpdateReply::SP reply(new UpdateReply(*cmd2, 8)); - UpdateReply::SP reply2(copyReply(reply)); - CPPUNIT_ASSERT_EQUAL(_testDocId, reply2->getDocumentId()); - CPPUNIT_ASSERT_EQUAL(Timestamp(14), reply2->getTimestamp()); - CPPUNIT_ASSERT_EQUAL(Timestamp(8), reply->getOldTimestamp()); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); -} - -void -StorageProtocolTest::testGet51() -{ - ScopedName test("testGet51"); - GetCommand::SP cmd(new GetCommand(_bucket, _testDocId, "foo,bar,vekterli", 123)); - GetCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(_testDocId, cmd2->getDocumentId()); - CPPUNIT_ASSERT_EQUAL(Timestamp(123), cmd2->getBeforeTimestamp()); - CPPUNIT_ASSERT_EQUAL(vespalib::string("foo,bar,vekterli"), cmd2->getFieldSet()); - - GetReply::SP reply(new GetReply(*cmd2, _testDoc, 100)); - GetReply::SP reply2(copyReply(reply)); - CPPUNIT_ASSERT(reply2.get()); - CPPUNIT_ASSERT(reply2->getDocument().get()); - CPPUNIT_ASSERT_EQUAL(*_testDoc, *reply2->getDocument()); - CPPUNIT_ASSERT_EQUAL(_testDoc->getId(), reply2->getDocumentId()); - CPPUNIT_ASSERT_EQUAL(Timestamp(123), reply2->getBeforeTimestamp()); - CPPUNIT_ASSERT_EQUAL(Timestamp(100), reply2->getLastModifiedTimestamp()); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); -} - -void -StorageProtocolTest::testRemove51() -{ - ScopedName test("testRemove51"); - RemoveCommand::SP cmd(new RemoveCommand(_bucket, _testDocId, 159)); - RemoveCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(_testDocId, cmd2->getDocumentId()); - CPPUNIT_ASSERT_EQUAL(Timestamp(159), cmd2->getTimestamp()); - - RemoveReply::SP reply(new RemoveReply(*cmd2, 48)); - reply->setBucketInfo(BucketInfo(1,2,3,4,5, true, false, 48)); - - RemoveReply::SP reply2(copyReply(reply)); - CPPUNIT_ASSERT_EQUAL(_testDocId, reply2->getDocumentId()); - CPPUNIT_ASSERT_EQUAL(Timestamp(159), reply2->getTimestamp()); - CPPUNIT_ASSERT_EQUAL(Timestamp(48), reply2->getOldTimestamp()); - CPPUNIT_ASSERT_EQUAL(BucketInfo(1,2,3,4,5, true, false, 48), - reply2->getBucketInfo()); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); -} - -void -StorageProtocolTest::testRevert51() -{ - ScopedName test("testRevertCommand51"); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); + EXPECT_EQ(_testDocId, cmd2->getDocumentId()); + EXPECT_EQ(Timestamp(14), cmd2->getTimestamp()); + EXPECT_EQ(Timestamp(10), cmd2->getOldTimestamp()); + EXPECT_EQ(*update, *cmd2->getUpdate()); + + auto reply = std::make_shared<UpdateReply>(*cmd2, 8); + set_dummy_bucket_info_reply_fields(*reply); + auto reply2 = copyReply(reply); + EXPECT_EQ(_testDocId, reply2->getDocumentId()); + EXPECT_EQ(Timestamp(14), reply2->getTimestamp()); + EXPECT_EQ(Timestamp(8), reply->getOldTimestamp()); + EXPECT_NO_FATAL_FAILURE(assert_bucket_info_reply_fields_propagated(*reply2)); +} + +TEST_P(StorageProtocolTest, get) { + auto cmd = std::make_shared<GetCommand>(_bucket, _testDocId, "foo,bar,vekterli", 123); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); + EXPECT_EQ(_testDocId, cmd2->getDocumentId()); + EXPECT_EQ(Timestamp(123), cmd2->getBeforeTimestamp()); + EXPECT_EQ(vespalib::string("foo,bar,vekterli"), cmd2->getFieldSet()); + + auto reply = std::make_shared<GetReply>(*cmd2, _testDoc, 100); + set_dummy_bucket_info_reply_fields(*reply); + auto reply2 = copyReply(reply); + ASSERT_TRUE(reply2.get() != nullptr); + ASSERT_TRUE(reply2->getDocument().get() != nullptr); + EXPECT_EQ(*_testDoc, *reply2->getDocument()); + EXPECT_EQ(_testDoc->getId(), reply2->getDocumentId()); + EXPECT_EQ(Timestamp(123), reply2->getBeforeTimestamp()); + EXPECT_EQ(Timestamp(100), reply2->getLastModifiedTimestamp()); + EXPECT_NO_FATAL_FAILURE(assert_bucket_info_reply_fields_propagated(*reply2)); +} + +TEST_P(StorageProtocolTest, remove) { + auto cmd = std::make_shared<RemoveCommand>(_bucket, _testDocId, 159); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); + EXPECT_EQ(_testDocId, cmd2->getDocumentId()); + EXPECT_EQ(Timestamp(159), cmd2->getTimestamp()); + + auto reply = std::make_shared<RemoveReply>(*cmd2, 48); + set_dummy_bucket_info_reply_fields(*reply); + + auto reply2 = copyReply(reply); + EXPECT_EQ(_testDocId, reply2->getDocumentId()); + EXPECT_EQ(Timestamp(159), reply2->getTimestamp()); + EXPECT_EQ(Timestamp(48), reply2->getOldTimestamp()); + EXPECT_NO_FATAL_FAILURE(assert_bucket_info_reply_fields_propagated(*reply2)); +} + +TEST_P(StorageProtocolTest, revert) { std::vector<Timestamp> tokens; tokens.push_back(59); - RevertCommand::SP cmd(new RevertCommand(_bucket, tokens)); - RevertCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(tokens, cmd2->getRevertTokens()); + auto cmd = std::make_shared<RevertCommand>(_bucket, tokens); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); + EXPECT_EQ(tokens, cmd2->getRevertTokens()); - RevertReply::SP reply(new RevertReply(*cmd2)); - BucketInfo info(0x12345432, 101, 520); - reply->setBucketInfo(info); - RevertReply::SP reply2(copyReply(reply)); - - CPPUNIT_ASSERT_EQUAL(info, reply2->getBucketInfo()); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); + auto reply = std::make_shared<RevertReply>(*cmd2); + set_dummy_bucket_info_reply_fields(*reply); + auto reply2 = copyReply(reply); + EXPECT_NO_FATAL_FAILURE(assert_bucket_info_reply_fields_propagated(*reply2)); } -void -StorageProtocolTest::testRequestBucketInfo51() -{ - ScopedName test("testRequestBucketInfo51"); +TEST_P(StorageProtocolTest, request_bucket_info) { { std::vector<document::BucketId> ids; ids.push_back(document::BucketId(3)); ids.push_back(document::BucketId(7)); - RequestBucketInfoCommand::SP cmd(new RequestBucketInfoCommand(makeBucketSpace(), ids)); - RequestBucketInfoCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(ids, cmd2->getBuckets()); - CPPUNIT_ASSERT(!cmd2->hasSystemState()); - - recordOutput(*cmd2); + auto cmd = std::make_shared<RequestBucketInfoCommand>(makeBucketSpace(), ids); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(ids, cmd2->getBuckets()); + EXPECT_FALSE(cmd2->hasSystemState()); } { ClusterState state("distributor:3 .1.s:d"); - RequestBucketInfoCommand::SP cmd(new RequestBucketInfoCommand( - makeBucketSpace(), - 3, state, "14")); - RequestBucketInfoCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT(cmd2->hasSystemState()); - CPPUNIT_ASSERT_EQUAL(uint16_t(3), cmd2->getDistributor()); - CPPUNIT_ASSERT_EQUAL(state, cmd2->getSystemState()); - CPPUNIT_ASSERT_EQUAL(size_t(0), cmd2->getBuckets().size()); - - RequestBucketInfoReply::SP reply(new RequestBucketInfoReply(*cmd)); + auto cmd = std::make_shared<RequestBucketInfoCommand>(makeBucketSpace(), 3, state, "14"); + auto cmd2 = copyCommand(cmd); + ASSERT_TRUE(cmd2->hasSystemState()); + EXPECT_EQ(uint16_t(3), cmd2->getDistributor()); + EXPECT_EQ(state, cmd2->getSystemState()); + EXPECT_EQ(size_t(0), cmd2->getBuckets().size()); + + auto reply = std::make_shared<RequestBucketInfoReply>(*cmd); RequestBucketInfoReply::Entry e; e._bucketId = document::BucketId(4); const uint64_t lastMod = 0x1337cafe98765432ULL; e._info = BucketInfo(43, 24, 123, 44, 124, false, true, lastMod); reply->getBucketInfo().push_back(e); - RequestBucketInfoReply::SP reply2(copyReply(reply)); - CPPUNIT_ASSERT_EQUAL(size_t(1), reply2->getBucketInfo().size()); + auto reply2 = copyReply(reply); + EXPECT_EQ(size_t(1), reply2->getBucketInfo().size()); auto& entries(reply2->getBucketInfo()); - CPPUNIT_ASSERT_EQUAL(e, entries[0]); + EXPECT_EQ(e, entries[0]); // "Last modified" not counted by operator== for some reason. Testing // separately until we can figure out if this is by design or not. - CPPUNIT_ASSERT_EQUAL(lastMod, entries[0]._info.getLastModified()); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); + EXPECT_EQ(lastMod, entries[0]._info.getLastModified()); } } -void -StorageProtocolTest::testNotifyBucketChange51() -{ - ScopedName test("testNotifyBucketChange51"); - BucketInfo info(2, 3, 4); - document::BucketId modifiedBucketId(20, 1000); - document::Bucket modifiedBucket(makeDocumentBucket(modifiedBucketId)); - NotifyBucketChangeCommand::SP cmd(new NotifyBucketChangeCommand( - modifiedBucket, info)); - NotifyBucketChangeCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(document::BucketId(20, 1000), - cmd2->getBucketId()); - CPPUNIT_ASSERT_EQUAL(info, cmd2->getBucketInfo()); - - NotifyBucketChangeReply::SP reply(new NotifyBucketChangeReply(*cmd)); - NotifyBucketChangeReply::SP reply2(copyReply(reply)); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); -} - -void -StorageProtocolTest::testCreateBucket51() -{ - ScopedName test("testCreateBucket51"); - document::BucketId bucketId(623); - document::Bucket bucket(makeDocumentBucket(bucketId)); +TEST_P(StorageProtocolTest, notify_bucket_change) { + auto cmd = std::make_shared<NotifyBucketChangeCommand>(_bucket, _dummy_bucket_info); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); + EXPECT_EQ(_dummy_bucket_info, cmd2->getBucketInfo()); - CreateBucketCommand::SP cmd(new CreateBucketCommand(bucket)); - CreateBucketCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(bucketId, cmd2->getBucketId()); + auto reply = std::make_shared<NotifyBucketChangeReply>(*cmd); + auto reply2 = copyReply(reply); +} - CreateBucketReply::SP reply(new CreateBucketReply(*cmd)); - CreateBucketReply::SP reply2(copyReply(reply)); - CPPUNIT_ASSERT_EQUAL(bucketId, reply2->getBucketId()); +TEST_P(StorageProtocolTest, create_bucket_without_activation) { + auto cmd = std::make_shared<CreateBucketCommand>(_bucket); + EXPECT_FALSE(cmd->getActive()); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); + EXPECT_FALSE(cmd2->getActive()); - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); + auto reply = std::make_shared<CreateBucketReply>(*cmd); + set_dummy_bucket_info_reply_fields(*reply); + auto reply2 = copyReply(reply); + EXPECT_NO_FATAL_FAILURE(assert_bucket_info_reply_fields_propagated(*reply2)); } -void -StorageProtocolTest::testDeleteBucket51() -{ - ScopedName test("testDeleteBucket51"); - document::BucketId bucketId(623); - document::Bucket bucket(makeDocumentBucket(bucketId)); - - DeleteBucketCommand::SP cmd(new DeleteBucketCommand(bucket)); - BucketInfo info(0x100, 200, 300); - cmd->setBucketInfo(info); - DeleteBucketCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(bucketId, cmd2->getBucketId()); - CPPUNIT_ASSERT_EQUAL(info, cmd2->getBucketInfo()); - - DeleteBucketReply::SP reply(new DeleteBucketReply(*cmd)); +TEST_P(StorageProtocolTest, create_bucket_propagates_activation_flag) { + auto cmd = std::make_shared<CreateBucketCommand>(_bucket); + cmd->setActive(true); + auto cmd2 = copyCommand(cmd); + EXPECT_TRUE(cmd2->getActive()); +} + +TEST_P(StorageProtocolTest, delete_bucket) { + auto cmd = std::make_shared<DeleteBucketCommand>(_bucket); + cmd->setBucketInfo(_dummy_bucket_info); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); + EXPECT_EQ(_dummy_bucket_info, cmd2->getBucketInfo()); + + auto reply = std::make_shared<DeleteBucketReply>(*cmd); // Not set automatically by constructor reply->setBucketInfo(cmd2->getBucketInfo()); - DeleteBucketReply::SP reply2(copyReply(reply)); - CPPUNIT_ASSERT_EQUAL(bucketId, reply2->getBucketId()); - CPPUNIT_ASSERT_EQUAL(info, reply2->getBucketInfo()); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); + auto reply2 = copyReply(reply); + EXPECT_EQ(_bucket_id, reply2->getBucketId()); + EXPECT_EQ(_dummy_bucket_info, reply2->getBucketInfo()); } -void -StorageProtocolTest::testMergeBucket51() -{ - ScopedName test("testMergeBucket51"); - document::BucketId bucketId(623); - document::Bucket bucket(makeDocumentBucket(bucketId)); - +TEST_P(StorageProtocolTest, merge_bucket) { typedef api::MergeBucketCommand::Node Node; std::vector<Node> nodes; nodes.push_back(Node(4, false)); @@ -522,152 +418,98 @@ StorageProtocolTest::testMergeBucket51() chain.push_back(7); chain.push_back(14); - MergeBucketCommand::SP cmd( - new MergeBucketCommand(bucket, nodes, Timestamp(1234), 567, chain)); - MergeBucketCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(bucketId, cmd2->getBucketId()); - CPPUNIT_ASSERT_EQUAL(nodes, cmd2->getNodes()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1234), cmd2->getMaxTimestamp()); - CPPUNIT_ASSERT_EQUAL(uint32_t(567), cmd2->getClusterStateVersion()); - CPPUNIT_ASSERT_EQUAL(chain, cmd2->getChain()); - - MergeBucketReply::SP reply(new MergeBucketReply(*cmd)); - MergeBucketReply::SP reply2(copyReply(reply)); - CPPUNIT_ASSERT_EQUAL(bucketId, reply2->getBucketId()); - CPPUNIT_ASSERT_EQUAL(nodes, reply2->getNodes()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1234), reply2->getMaxTimestamp()); - CPPUNIT_ASSERT_EQUAL(uint32_t(567), reply2->getClusterStateVersion()); - CPPUNIT_ASSERT_EQUAL(chain, reply2->getChain()); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); -} - -void -StorageProtocolTest::testSplitBucket51() -{ - ScopedName test("testSplitBucket51"); - - document::BucketId bucketId(16, 0); - document::Bucket bucket(makeDocumentBucket(bucketId)); - SplitBucketCommand::SP cmd(new SplitBucketCommand(bucket)); - CPPUNIT_ASSERT_EQUAL(0u, (uint32_t) cmd->getMinSplitBits()); - CPPUNIT_ASSERT_EQUAL(58u, (uint32_t) cmd->getMaxSplitBits()); - CPPUNIT_ASSERT_EQUAL(std::numeric_limits<uint32_t>().max(), - cmd->getMinByteSize()); - CPPUNIT_ASSERT_EQUAL(std::numeric_limits<uint32_t>().max(), - cmd->getMinDocCount()); + auto cmd = std::make_shared<MergeBucketCommand>(_bucket, nodes, Timestamp(1234), 567, chain); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); + EXPECT_EQ(nodes, cmd2->getNodes()); + EXPECT_EQ(Timestamp(1234), cmd2->getMaxTimestamp()); + EXPECT_EQ(uint32_t(567), cmd2->getClusterStateVersion()); + EXPECT_EQ(chain, cmd2->getChain()); + + auto reply = std::make_shared<MergeBucketReply>(*cmd); + auto reply2 = copyReply(reply); + EXPECT_EQ(_bucket_id, reply2->getBucketId()); + EXPECT_EQ(nodes, reply2->getNodes()); + EXPECT_EQ(Timestamp(1234), reply2->getMaxTimestamp()); + EXPECT_EQ(uint32_t(567), reply2->getClusterStateVersion()); + EXPECT_EQ(chain, reply2->getChain()); +} + +TEST_P(StorageProtocolTest, split_bucket) { + auto cmd = std::make_shared<SplitBucketCommand>(_bucket); + EXPECT_EQ(0u, cmd->getMinSplitBits()); + EXPECT_EQ(58u, cmd->getMaxSplitBits()); + EXPECT_EQ(std::numeric_limits<uint32_t>().max(), cmd->getMinByteSize()); + EXPECT_EQ(std::numeric_limits<uint32_t>().max(), cmd->getMinDocCount()); cmd->setMinByteSize(1000); cmd->setMinDocCount(5); cmd->setMaxSplitBits(40); cmd->setMinSplitBits(20); - SplitBucketCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(20u, (uint32_t) cmd2->getMinSplitBits()); - CPPUNIT_ASSERT_EQUAL(40u, (uint32_t) cmd2->getMaxSplitBits()); - CPPUNIT_ASSERT_EQUAL(1000u, cmd2->getMinByteSize()); - CPPUNIT_ASSERT_EQUAL(5u, cmd2->getMinDocCount()); - - SplitBucketReply::SP reply(new SplitBucketReply(*cmd2)); - reply->getSplitInfo().push_back(SplitBucketReply::Entry( - document::BucketId(17, 0), BucketInfo(100, 1000, 10000, true, true))); - reply->getSplitInfo().push_back(SplitBucketReply::Entry( - document::BucketId(17, 1), BucketInfo(101, 1001, 10001, true, true))); - SplitBucketReply::SP reply2(copyReply(reply)); - - CPPUNIT_ASSERT_EQUAL(bucketId, reply2->getBucketId()); - CPPUNIT_ASSERT_EQUAL(size_t(2), reply2->getSplitInfo().size()); - CPPUNIT_ASSERT_EQUAL(document::BucketId(17, 0), - reply2->getSplitInfo()[0].first); - CPPUNIT_ASSERT_EQUAL(document::BucketId(17, 1), - reply2->getSplitInfo()[1].first); - CPPUNIT_ASSERT_EQUAL(BucketInfo(100, 1000, 10000, true, true), - reply2->getSplitInfo()[0].second); - CPPUNIT_ASSERT_EQUAL(BucketInfo(101, 1001, 10001, true, true), - reply2->getSplitInfo()[1].second); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); -} - -void -StorageProtocolTest::testJoinBuckets51() -{ - ScopedName test("testJoinBuckets51"); - document::BucketId bucketId(16, 0); - document::Bucket bucket(makeDocumentBucket(bucketId)); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); + EXPECT_EQ(20u, cmd2->getMinSplitBits()); + EXPECT_EQ(40u, cmd2->getMaxSplitBits()); + EXPECT_EQ(1000u, cmd2->getMinByteSize()); + EXPECT_EQ(5u, cmd2->getMinDocCount()); + + auto reply = std::make_shared<SplitBucketReply>(*cmd2); + reply->getSplitInfo().emplace_back(document::BucketId(17, 0), BucketInfo(100, 1000, 10000, true, true)); + reply->getSplitInfo().emplace_back(document::BucketId(17, 1), BucketInfo(101, 1001, 10001, true, true)); + auto reply2 = copyReply(reply); + + EXPECT_EQ(_bucket, reply2->getBucket()); + EXPECT_EQ(size_t(2), reply2->getSplitInfo().size()); + EXPECT_EQ(document::BucketId(17, 0), reply2->getSplitInfo()[0].first); + EXPECT_EQ(document::BucketId(17, 1), reply2->getSplitInfo()[1].first); + EXPECT_EQ(BucketInfo(100, 1000, 10000, true, true), reply2->getSplitInfo()[0].second); + EXPECT_EQ(BucketInfo(101, 1001, 10001, true, true), reply2->getSplitInfo()[1].second); +} + +TEST_P(StorageProtocolTest, join_buckets) { std::vector<document::BucketId> sources; sources.push_back(document::BucketId(17, 0)); sources.push_back(document::BucketId(17, 1)); - JoinBucketsCommand::SP cmd(new JoinBucketsCommand(bucket)); + auto cmd = std::make_shared<JoinBucketsCommand>(_bucket); cmd->getSourceBuckets() = sources; cmd->setMinJoinBits(3); - JoinBucketsCommand::SP cmd2(copyCommand(cmd, _version5_1)); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); - JoinBucketsReply::SP reply(new JoinBucketsReply(*cmd2)); + auto reply = std::make_shared<JoinBucketsReply>(*cmd2); reply->setBucketInfo(BucketInfo(3,4,5)); - JoinBucketsReply::SP reply2(copyReply(reply)); + auto reply2 = copyReply(reply); - CPPUNIT_ASSERT_EQUAL(sources, reply2->getSourceBuckets()); - CPPUNIT_ASSERT_EQUAL(3, (int)cmd2->getMinJoinBits()); - CPPUNIT_ASSERT_EQUAL(BucketInfo(3,4,5), reply2->getBucketInfo()); - CPPUNIT_ASSERT_EQUAL(bucketId, reply2->getBucketId()); - - recordOutput(*cmd2); - recordOutput(*reply2); + EXPECT_EQ(sources, reply2->getSourceBuckets()); + EXPECT_EQ(3, cmd2->getMinJoinBits()); + EXPECT_EQ(BucketInfo(3,4,5), reply2->getBucketInfo()); + EXPECT_EQ(_bucket, reply2->getBucket()); } -void -StorageProtocolTest::testDestroyVisitor51() -{ - ScopedName test("testDestroyVisitor51"); +TEST_P(StorageProtocolTest, destroy_visitor) { + auto cmd = std::make_shared<DestroyVisitorCommand>("instance"); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ("instance", cmd2->getInstanceId()); - DestroyVisitorCommand::SP cmd( - new DestroyVisitorCommand("instance")); - DestroyVisitorCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(string("instance"), cmd2->getInstanceId()); - - DestroyVisitorReply::SP reply(new DestroyVisitorReply(*cmd2)); - DestroyVisitorReply::SP reply2(copyReply(reply)); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); + auto reply = std::make_shared<DestroyVisitorReply>(*cmd2); + auto reply2 = copyReply(reply); } -void -StorageProtocolTest::testRemoveLocation51() -{ - ScopedName test("testRemoveLocation51"); - document::BucketId bucketId(16, 1234); - document::Bucket bucket(makeDocumentBucket(bucketId)); +TEST_P(StorageProtocolTest, remove_location) { + auto cmd = std::make_shared<RemoveLocationCommand>("id.group == \"mygroup\"", _bucket); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ("id.group == \"mygroup\"", cmd2->getDocumentSelection()); + EXPECT_EQ(_bucket, cmd2->getBucket()); - RemoveLocationCommand::SP cmd( - new RemoveLocationCommand("id.group == \"mygroup\"", bucket)); - RemoveLocationCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(vespalib::string("id.group == \"mygroup\""), cmd2->getDocumentSelection()); - CPPUNIT_ASSERT_EQUAL(bucketId, cmd2->getBucketId()); - - RemoveLocationReply::SP reply(new RemoveLocationReply(*cmd2)); - RemoveLocationReply::SP reply2(copyReply(reply)); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); + auto reply = std::make_shared<RemoveLocationReply>(*cmd2); + auto reply2 = copyReply(reply); } -void -StorageProtocolTest::testCreateVisitor51() -{ - ScopedName test("testCreateVisitor51"); - +TEST_P(StorageProtocolTest, create_visitor) { std::vector<document::BucketId> buckets; buckets.push_back(document::BucketId(16, 1)); buckets.push_back(document::BucketId(16, 2)); - CreateVisitorCommand::SP cmd( - new CreateVisitorCommand(makeBucketSpace(), "library", "id", "doc selection")); + auto cmd = std::make_shared<CreateVisitorCommand>(makeBucketSpace(), "library", "id", "doc selection"); cmd->setControlDestination("controldest"); cmd->setDataDestination("datadest"); cmd->setVisitorCmdId(1); @@ -681,40 +523,26 @@ StorageProtocolTest::testCreateVisitor51() cmd->setFieldSet("foo,bar,vekterli"); cmd->setVisitInconsistentBuckets(); cmd->setQueueTimeout(100); - cmd->setVisitorOrdering(document::OrderingSpecification::DESCENDING); cmd->setPriority(149); - CreateVisitorCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(string("library"), cmd2->getLibraryName()); - CPPUNIT_ASSERT_EQUAL(string("id"), cmd2->getInstanceId()); - CPPUNIT_ASSERT_EQUAL(string("doc selection"), - cmd2->getDocumentSelection()); - CPPUNIT_ASSERT_EQUAL(string("controldest"), - cmd2->getControlDestination()); - CPPUNIT_ASSERT_EQUAL(string("datadest"), cmd2->getDataDestination()); - CPPUNIT_ASSERT_EQUAL(api::Timestamp(123), cmd2->getFromTime()); - CPPUNIT_ASSERT_EQUAL(api::Timestamp(456), cmd2->getToTime()); - CPPUNIT_ASSERT_EQUAL(2u, cmd2->getMaximumPendingReplyCount()); - CPPUNIT_ASSERT_EQUAL(buckets, cmd2->getBuckets()); - CPPUNIT_ASSERT_EQUAL(vespalib::string("foo,bar,vekterli"), cmd2->getFieldSet()); - CPPUNIT_ASSERT(cmd2->visitInconsistentBuckets()); - CPPUNIT_ASSERT_EQUAL(document::OrderingSpecification::DESCENDING, cmd2->getVisitorOrdering()); - CPPUNIT_ASSERT_EQUAL(149, (int)cmd2->getPriority()); - - CreateVisitorReply::SP reply(new CreateVisitorReply(*cmd2)); - CreateVisitorReply::SP reply2(copyReply(reply)); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); -} - -void -StorageProtocolTest::testGetBucketDiff51() -{ - ScopedName test("testGetBucketDiff51"); - document::BucketId bucketId(623); - document::Bucket bucket(makeDocumentBucket(bucketId)); - + auto cmd2 = copyCommand(cmd); + EXPECT_EQ("library", cmd2->getLibraryName()); + EXPECT_EQ("id", cmd2->getInstanceId()); + EXPECT_EQ("doc selection", cmd2->getDocumentSelection()); + EXPECT_EQ("controldest", cmd2->getControlDestination()); + EXPECT_EQ("datadest", cmd2->getDataDestination()); + EXPECT_EQ(api::Timestamp(123), cmd2->getFromTime()); + EXPECT_EQ(api::Timestamp(456), cmd2->getToTime()); + EXPECT_EQ(2u, cmd2->getMaximumPendingReplyCount()); + EXPECT_EQ(buckets, cmd2->getBuckets()); + EXPECT_EQ("foo,bar,vekterli", cmd2->getFieldSet()); + EXPECT_TRUE(cmd2->visitInconsistentBuckets()); + EXPECT_EQ(149, cmd2->getPriority()); + + auto reply = std::make_shared<CreateVisitorReply>(*cmd2); + auto reply2 = copyReply(reply); +} + +TEST_P(StorageProtocolTest, get_bucket_diff) { std::vector<api::MergeBucketCommand::Node> nodes; nodes.push_back(4); nodes.push_back(13); @@ -727,56 +555,68 @@ StorageProtocolTest::testGetBucketDiff51() entries.back()._flags = 1; entries.back()._hasMask = 3; - CPPUNIT_ASSERT_EQUAL(std::string( - "Entry(timestamp: 123456, gid(0x313233343536373839306162), " - "hasMask: 0x3,\n" - " header size: 100, body size: 65536, flags 0x1)"), - entries.back().toString(true)); + EXPECT_EQ("Entry(timestamp: 123456, gid(0x313233343536373839306162), hasMask: 0x3,\n" + " header size: 100, body size: 65536, flags 0x1)", + entries.back().toString(true)); - GetBucketDiffCommand::SP cmd(new GetBucketDiffCommand(bucket, nodes, 1056)); + auto cmd = std::make_shared<GetBucketDiffCommand>(_bucket, nodes, 1056); cmd->getDiff() = entries; - GetBucketDiffCommand::SP cmd2(copyCommand(cmd, _version5_1)); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); + + auto reply = std::make_shared<GetBucketDiffReply>(*cmd2); + EXPECT_EQ(entries, reply->getDiff()); + auto reply2 = copyReply(reply); - GetBucketDiffReply::SP reply(new GetBucketDiffReply(*cmd2)); - CPPUNIT_ASSERT_EQUAL(entries, reply->getDiff()); - GetBucketDiffReply::SP reply2(copyReply(reply)); + EXPECT_EQ(nodes, reply2->getNodes()); + EXPECT_EQ(entries, reply2->getDiff()); + EXPECT_EQ(Timestamp(1056), reply2->getMaxTimestamp()); +} + +namespace { + +ApplyBucketDiffCommand::Entry dummy_apply_entry() { + ApplyBucketDiffCommand::Entry e; + e._docName = "my cool id"; + vespalib::string header_data = "fancy header"; + e._headerBlob.resize(header_data.size()); + memcpy(&e._headerBlob[0], header_data.data(), header_data.size()); - CPPUNIT_ASSERT_EQUAL(nodes, reply2->getNodes()); - CPPUNIT_ASSERT_EQUAL(entries, reply2->getDiff()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1056), reply2->getMaxTimestamp()); + vespalib::string body_data = "fancier body!"; + e._bodyBlob.resize(body_data.size()); + memcpy(&e._bodyBlob[0], body_data.data(), body_data.size()); - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); + GetBucketDiffCommand::Entry meta; + meta._timestamp = 567890; + meta._hasMask = 0x3; + meta._flags = 0x1; + meta._headerSize = 12345; + meta._headerSize = header_data.size(); + meta._bodySize = body_data.size(); + + e._entry = meta; + return e; } -void -StorageProtocolTest::testApplyBucketDiff51() -{ - ScopedName test("testApplyBucketDiff51"); - document::BucketId bucketId(16, 623); - document::Bucket bucket(makeDocumentBucket(bucketId)); +} +TEST_P(StorageProtocolTest, apply_bucket_diff) { std::vector<api::MergeBucketCommand::Node> nodes; nodes.push_back(4); nodes.push_back(13); - std::vector<ApplyBucketDiffCommand::Entry> entries; - entries.push_back(ApplyBucketDiffCommand::Entry()); + std::vector<ApplyBucketDiffCommand::Entry> entries = {dummy_apply_entry()}; - ApplyBucketDiffCommand::SP cmd(new ApplyBucketDiffCommand(bucket, nodes, 1234)); + auto cmd = std::make_shared<ApplyBucketDiffCommand>(_bucket, nodes, 1234); cmd->getDiff() = entries; - ApplyBucketDiffCommand::SP cmd2(copyCommand(cmd, _version5_1)); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); - ApplyBucketDiffReply::SP reply(new ApplyBucketDiffReply(*cmd2)); - ApplyBucketDiffReply::SP reply2(copyReply(reply)); + auto reply = std::make_shared<ApplyBucketDiffReply>(*cmd2); + auto reply2 = copyReply(reply); - CPPUNIT_ASSERT_EQUAL(nodes, reply2->getNodes()); - CPPUNIT_ASSERT_EQUAL(entries, reply2->getDiff()); - CPPUNIT_ASSERT_EQUAL(1234u, reply2->getMaxBufferSize()); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); + EXPECT_EQ(nodes, reply2->getNodes()); + EXPECT_EQ(entries, reply2->getDiff()); + EXPECT_EQ(1234u, reply2->getMaxBufferSize()); } namespace { @@ -807,161 +647,97 @@ namespace { }; api::StorageReply::UP MyCommand::makeReply() { - return api::StorageReply::UP(new MyReply(*this)); + return std::make_unique<MyReply>(*this); } } -void -StorageProtocolTest::testInternalMessage() -{ - ScopedName test("testInternal51"); +TEST_P(StorageProtocolTest, internal_message) { MyCommand cmd; MyReply reply(cmd); - - recordOutput(cmd); - recordOutput(reply); + // TODO what's this even intended to test? } -void -StorageProtocolTest::testSetBucketState51() -{ - ScopedName test("testSetBucketState51"); - document::BucketId bucketId(16, 0); - document::Bucket bucket(makeDocumentBucket(bucketId)); - SetBucketStateCommand::SP cmd( - new SetBucketStateCommand(bucket, SetBucketStateCommand::ACTIVE)); - SetBucketStateCommand::SP cmd2(copyCommand(cmd, _version5_1)); +TEST_P(StorageProtocolTest, set_bucket_state_with_inactive_state) { + auto cmd = std::make_shared<SetBucketStateCommand>(_bucket, SetBucketStateCommand::INACTIVE); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); - SetBucketStateReply::SP reply(new SetBucketStateReply(*cmd2)); - SetBucketStateReply::SP reply2(copyReply(reply)); + auto reply = std::make_shared<SetBucketStateReply>(*cmd2); + auto reply2 = copyReply(reply); - CPPUNIT_ASSERT_EQUAL(SetBucketStateCommand::ACTIVE, cmd2->getState()); - CPPUNIT_ASSERT_EQUAL(bucketId, cmd2->getBucketId()); - CPPUNIT_ASSERT_EQUAL(bucketId, reply2->getBucketId()); - - recordOutput(*cmd2); - recordOutput(*reply2); + EXPECT_EQ(SetBucketStateCommand::INACTIVE, cmd2->getState()); + EXPECT_EQ(_bucket, reply2->getBucket()); } -void -StorageProtocolTest::testPutCommand52() -{ - ScopedName test("testPutCommand52"); +TEST_P(StorageProtocolTest, set_bucket_state_with_active_state) { + auto cmd = std::make_shared<SetBucketStateCommand>(_bucket, SetBucketStateCommand::ACTIVE); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(SetBucketStateCommand::ACTIVE, cmd2->getState()); +} - PutCommand::SP cmd(new PutCommand(_bucket, _testDoc, 14)); +TEST_P(StorageProtocolTest, put_command_with_condition) { + auto cmd = std::make_shared<PutCommand>(_bucket, _testDoc, 14); cmd->setCondition(TestAndSetCondition(CONDITION_STRING)); - PutCommand::SP cmd2(copyCommand(cmd, _version5_2)); - CPPUNIT_ASSERT_EQUAL(cmd->getCondition().getSelection(), cmd2->getCondition().getSelection()); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(cmd->getCondition().getSelection(), cmd2->getCondition().getSelection()); } -void -StorageProtocolTest::testUpdateCommand52() -{ - ScopedName test("testUpdateCommand52"); - - document::DocumentUpdate::SP update(new document::DocumentUpdate(_docMan.getTypeRepo(), *_testDoc->getDataType(), _testDoc->getId())); - UpdateCommand::SP cmd(new UpdateCommand(_bucket, update, 14)); +TEST_P(StorageProtocolTest, update_command_with_condition) { + auto update = std::make_shared<document::DocumentUpdate>( + _docMan.getTypeRepo(), *_testDoc->getDataType(), _testDoc->getId()); + auto cmd = std::make_shared<UpdateCommand>(_bucket, update, 14); cmd->setCondition(TestAndSetCondition(CONDITION_STRING)); - UpdateCommand::SP cmd2(copyCommand(cmd, _version5_2)); - CPPUNIT_ASSERT_EQUAL(cmd->getCondition().getSelection(), cmd2->getCondition().getSelection()); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(cmd->getCondition().getSelection(), cmd2->getCondition().getSelection()); } -void -StorageProtocolTest::testRemoveCommand52() -{ - ScopedName test("testRemoveCommand52"); - - RemoveCommand::SP cmd(new RemoveCommand(_bucket, _testDocId, 159)); +TEST_P(StorageProtocolTest, remove_command_with_condition) { + auto cmd = std::make_shared<RemoveCommand>(_bucket, _testDocId, 159); cmd->setCondition(TestAndSetCondition(CONDITION_STRING)); - RemoveCommand::SP cmd2(copyCommand(cmd, _version5_2)); - CPPUNIT_ASSERT_EQUAL(cmd->getCondition().getSelection(), cmd2->getCondition().getSelection()); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(cmd->getCondition().getSelection(), cmd2->getCondition().getSelection()); } -void -StorageProtocolTest::testPutCommandWithBucketSpace6_0() -{ - ScopedName test("testPutCommandWithBucketSpace6_0"); - - document::Bucket bucket(document::BucketSpace(5), _bucket.getBucketId()); +TEST_P(StorageProtocolTest, put_command_with_bucket_space) { + document::Bucket bucket(document::BucketSpace(5), _bucket_id); auto cmd = std::make_shared<PutCommand>(bucket, _testDoc, 14); - auto cmd2 = copyCommand(cmd, _version6_0); - CPPUNIT_ASSERT_EQUAL(bucket, cmd2->getBucket()); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(bucket, cmd2->getBucket()); } -void -StorageProtocolTest::testCreateVisitorWithBucketSpace6_0() -{ - ScopedName test("testCreateVisitorWithBucketSpace6_0"); - +TEST_P(StorageProtocolTest, create_visitor_with_bucket_space) { document::BucketSpace bucketSpace(5); auto cmd = std::make_shared<CreateVisitorCommand>(bucketSpace, "library", "id", "doc selection"); - auto cmd2 = copyCommand(cmd, _version6_0); - CPPUNIT_ASSERT_EQUAL(bucketSpace, cmd2->getBucketSpace()); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(bucketSpace, cmd2->getBucketSpace()); } -void -StorageProtocolTest::testRequestBucketInfoWithBucketSpace6_0() -{ - ScopedName test("testRequestBucketInfoWithBucketSpace6_0"); - +TEST_P(StorageProtocolTest, request_bucket_info_with_bucket_space) { document::BucketSpace bucketSpace(5); std::vector<document::BucketId> ids = {document::BucketId(3)}; auto cmd = std::make_shared<RequestBucketInfoCommand>(bucketSpace, ids); - auto cmd2 = copyCommand(cmd, _version6_0); - CPPUNIT_ASSERT_EQUAL(bucketSpace, cmd2->getBucketSpace()); - CPPUNIT_ASSERT_EQUAL(ids, cmd2->getBuckets()); -} - -void -StorageProtocolTest::serialized_size_is_used_to_set_approx_size_of_storage_message() -{ - ScopedName test("serialized_size_is_used_to_set_approx_size_of_storage_message"); - - PutCommand::SP cmd(new PutCommand(_bucket, _testDoc, 14)); - CPPUNIT_ASSERT_EQUAL(50u, cmd->getApproxByteSize()); - - PutCommand::SP cmd2(copyCommand(cmd, _version6_0)); - CPPUNIT_ASSERT_EQUAL(181u, cmd2->getApproxByteSize()); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(bucketSpace, cmd2->getBucketSpace()); + EXPECT_EQ(ids, cmd2->getBuckets()); } -void -StorageProtocolTest::testStringOutputs() -{ - std::cerr << "\nNon verbose output:\n"; - for (uint32_t i=0, n=_nonVerboseMessageStrings.size(); i<n; ++i) { - std::cerr << _nonVerboseMessageStrings[i] << "\n"; - } - std::cerr << "\nVerbose output:\n"; - for (uint32_t i=0, n=_verboseMessageStrings.size(); i<n; ++i) { - std::cerr << _verboseMessageStrings[i] << "\n"; - } -} +TEST_P(StorageProtocolTest, serialized_size_is_used_to_set_approx_size_of_storage_message) { + auto cmd = std::make_shared<PutCommand>(_bucket, _testDoc, 14); + EXPECT_EQ(50u, cmd->getApproxByteSize()); -void -StorageProtocolTest::testWriteSerialization50() -{ - std::ofstream of("mbusprot/mbusprot.5.0.serialization.5.1"); - of << std::hex << std::setfill('0'); - for (uint32_t i=0, n=_serialization50.size(); i<n; ++i) { - char c = _serialization50[i]; - if (c > 126 || (c < 32 && c != 10)) { - int32_t num = static_cast<int32_t>(c); - if (num < 0) num += 256; - of << '\\' << std::setw(2) << num; - } else if (c == '\\') { - of << "\\\\"; - } else { - of << c; - } + auto cmd2 = copyCommand(cmd); + auto version = GetParam(); + if (version.getMajor() == 7) { // Protobuf-based encoding + EXPECT_EQ(158u, cmd2->getApproxByteSize()); + } else { // Legacy encoding + EXPECT_EQ(181u, cmd2->getApproxByteSize()); } - of.close(); } -} // mbusprot -} // storage +} // storage::api diff --git a/storageapi/src/vespa/storageapi/CMakeLists.txt b/storageapi/src/vespa/storageapi/CMakeLists.txt index c08dcbc2419..90eb6dd9eca 100644 --- a/storageapi/src/vespa/storageapi/CMakeLists.txt +++ b/storageapi/src/vespa/storageapi/CMakeLists.txt @@ -1,4 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + vespa_add_library(storageapi SOURCES $<TARGET_OBJECTS:storageapi_message> @@ -8,3 +9,6 @@ vespa_add_library(storageapi INSTALL lib64 DEPENDS ) + +vespa_add_target_package_dependency(storageapi Protobuf) + diff --git a/storageapi/src/vespa/storageapi/mbusprot/.gitignore b/storageapi/src/vespa/storageapi/mbusprot/.gitignore index 526f91c6668..8e91fe9cab0 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/.gitignore +++ b/storageapi/src/vespa/storageapi/mbusprot/.gitignore @@ -5,3 +5,6 @@ .deps .libs Makefile +*.pb.h +*.pb.cc + diff --git a/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt b/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt index d5952d7cb91..dc4e3897e49 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt +++ b/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt @@ -1,4 +1,19 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +find_package(Protobuf REQUIRED) +PROTOBUF_GENERATE_CPP(storageapi_PROTOBUF_SRCS storageapi_PROTOBUF_HDRS + protobuf/common.proto + protobuf/feed.proto + protobuf/visiting.proto + protobuf/maintenance.proto) + +# protoc-generated files emit compiler warnings that we normally treat as errors. +# Instead of rolling our own compiler plugin we'll pragmatically disable the noise. +set_source_files_properties(${storageapi_PROTOBUF_SRCS} PROPERTIES COMPILE_FLAGS "-Wno-suggest-override -Wno-inline") +# protoc explicitly annotates methods with inline, which triggers -Werror=inline when +# the header file grows over a certain size. +set_source_files_properties(protocolserialization7.cpp PROPERTIES COMPILE_FLAGS "-Wno-inline") + vespa_add_library(storageapi_mbusprot OBJECT SOURCES storagemessage.cpp @@ -11,5 +26,7 @@ vespa_add_library(storageapi_mbusprot OBJECT protocolserialization5_1.cpp protocolserialization5_2.cpp protocolserialization6_0.cpp + protocolserialization7.cpp + ${storageapi_PROTOBUF_SRCS} DEPENDS ) diff --git a/storageapi/src/vespa/storageapi/mbusprot/legacyprotocolserialization.h b/storageapi/src/vespa/storageapi/mbusprot/legacyprotocolserialization.h new file mode 100644 index 00000000000..ef4a6b28749 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/legacyprotocolserialization.h @@ -0,0 +1,31 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "protocolserialization.h" + +namespace storage::mbusprot { + +/* + * Utility base class for pre-v7 (protobuf) serialization implementations. + * + * TODO remove on Vespa 8 alongside legacy serialization formats. + */ +class LegacyProtocolSerialization : public ProtocolSerialization { + const std::shared_ptr<const document::DocumentTypeRepo> _repo; +public: + explicit LegacyProtocolSerialization(const std::shared_ptr<const document::DocumentTypeRepo>& repo) + : _repo(repo) + {} + + const document::DocumentTypeRepo& getTypeRepo() const { return *_repo; } + const std::shared_ptr<const document::DocumentTypeRepo> getTypeRepoSp() const { return _repo; } + + virtual document::Bucket getBucket(document::ByteBuffer& buf) const = 0; + virtual void putBucket(const document::Bucket& bucket, vespalib::GrowableByteBuffer& buf) const = 0; + virtual document::BucketSpace getBucketSpace(document::ByteBuffer& buf) const = 0; + virtual void putBucketSpace(document::BucketSpace bucketSpace, vespalib::GrowableByteBuffer& buf) const = 0; + virtual api::BucketInfo getBucketInfo(document::ByteBuffer& buf) const = 0; + virtual void putBucketInfo(const api::BucketInfo& info, vespalib::GrowableByteBuffer& buf) const = 0; +}; + +} // storage::mbusprot diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/common.proto b/storageapi/src/vespa/storageapi/mbusprot/protobuf/common.proto new file mode 100644 index 00000000000..d641449995d --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protobuf/common.proto @@ -0,0 +1,68 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +syntax = "proto3"; + +option cc_enable_arenas = true; + +package storage.mbusprot.protobuf; + +// Note: we use a *Request/*Response naming convention rather than *Command/*Reply, +// as the former is the gRPC convention and that's where we intend to move. + +message BucketSpace { + uint64 space_id = 1; +} + +message BucketId { + fixed64 raw_id = 1; +} + +message Bucket { + uint64 space_id = 1; + fixed64 raw_bucket_id = 2; +} + +// Next tag to use: 3 +message BucketInfo { + uint64 last_modified_timestamp = 1; + fixed32 legacy_checksum = 2; + // TODO v2 checksum + uint32 doc_count = 3; + uint32 total_doc_size = 4; + uint32 meta_count = 5; + uint32 used_file_size = 6; + bool ready = 7; + bool active = 8; +} + +message GlobalId { + // 96 bits of GID data in _little_ endian. High entropy, so fixed encoding is better than varint. + // Low 64 bits as if memcpy()ed from bytes [0, 8) of the GID buffer + fixed64 lo_64 = 1; + // High 32 bits as if memcpy()ed from bytes [8, 12) of the GID buffer + fixed32 hi_32 = 2; +} + +// TODO these should ideally be gRPC headers.. +message RequestHeader { + uint64 message_id = 1; + uint32 priority = 2; // Always in range [0, 255] + uint32 source_index = 3; // Always in range [0, 65535] + fixed32 loadtype_id = 4; // It's a hash with high entropy, so fixed encoding is better than varint +} + +// TODO these should ideally be gRPC headers.. +message ResponseHeader { + // TODO this should ideally be gRPC Status... + uint32 return_code_id = 1; + bytes return_code_message = 2; // FIXME it's `bytes` since `string` will check for UTF-8... might not hold... + uint64 message_id = 3; + uint32 priority = 4; // Always in range [0, 255] +} + +message Document { + bytes payload = 1; +} + +message DocumentId { + bytes id = 1; +} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto b/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto new file mode 100644 index 00000000000..58da24df836 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto @@ -0,0 +1,91 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +syntax = "proto3"; + +option cc_enable_arenas = true; + +package storage.mbusprot.protobuf; + +import "common.proto"; + +message TestAndSetCondition { + bytes selection = 1; +} + +message PutRequest { + Bucket bucket = 1; + Document document = 2; + uint64 new_timestamp = 3; + uint64 expected_old_timestamp = 4; // If zero; no expectation. + TestAndSetCondition condition = 5; +} + +message PutResponse { + BucketInfo bucket_info = 1; + BucketId remapped_bucket_id = 2; + bool was_found = 3; +} + +message Update { + bytes payload = 1; +} + +message UpdateRequest { + Bucket bucket = 1; + Update update = 2; + uint64 new_timestamp = 3; + uint64 expected_old_timestamp = 4; // If zero; no expectation. + TestAndSetCondition condition = 5; +} + +message UpdateResponse { + BucketInfo bucket_info = 1; + BucketId remapped_bucket_id = 2; + uint64 updated_timestamp = 3; +} + +message RemoveRequest { + Bucket bucket = 1; + bytes document_id = 2; + uint64 new_timestamp = 3; + TestAndSetCondition condition = 4; +} + +message RemoveResponse { + BucketInfo bucket_info = 1; + BucketId remapped_bucket_id = 2; + uint64 removed_timestamp = 3; +} + +message GetRequest { + Bucket bucket = 1; + bytes document_id = 2; + bytes field_set = 3; + uint64 before_timestamp = 4; +} + +message GetResponse { + Document document = 1; + uint64 last_modified_timestamp = 2; + BucketInfo bucket_info = 3; + BucketId remapped_bucket_id = 4; +} + +message RevertRequest { + Bucket bucket = 1; + repeated uint64 revert_tokens = 2; +} + +message RevertResponse { + BucketInfo bucket_info = 1; + BucketId remapped_bucket_id = 2; +} + +message RemoveLocationRequest { + Bucket bucket = 1; + bytes document_selection = 2; +} + +message RemoveLocationResponse { + BucketInfo bucket_info = 1; + BucketId remapped_bucket_id = 2; +} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/maintenance.proto b/storageapi/src/vespa/storageapi/mbusprot/protobuf/maintenance.proto new file mode 100644 index 00000000000..c4766d2900a --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protobuf/maintenance.proto @@ -0,0 +1,160 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +syntax = "proto3"; + +option cc_enable_arenas = true; + +package storage.mbusprot.protobuf; + +import "common.proto"; + +message DeleteBucketRequest { + Bucket bucket = 1; + BucketInfo expected_bucket_info = 2; +} + +message DeleteBucketResponse { + BucketInfo bucket_info = 1; + BucketId remapped_bucket_id = 2; +} + +message CreateBucketRequest { + Bucket bucket = 1; + bool create_as_active = 2; +} + +message CreateBucketResponse { + BucketInfo bucket_info = 1; + BucketId remapped_bucket_id = 2; +} + +message MergeNode { + uint32 index = 1; + bool source_only = 2; +} + +message MergeBucketRequest { + Bucket bucket = 1; + uint32 cluster_state_version = 2; + uint64 max_timestamp = 3; + repeated MergeNode nodes = 4; + repeated uint32 node_chain = 5; +} + +message MergeBucketResponse { + BucketId remapped_bucket_id = 1; +} + +message MetaDiffEntry { + uint64 timestamp = 1; + GlobalId gid = 2; + uint32 header_size = 3; + uint32 body_size = 4; + uint32 flags = 5; + uint32 presence_mask = 6; +} + +message GetBucketDiffRequest { + Bucket bucket = 1; + uint64 max_timestamp = 2; + repeated MergeNode nodes = 3; + repeated MetaDiffEntry diff = 4; +} + +message GetBucketDiffResponse { + BucketId remapped_bucket_id = 1; + repeated MetaDiffEntry diff = 2; +} + +message ApplyDiffEntry { + MetaDiffEntry entry_meta = 1; + bytes document_id = 2; + bytes header_blob = 3; + bytes body_blob = 4; +} + +message ApplyBucketDiffRequest { + Bucket bucket = 1; + repeated MergeNode nodes = 2; + uint32 max_buffer_size = 3; + repeated ApplyDiffEntry entries = 4; +} + +message ApplyBucketDiffResponse { + BucketId remapped_bucket_id = 1; + repeated ApplyDiffEntry entries = 4; +} + +message ExplicitBucketSet { + // `Bucket` is not needed, as the space is inferred from the owning message. + repeated BucketId bucket_ids = 2; +} + +message AllBuckets { + uint32 distributor_index = 1; + bytes cluster_state = 2; + bytes distribution_hash = 3; +} + +message RequestBucketInfoRequest { + BucketSpace bucket_space = 1; + oneof request_for { + ExplicitBucketSet explicit_bucket_set = 2; + AllBuckets all_buckets = 3; + } +} + +message BucketAndBucketInfo { + fixed64 raw_bucket_id = 1; + BucketInfo bucket_info = 2; +} + +message RequestBucketInfoResponse { + repeated BucketAndBucketInfo bucket_infos = 1; +} + +message NotifyBucketChangeRequest { + Bucket bucket = 1; + BucketInfo bucket_info = 2; +} + +message NotifyBucketChangeResponse { + // Currently empty +} + +message SplitBucketRequest { + Bucket bucket = 1; + uint32 min_split_bits = 2; + uint32 max_split_bits = 3; + uint32 min_byte_size = 4; + uint32 min_doc_count = 5; +} + +message SplitBucketResponse { + BucketId remapped_bucket_id = 1; + repeated BucketAndBucketInfo split_info = 2; +} + +message JoinBucketsRequest { + Bucket bucket = 1; + repeated BucketId source_buckets = 2; + uint32 min_join_bits = 3; +} + +message JoinBucketsResponse { + BucketInfo bucket_info = 1; + BucketId remapped_bucket_id = 2; +} + +message SetBucketStateRequest { + enum BucketState { + Inactive = 0; + Active = 1; + } + + Bucket bucket = 1; + BucketState state = 2; +} + +message SetBucketStateResponse { + BucketId remapped_bucket_id = 1; +} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/visiting.proto b/storageapi/src/vespa/storageapi/mbusprot/protobuf/visiting.proto new file mode 100644 index 00000000000..89ce39e52a0 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protobuf/visiting.proto @@ -0,0 +1,66 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +syntax = "proto3"; + +option cc_enable_arenas = true; + +package storage.mbusprot.protobuf; + +import "common.proto"; + +message ClientVisitorParameter { + bytes key = 1; + bytes value = 2; +} + +message VisitorConstraints { + bytes document_selection = 1; + uint64 from_time_usec = 2; + uint64 to_time_usec = 3; + bool visit_removes = 4; + bytes field_set = 5; + bool visit_inconsistent_buckets = 6; +} + +message VisitorControlMeta { + bytes instance_id = 1; + bytes library_name = 2; + uint32 visitor_command_id = 3; + bytes control_destination = 4; + bytes data_destination = 5; + + // TODO move? + uint32 max_pending_reply_count = 6; + uint32 queue_timeout = 7; + uint32 max_buckets_per_visitor = 8; +} + +message CreateVisitorRequest { + BucketSpace bucket_space = 1; + repeated BucketId buckets = 2; + + VisitorConstraints constraints = 3; + VisitorControlMeta control_meta = 4; + repeated ClientVisitorParameter client_parameters = 5; +} + +message VisitorStatistics { + uint32 buckets_visited = 1; + uint64 documents_visited = 2; + uint64 bytes_visited = 3; + uint64 documents_returned = 4; + uint64 bytes_returned = 5; + uint64 second_pass_documents_returned = 6; // TODO don't include? orderdoc only + uint64 second_pass_bytes_returned = 7; // TODO don't include? orderdoc only +} + +message CreateVisitorResponse { + VisitorStatistics visitor_statistics = 1; +} + +message DestroyVisitorRequest { + bytes instance_id = 1; +} + +message DestroyVisitorResponse { + // Currently empty +} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf_includes.h b/storageapi/src/vespa/storageapi/mbusprot/protobuf_includes.h new file mode 100644 index 00000000000..8e878cf0560 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protobuf_includes.h @@ -0,0 +1,13 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +// Disable warnings emitted by protoc generated files +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wsuggest-override" + +#include "feed.pb.h" +#include "visiting.pb.h" +#include "maintenance.pb.h" + +#pragma GCC diagnostic pop diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp index 172cd6c8de5..917b60c50c3 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp @@ -17,11 +17,6 @@ LOG_SETUP(".storage.api.mbusprot.serialization.base"); namespace storage::mbusprot { -ProtocolSerialization::ProtocolSerialization(const std::shared_ptr<const document::DocumentTypeRepo>& repo) - : _repo(repo) -{ -} - mbus::Blob ProtocolSerialization::encode(const api::StorageMessage& msg) const { diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h index 9c3ddb88bdf..a57627b9ba9 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h @@ -59,21 +59,14 @@ class StorageCommand; class StorageReply; class ProtocolSerialization { - const std::shared_ptr<const document::DocumentTypeRepo> _repo; - public: virtual mbus::Blob encode(const api::StorageMessage&) const; virtual std::unique_ptr<StorageCommand> decodeCommand(mbus::BlobRef) const; virtual std::unique_ptr<StorageReply> decodeReply( mbus::BlobRef, const api::StorageCommand&) const; - protected: - const document::DocumentTypeRepo& getTypeRepo() const { return *_repo; } - const std::shared_ptr<const document::DocumentTypeRepo> getTypeRepoSp() const - { return _repo; } - - ProtocolSerialization(const std::shared_ptr<const document::DocumentTypeRepo> &repo); - virtual ~ProtocolSerialization() {} + ProtocolSerialization() = default; + virtual ~ProtocolSerialization() = default; typedef api::StorageCommand SCmd; typedef api::StorageReply SRep; @@ -102,13 +95,10 @@ protected: virtual void onEncode(GBBuf&, const api::GetBucketDiffReply&) const = 0; virtual void onEncode(GBBuf&, const api::ApplyBucketDiffCommand&) const = 0; virtual void onEncode(GBBuf&, const api::ApplyBucketDiffReply&) const = 0; - virtual void onEncode(GBBuf&, - const api::RequestBucketInfoCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::RequestBucketInfoCommand&) const = 0; virtual void onEncode(GBBuf&, const api::RequestBucketInfoReply&) const = 0; - virtual void onEncode(GBBuf&, - const api::NotifyBucketChangeCommand&) const = 0; - virtual void onEncode(GBBuf&, - const api::NotifyBucketChangeReply&) const = 0; + virtual void onEncode(GBBuf&, const api::NotifyBucketChangeCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::NotifyBucketChangeReply&) const = 0; virtual void onEncode(GBBuf&, const api::SplitBucketCommand&) const = 0; virtual void onEncode(GBBuf&, const api::SplitBucketReply&) const = 0; virtual void onEncode(GBBuf&, const api::JoinBucketsCommand&) const = 0; @@ -143,11 +133,9 @@ protected: virtual SCmd::UP onDecodeApplyBucketDiffCommand(BBuf&) const = 0; virtual SRep::UP onDecodeApplyBucketDiffReply(const SCmd&, BBuf&) const = 0; virtual SCmd::UP onDecodeRequestBucketInfoCommand(BBuf&) const = 0; - virtual SRep::UP onDecodeRequestBucketInfoReply(const SCmd&, - BBuf&) const = 0; + virtual SRep::UP onDecodeRequestBucketInfoReply(const SCmd&, BBuf&) const = 0; virtual SCmd::UP onDecodeNotifyBucketChangeCommand(BBuf&) const = 0; - virtual SRep::UP onDecodeNotifyBucketChangeReply(const SCmd&, - BBuf&) const = 0; + virtual SRep::UP onDecodeNotifyBucketChangeReply(const SCmd&, BBuf&) const = 0; virtual SCmd::UP onDecodeSplitBucketCommand(BBuf&) const = 0; virtual SRep::UP onDecodeSplitBucketReply(const SCmd&, BBuf&) const = 0; virtual SCmd::UP onDecodeJoinBucketsCommand(BBuf&) const = 0; @@ -160,14 +148,6 @@ protected: virtual SRep::UP onDecodeDestroyVisitorReply(const SCmd&, BBuf&) const = 0; virtual SCmd::UP onDecodeRemoveLocationCommand(BBuf&) const = 0; virtual SRep::UP onDecodeRemoveLocationReply(const SCmd&, BBuf&) const = 0; - - virtual document::Bucket getBucket(document::ByteBuffer& buf) const = 0; - virtual void putBucket(const document::Bucket& bucket, vespalib::GrowableByteBuffer& buf) const = 0; - virtual document::BucketSpace getBucketSpace(document::ByteBuffer& buf) const = 0; - virtual void putBucketSpace(document::BucketSpace bucketSpace, vespalib::GrowableByteBuffer& buf) const = 0; - virtual api::BucketInfo getBucketInfo(document::ByteBuffer& buf) const = 0; - virtual void putBucketInfo(const api::BucketInfo& info, vespalib::GrowableByteBuffer& buf) const = 0; - }; } diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp index 74a0c964d19..466ff85f398 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp @@ -20,7 +20,7 @@ namespace storage::mbusprot { ProtocolSerialization4_2::ProtocolSerialization4_2( const std::shared_ptr<const document::DocumentTypeRepo>& repo) - : ProtocolSerialization(repo) + : LegacyProtocolSerialization(repo) { } diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h index 56aa3d4ed30..e4ab36dc989 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h @@ -1,13 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "protocolserialization.h" +#include "legacyprotocolserialization.h" namespace storage::mbusprot { -class ProtocolSerialization4_2 : public ProtocolSerialization { +class ProtocolSerialization4_2 : public LegacyProtocolSerialization { public: - ProtocolSerialization4_2(const std::shared_ptr<const document::DocumentTypeRepo>&); + explicit ProtocolSerialization4_2(const std::shared_ptr<const document::DocumentTypeRepo>&); protected: void onEncode(GBBuf&, const api::GetCommand&) const override; diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h index 042ec7850ef..67f02aa2d2a 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h @@ -73,6 +73,9 @@ public: SRep::UP onDecodeCreateVisitorReply(const SCmd& cmd, BBuf& buf) const override; void onDecodeCommand(BBuf& buf, api::StorageCommand& msg) const override; void onDecodeReply(BBuf&, api::StorageReply&) const override; + +protected: + const documentapi::LoadTypeSet& loadTypes() const noexcept { return _loadTypes; }; }; } diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp new file mode 100644 index 00000000000..d0446f52893 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp @@ -0,0 +1,1274 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "protocolserialization7.h" +#include "serializationhelper.h" +#include "protobuf_includes.h" + +#include <vespa/document/update/documentupdate.h> +#include <vespa/document/util/bufferexceptions.h> +#include <vespa/storageapi/message/bucketsplitting.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/removelocation.h> +#include <vespa/storageapi/message/visitor.h> + +namespace storage::mbusprot { + +ProtocolSerialization7::ProtocolSerialization7(std::shared_ptr<const document::DocumentTypeRepo> repo, + const documentapi::LoadTypeSet& load_types) + : ProtocolSerialization(), + _repo(std::move(repo)), + _load_types(load_types) +{ +} + +namespace { + +void set_bucket(protobuf::Bucket& dest, const document::Bucket& src) { + dest.set_raw_bucket_id(src.getBucketId().getRawId()); + dest.set_space_id(src.getBucketSpace().getId()); +} + +void set_bucket_id(protobuf::BucketId& dest, const document::BucketId& src) { + dest.set_raw_id(src.getRawId()); +} + +document::BucketId get_bucket_id(const protobuf::BucketId& src) { + return document::BucketId(src.raw_id()); +} + +void set_bucket_space(protobuf::BucketSpace& dest, const document::BucketSpace& src) { + dest.set_space_id(src.getId()); +} + +document::BucketSpace get_bucket_space(const protobuf::BucketSpace& src) { + return document::BucketSpace(src.space_id()); +} + +void set_bucket_info(protobuf::BucketInfo& dest, const api::BucketInfo& src) { + dest.set_last_modified_timestamp(src.getLastModified()); + dest.set_legacy_checksum(src.getChecksum()); + dest.set_doc_count(src.getDocumentCount()); + dest.set_total_doc_size(src.getTotalDocumentSize()); + dest.set_meta_count(src.getMetaCount()); + dest.set_used_file_size(src.getUsedFileSize()); + dest.set_active(src.isActive()); + dest.set_ready(src.isReady()); +} + +document::Bucket get_bucket(const protobuf::Bucket& src) { + return document::Bucket(document::BucketSpace(src.space_id()), + document::BucketId(src.raw_bucket_id())); +} + +api::BucketInfo get_bucket_info(const protobuf::BucketInfo& src) { + api::BucketInfo info; + info.setLastModified(src.last_modified_timestamp()); + info.setChecksum(src.legacy_checksum()); + info.setDocumentCount(src.doc_count()); + info.setTotalDocumentSize(src.total_doc_size()); + info.setMetaCount(src.meta_count()); + info.setUsedFileSize(src.used_file_size()); + info.setActive(src.active()); + info.setReady(src.ready()); + return info; +} + +documentapi::TestAndSetCondition get_tas_condition(const protobuf::TestAndSetCondition& src) { + return documentapi::TestAndSetCondition(src.selection()); +} + +void set_tas_condition(protobuf::TestAndSetCondition& dest, const documentapi::TestAndSetCondition& src) { + dest.set_selection(src.getSelection().data(), src.getSelection().size()); +} + +std::shared_ptr<document::Document> get_document(const protobuf::Document& src_doc, + const document::DocumentTypeRepo& type_repo) +{ + if (!src_doc.payload().empty()) { + document::ByteBuffer doc_buf(src_doc.payload().data(), src_doc.payload().size()); + return std::make_shared<document::Document>(type_repo, doc_buf); + } + return std::shared_ptr<document::Document>(); +} + +void set_update(protobuf::Update& dest, const document::DocumentUpdate& src) { + vespalib::nbostream stream; + src.serializeHEAD(stream); + dest.set_payload(stream.peek(), stream.size()); +} + +std::shared_ptr<document::DocumentUpdate> get_update(const protobuf::Update& src, + const document::DocumentTypeRepo& type_repo) +{ + if (!src.payload().empty()) { + return document::DocumentUpdate::createHEAD( + type_repo, vespalib::nbostream(src.payload().data(), src.payload().size())); + } + return std::shared_ptr<document::DocumentUpdate>(); +} + +void write_request_header(vespalib::GrowableByteBuffer& buf, const api::StorageCommand& cmd) { + protobuf::RequestHeader hdr; // Arena alloc not needed since there are no nested messages + hdr.set_message_id(cmd.getMsgId()); + hdr.set_priority(cmd.getPriority()); + hdr.set_source_index(cmd.getSourceIndex()); + hdr.set_loadtype_id(cmd.getLoadType().getId()); + + uint8_t dest[128]; // Only primitive fields, should be plenty large enough. + auto encoded_size = static_cast<uint32_t>(hdr.ByteSizeLong()); + assert(encoded_size <= sizeof(dest)); + [[maybe_unused]] bool ok = hdr.SerializeWithCachedSizesToArray(dest); + assert(ok); + buf.putInt(encoded_size); + buf.putBytes(reinterpret_cast<const char*>(dest), encoded_size); +} + +void write_response_header(vespalib::GrowableByteBuffer& buf, const api::StorageReply& reply) { + protobuf::ResponseHeader hdr; // Arena alloc not needed since there are no nested messages + const auto& result = reply.getResult(); + hdr.set_return_code_id(static_cast<uint32_t>(result.getResult())); + if (!result.getMessage().empty()) { + hdr.set_return_code_message(result.getMessage().data(), result.getMessage().size()); + } + hdr.set_message_id(reply.getMsgId()); + hdr.set_priority(reply.getPriority()); + + const auto header_size = hdr.ByteSizeLong(); + assert(header_size <= UINT32_MAX); + buf.putInt(static_cast<uint32_t>(header_size)); + + auto* dest_buf = reinterpret_cast<uint8_t*>(buf.allocate(header_size)); + [[maybe_unused]] bool ok = hdr.SerializeWithCachedSizesToArray(dest_buf); + assert(ok); +} + +void decode_request_header(document::ByteBuffer& buf, protobuf::RequestHeader& hdr) { + auto hdr_len = static_cast<uint32_t>(SerializationHelper::getInt(buf)); + if (hdr_len > buf.getRemaining()) { + throw document::BufferOutOfBoundsException(buf.getPos(), hdr_len); + } + bool ok = hdr.ParseFromArray(buf.getBufferAtPos(), hdr_len); + if (!ok) { + throw vespalib::IllegalArgumentException("Malformed protobuf request header"); + } + buf.incPos(hdr_len); +} + +void decode_response_header(document::ByteBuffer& buf, protobuf::ResponseHeader& hdr) { + auto hdr_len = static_cast<uint32_t>(SerializationHelper::getInt(buf)); + if (hdr_len > buf.getRemaining()) { + throw document::BufferOutOfBoundsException(buf.getPos(), hdr_len); + } + bool ok = hdr.ParseFromArray(buf.getBufferAtPos(), hdr_len); + if (!ok) { + throw vespalib::IllegalArgumentException("Malformed protobuf response header"); + } + buf.incPos(hdr_len); +} + +} // anonymous namespace + +template <typename ProtobufType> +class BaseEncoder { + vespalib::GrowableByteBuffer& _out_buf; + ::google::protobuf::Arena _arena; + ProtobufType* _proto_obj; +public: + explicit BaseEncoder(vespalib::GrowableByteBuffer& out_buf) + : _out_buf(out_buf), + _arena(), + _proto_obj(::google::protobuf::Arena::Create<ProtobufType>(&_arena)) + { + } + + void encode() { + assert(_proto_obj != nullptr); + const auto sz = _proto_obj->ByteSizeLong(); + assert(sz <= UINT32_MAX); + auto* buf = reinterpret_cast<uint8_t*>(_out_buf.allocate(sz)); + [[maybe_unused]] bool ok = _proto_obj->SerializeWithCachedSizesToArray(buf); + assert(ok); + _proto_obj = nullptr; + } +protected: + vespalib::GrowableByteBuffer& buffer() noexcept { return _out_buf; } + + // Precondition: encode() is not called + ProtobufType& proto_obj() noexcept { return *_proto_obj; } + const ProtobufType& proto_obj() const noexcept { return *_proto_obj; } +}; + +template <typename ProtobufType> +class RequestEncoder : public BaseEncoder<ProtobufType> { +public: + RequestEncoder(vespalib::GrowableByteBuffer& out_buf, const api::StorageCommand& cmd) + : BaseEncoder<ProtobufType>(out_buf) + { + write_request_header(out_buf, cmd); + } + + // Precondition: encode() is not called + ProtobufType& request() noexcept { return this->proto_obj(); } + const ProtobufType& request() const noexcept { return this->proto_obj(); } +}; + +template <typename ProtobufType> +class ResponseEncoder : public BaseEncoder<ProtobufType> { +public: + ResponseEncoder(vespalib::GrowableByteBuffer& out_buf, const api::StorageReply& reply) + : BaseEncoder<ProtobufType>(out_buf) + { + write_response_header(out_buf, reply); + } + + // Precondition: encode() is not called + ProtobufType& response() noexcept { return this->proto_obj(); } + const ProtobufType& response() const noexcept { return this->proto_obj(); } +}; + +template <typename ProtobufType> +class RequestDecoder { + protobuf::RequestHeader _hdr; + ::google::protobuf::Arena _arena; + ProtobufType* _proto_obj; + const documentapi::LoadTypeSet& _load_types; +public: + RequestDecoder(document::ByteBuffer& in_buf, const documentapi::LoadTypeSet& load_types) + : _arena(), + _proto_obj(::google::protobuf::Arena::Create<ProtobufType>(&_arena)), + _load_types(load_types) + { + decode_request_header(in_buf, _hdr); + assert(in_buf.getRemaining() <= INT_MAX); + bool ok = _proto_obj->ParseFromArray(in_buf.getBufferAtPos(), in_buf.getRemaining()); + if (!ok) { + throw vespalib::IllegalArgumentException( + vespalib::make_string("Malformed protobuf request payload for %s", + ProtobufType::descriptor()->full_name().c_str())); + } + } + + void transfer_meta_information_to(api::StorageCommand& dest) { + dest.forceMsgId(_hdr.message_id()); + dest.setPriority(static_cast<uint8_t>(_hdr.priority())); + dest.setSourceIndex(static_cast<uint16_t>(_hdr.source_index())); + dest.setLoadType(_load_types[_hdr.loadtype_id()]); + } + + ProtobufType& request() noexcept { return *_proto_obj; } + const ProtobufType& request() const noexcept { return *_proto_obj; } +}; + +template <typename ProtobufType> +class ResponseDecoder { + protobuf::ResponseHeader _hdr; + ::google::protobuf::Arena _arena; + ProtobufType* _proto_obj; +public: + explicit ResponseDecoder(document::ByteBuffer& in_buf) + : _arena(), + _proto_obj(::google::protobuf::Arena::Create<ProtobufType>(&_arena)) + { + decode_response_header(in_buf, _hdr); + assert(in_buf.getRemaining() <= INT_MAX); + bool ok = _proto_obj->ParseFromArray(in_buf.getBufferAtPos(), in_buf.getRemaining()); + if (!ok) { + throw vespalib::IllegalArgumentException( + vespalib::make_string("Malformed protobuf response payload for %s", + ProtobufType::descriptor()->full_name().c_str())); + } + } + + void transfer_meta_information_to(api::StorageReply& dest) { + dest.forceMsgId(_hdr.message_id()); + dest.setPriority(static_cast<uint8_t>(_hdr.priority())); + dest.setResult(api::ReturnCode(static_cast<api::ReturnCode::Result>(_hdr.return_code_id()), + _hdr.return_code_message())); + } + + ProtobufType& response() noexcept { return *_proto_obj; } + const ProtobufType& response() const noexcept { return *_proto_obj; } +}; + +template <typename ProtobufType, typename Func> +void encode_request(vespalib::GrowableByteBuffer& out_buf, const api::StorageCommand& msg, Func&& f) { + RequestEncoder<ProtobufType> enc(out_buf, msg); + f(enc.request()); + enc.encode(); +} + +template <typename ProtobufType, typename Func> +void encode_response(vespalib::GrowableByteBuffer& out_buf, const api::StorageReply& reply, Func&& f) { + ResponseEncoder<ProtobufType> enc(out_buf, reply); + auto& res = enc.response(); + f(res); + enc.encode(); +} + +template <typename ProtobufType, typename Func> +std::unique_ptr<api::StorageCommand> +ProtocolSerialization7::decode_request(document::ByteBuffer& in_buf, Func&& f) const { + RequestDecoder<ProtobufType> dec(in_buf, _load_types); + const auto& req = dec.request(); + auto cmd = f(req); + dec.transfer_meta_information_to(*cmd); + return cmd; +} + +template <typename ProtobufType, typename Func> +std::unique_ptr<api::StorageReply> +ProtocolSerialization7::decode_response(document::ByteBuffer& in_buf, Func&& f) const { + ResponseDecoder<ProtobufType> dec(in_buf); + const auto& res = dec.response(); + auto reply = f(res); + dec.transfer_meta_information_to(*reply); + return reply; +} + +template <typename ProtobufType, typename Func> +void encode_bucket_request(vespalib::GrowableByteBuffer& out_buf, const api::BucketCommand& msg, Func&& f) { + encode_request<ProtobufType>(out_buf, msg, [&](ProtobufType& req) { + set_bucket(*req.mutable_bucket(), msg.getBucket()); + f(req); + }); +} + +template <typename ProtobufType, typename Func> +std::unique_ptr<api::StorageCommand> +ProtocolSerialization7::decode_bucket_request(document::ByteBuffer& in_buf, Func&& f) const { + return decode_request<ProtobufType>(in_buf, [&](const ProtobufType& req) { + if (!req.has_bucket()) { + throw vespalib::IllegalArgumentException( + vespalib::make_string("Malformed protocol buffer request for %s; no bucket", + ProtobufType::descriptor()->full_name().c_str())); + } + const auto bucket = get_bucket(req.bucket()); + return f(req, bucket); + }); +} + +template <typename ProtobufType, typename Func> +void encode_bucket_response(vespalib::GrowableByteBuffer& out_buf, const api::BucketReply& reply, Func&& f) { + encode_response<ProtobufType>(out_buf, reply, [&](ProtobufType& res) { + if (reply.hasBeenRemapped()) { + set_bucket_id(*res.mutable_remapped_bucket_id(), reply.getBucketId()); + } + f(res); + }); +} + +template <typename ProtobufType, typename Func> +std::unique_ptr<api::StorageReply> +ProtocolSerialization7::decode_bucket_response(document::ByteBuffer& in_buf, Func&& f) const { + return decode_response<ProtobufType>(in_buf, [&](const ProtobufType& res) { + auto reply = f(res); + if (res.has_remapped_bucket_id()) { + reply->remapBucketId(get_bucket_id(res.remapped_bucket_id())); + } + return reply; + }); +} + +template <typename ProtobufType, typename Func> +void encode_bucket_info_response(vespalib::GrowableByteBuffer& out_buf, const api::BucketInfoReply& reply, Func&& f) { + encode_bucket_response<ProtobufType>(out_buf, reply, [&](ProtobufType& res) { + set_bucket_info(*res.mutable_bucket_info(), reply.getBucketInfo()); + f(res); + }); +} + +template <typename ProtobufType, typename Func> +std::unique_ptr<api::StorageReply> +ProtocolSerialization7::decode_bucket_info_response(document::ByteBuffer& in_buf, Func&& f) const { + return decode_bucket_response<ProtobufType>(in_buf, [&](const ProtobufType& res) { + auto reply = f(res); + reply->setBucketInfo(get_bucket_info(res.bucket_info())); // If not present, default of all zeroes is correct + return reply; + }); +} + +// TODO document protobuf ducktyping assumptions + +namespace { +// Inherit from known base class just to avoid having to template this. We don't care about its subtype anyway. +void no_op_encode([[maybe_unused]] ::google::protobuf::Message&) { + // nothing to do here. +} + +void set_document(protobuf::Document& target_doc, const document::Document& src_doc) { + vespalib::nbostream stream; + src_doc.serialize(stream); + target_doc.set_payload(stream.peek(), stream.size()); +} + +} + +// ----------------------------------------------------------------- +// Put +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::PutCommand& msg) const { + encode_bucket_request<protobuf::PutRequest>(buf, msg, [&](auto& req) { + req.set_new_timestamp(msg.getTimestamp()); + req.set_expected_old_timestamp(msg.getUpdateTimestamp()); + if (msg.getCondition().isPresent()) { + set_tas_condition(*req.mutable_condition(), msg.getCondition()); + } + if (msg.getDocument()) { + set_document(*req.mutable_document(), *msg.getDocument()); + } + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::PutReply& msg) const { + encode_bucket_info_response<protobuf::PutResponse>(buf, msg, [&](auto& res) { + res.set_was_found(msg.wasFound()); + }); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodePutCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::PutRequest>(buf, [&](auto& req, auto& bucket) { + auto document = get_document(req.document(), type_repo()); + auto cmd = std::make_unique<api::PutCommand>(bucket, std::move(document), req.new_timestamp()); + cmd->setUpdateTimestamp(req.expected_old_timestamp()); + if (req.has_condition()) { + cmd->setCondition(get_tas_condition(req.condition())); + } + return cmd; + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodePutReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_info_response<protobuf::PutResponse>(buf, [&](auto& res) { + return std::make_unique<api::PutReply>(static_cast<const api::PutCommand&>(cmd), res.was_found()); + }); +} + +// ----------------------------------------------------------------- +// Update +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::UpdateCommand& msg) const { + encode_bucket_request<protobuf::UpdateRequest>(buf, msg, [&](auto& req) { + auto* update = msg.getUpdate().get(); + if (update) { + set_update(*req.mutable_update(), *update); + } + req.set_new_timestamp(msg.getTimestamp()); + req.set_expected_old_timestamp(msg.getOldTimestamp()); + if (msg.getCondition().isPresent()) { + set_tas_condition(*req.mutable_condition(), msg.getCondition()); + } + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::UpdateReply& msg) const { + encode_bucket_info_response<protobuf::UpdateResponse>(buf, msg, [&](auto& res) { + res.set_updated_timestamp(msg.getOldTimestamp()); + }); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeUpdateCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::UpdateRequest>(buf, [&](auto& req, auto& bucket) { + auto update = get_update(req.update(), type_repo()); + auto cmd = std::make_unique<api::UpdateCommand>(bucket, std::move(update), req.new_timestamp()); + cmd->setOldTimestamp(req.expected_old_timestamp()); + if (req.has_condition()) { + cmd->setCondition(get_tas_condition(req.condition())); + } + return cmd; + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeUpdateReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_info_response<protobuf::UpdateResponse>(buf, [&](auto& res) { + return std::make_unique<api::UpdateReply>(static_cast<const api::UpdateCommand&>(cmd), + res.updated_timestamp()); + }); +} + +// ----------------------------------------------------------------- +// Remove +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RemoveCommand& msg) const { + encode_bucket_request<protobuf::RemoveRequest>(buf, msg, [&](auto& req) { + auto doc_id_str = msg.getDocumentId().toString(); + req.set_document_id(doc_id_str.data(), doc_id_str.size()); + req.set_new_timestamp(msg.getTimestamp()); + if (msg.getCondition().isPresent()) { + set_tas_condition(*req.mutable_condition(), msg.getCondition()); + } + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RemoveReply& msg) const { + encode_bucket_info_response<protobuf::RemoveResponse>(buf, msg, [&](auto& res) { + res.set_removed_timestamp(msg.getOldTimestamp()); + }); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeRemoveCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::RemoveRequest>(buf, [&](auto& req, auto& bucket) { + document::DocumentId doc_id(vespalib::stringref(req.document_id().data(), req.document_id().size())); + auto cmd = std::make_unique<api::RemoveCommand>(bucket, doc_id, req.new_timestamp()); + if (req.has_condition()) { + cmd->setCondition(get_tas_condition(req.condition())); + } + return cmd; + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeRemoveReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_info_response<protobuf::RemoveResponse>(buf, [&](auto& res) { + return std::make_unique<api::RemoveReply>(static_cast<const api::RemoveCommand&>(cmd), + res.removed_timestamp()); + }); +} + +// ----------------------------------------------------------------- +// Get +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::GetCommand& msg) const { + encode_bucket_request<protobuf::GetRequest>(buf, msg, [&](auto& req) { + auto doc_id = msg.getDocumentId().toString(); + req.set_document_id(doc_id.data(), doc_id.size()); + req.set_before_timestamp(msg.getBeforeTimestamp()); + if (!msg.getFieldSet().empty()) { + req.set_field_set(msg.getFieldSet().data(), msg.getFieldSet().size()); + } + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::GetReply& msg) const { + encode_bucket_info_response<protobuf::GetResponse>(buf, msg, [&](auto& res) { + if (msg.getDocument()) { + set_document(*res.mutable_document(), *msg.getDocument()); + } + res.set_last_modified_timestamp(msg.getLastModifiedTimestamp()); + }); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeGetCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::GetRequest>(buf, [&](auto& req, auto& bucket) { + document::DocumentId doc_id(vespalib::stringref(req.document_id().data(), req.document_id().size())); + return std::make_unique<api::GetCommand>(bucket, std::move(doc_id), + req.field_set(), req.before_timestamp()); + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeGetReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_info_response<protobuf::GetResponse>(buf, [&](auto& res) { + try { + auto document = get_document(res.document(), type_repo()); + return std::make_unique<api::GetReply>(static_cast<const api::GetCommand&>(cmd), + std::move(document), res.last_modified_timestamp()); + } catch (std::exception& e) { + auto reply = std::make_unique<api::GetReply>(static_cast<const api::GetCommand&>(cmd), + std::shared_ptr<document::Document>(), 0u); + reply->setResult(api::ReturnCode(api::ReturnCode::UNPARSEABLE, e.what())); + return reply; + } + }); +} + +// ----------------------------------------------------------------- +// Revert +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RevertCommand& msg) const { + encode_bucket_request<protobuf::RevertRequest>(buf, msg, [&](auto& req) { + auto* tokens = req.mutable_revert_tokens(); + assert(msg.getRevertTokens().size() <= INT_MAX); + tokens->Reserve(static_cast<int>(msg.getRevertTokens().size())); + for (auto token : msg.getRevertTokens()) { + tokens->Add(token); + } + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RevertReply& msg) const { + encode_bucket_info_response<protobuf::RevertResponse>(buf, msg, no_op_encode); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeRevertCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::RevertRequest>(buf, [&](auto& req, auto& bucket) { + std::vector<api::Timestamp> tokens; + tokens.reserve(req.revert_tokens_size()); + for (auto token : req.revert_tokens()) { + tokens.emplace_back(api::Timestamp(token)); + } + return std::make_unique<api::RevertCommand>(bucket, std::move(tokens)); + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeRevertReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_info_response<protobuf::RevertResponse>(buf, [&]([[maybe_unused]] auto& res) { + return std::make_unique<api::RevertReply>(static_cast<const api::RevertCommand&>(cmd)); + }); +} + +// ----------------------------------------------------------------- +// RemoveLocation +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RemoveLocationCommand& msg) const { + encode_bucket_request<protobuf::RemoveLocationRequest>(buf, msg, [&](auto& req) { + req.set_document_selection(msg.getDocumentSelection().data(), msg.getDocumentSelection().size()); + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RemoveLocationReply& msg) const { + encode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, msg, no_op_encode); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeRemoveLocationCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::RemoveLocationRequest>(buf, [&](auto& req, auto& bucket) { + return std::make_unique<api::RemoveLocationCommand>(req.document_selection(), bucket); + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeRemoveLocationReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, [&]([[maybe_unused]] auto& res) { + return std::make_unique<api::RemoveLocationReply>(static_cast<const api::RemoveLocationCommand&>(cmd)); + }); +} + +// ----------------------------------------------------------------- +// DeleteBucket +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::DeleteBucketCommand& msg) const { + encode_bucket_request<protobuf::DeleteBucketRequest>(buf, msg, [&](auto& req) { + set_bucket_info(*req.mutable_expected_bucket_info(), msg.getBucketInfo()); + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::DeleteBucketReply& msg) const { + encode_bucket_info_response<protobuf::DeleteBucketResponse>(buf, msg, no_op_encode); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeDeleteBucketCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::DeleteBucketRequest>(buf, [&](auto& req, auto& bucket) { + auto cmd = std::make_unique<api::DeleteBucketCommand>(bucket); + if (req.has_expected_bucket_info()) { + cmd->setBucketInfo(get_bucket_info(req.expected_bucket_info())); + } + return cmd; + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeDeleteBucketReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_info_response<protobuf::DeleteBucketResponse>(buf, [&]([[maybe_unused]] auto& res) { + return std::make_unique<api::DeleteBucketReply>(static_cast<const api::DeleteBucketCommand&>(cmd)); + }); +} + +// ----------------------------------------------------------------- +// CreateBucket +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::CreateBucketCommand& msg) const { + encode_bucket_request<protobuf::CreateBucketRequest>(buf, msg, [&](auto& req) { + req.set_create_as_active(msg.getActive()); + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::CreateBucketReply& msg) const { + encode_bucket_info_response<protobuf::CreateBucketResponse>(buf, msg, no_op_encode); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeCreateBucketCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::CreateBucketRequest>(buf, [&](auto& req, auto& bucket) { + auto cmd = std::make_unique<api::CreateBucketCommand>(bucket); + cmd->setActive(req.create_as_active()); + return cmd; + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeCreateBucketReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_info_response<protobuf::CreateBucketResponse>(buf, [&]([[maybe_unused]] auto& res) { + return std::make_unique<api::CreateBucketReply>(static_cast<const api::CreateBucketCommand&>(cmd)); + }); +} + +// ----------------------------------------------------------------- +// MergeBucket +// ----------------------------------------------------------------- + +namespace { + +void set_merge_nodes(::google::protobuf::RepeatedPtrField<protobuf::MergeNode>& dest, + const std::vector<api::MergeBucketCommand::Node>& src) +{ + dest.Reserve(src.size()); + for (const auto& src_node : src) { + auto* dest_node = dest.Add(); + dest_node->set_index(src_node.index); + dest_node->set_source_only(src_node.sourceOnly); + } +} + +std::vector<api::MergeBucketCommand::Node> get_merge_nodes( + const ::google::protobuf::RepeatedPtrField<protobuf::MergeNode>& src) +{ + std::vector<api::MergeBucketCommand::Node> nodes; + nodes.reserve(src.size()); + for (const auto& node : src) { + nodes.emplace_back(node.index(), node.source_only()); + } + return nodes; +} + +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::MergeBucketCommand& msg) const { + encode_bucket_request<protobuf::MergeBucketRequest>(buf, msg, [&](auto& req) { + set_merge_nodes(*req.mutable_nodes(), msg.getNodes()); + req.set_max_timestamp(msg.getMaxTimestamp()); + req.set_cluster_state_version(msg.getClusterStateVersion()); + for (uint16_t chain_node : msg.getChain()) { + req.add_node_chain(chain_node); + } + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::MergeBucketReply& msg) const { + encode_bucket_response<protobuf::MergeBucketResponse>(buf, msg, no_op_encode); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeMergeBucketCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::MergeBucketRequest>(buf, [&](auto& req, auto& bucket) { + auto nodes = get_merge_nodes(req.nodes()); + auto cmd = std::make_unique<api::MergeBucketCommand>(bucket, std::move(nodes), req.max_timestamp()); + cmd->setClusterStateVersion(req.cluster_state_version()); + std::vector<uint16_t> chain; + chain.reserve(req.node_chain_size()); + for (uint16_t node : req.node_chain()) { + chain.emplace_back(node); + } + cmd->setChain(std::move(chain)); + return cmd; + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeMergeBucketReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_response<protobuf::MergeBucketResponse>(buf, [&]([[maybe_unused]] auto& res) { + return std::make_unique<api::MergeBucketReply>(static_cast<const api::MergeBucketCommand&>(cmd)); + }); +} + +// ----------------------------------------------------------------- +// GetBucketDiff +// ----------------------------------------------------------------- + +namespace { + +void set_global_id(protobuf::GlobalId& dest, const document::GlobalId& src) { + static_assert(document::GlobalId::LENGTH == 12); + uint64_t lo64; + uint32_t hi32; + memcpy(&lo64, src.get(), sizeof(uint64_t)); + memcpy(&hi32, src.get() + sizeof(uint64_t), sizeof(uint32_t)); + dest.set_lo_64(lo64); + dest.set_hi_32(hi32); +} + +document::GlobalId get_global_id(const protobuf::GlobalId& src) { + static_assert(document::GlobalId::LENGTH == 12); + const uint64_t lo64 = src.lo_64(); + const uint32_t hi32 = src.hi_32(); + + char buf[document::GlobalId::LENGTH]; + memcpy(buf, &lo64, sizeof(uint64_t)); + memcpy(buf + sizeof(uint64_t), &hi32, sizeof(uint32_t)); + return document::GlobalId(buf); +} + +void set_diff_entry(protobuf::MetaDiffEntry& dest, const api::GetBucketDiffCommand::Entry& src) { + dest.set_timestamp(src._timestamp); + set_global_id(*dest.mutable_gid(), src._gid); + dest.set_header_size(src._headerSize); + dest.set_body_size(src._bodySize); + dest.set_flags(src._flags); + dest.set_presence_mask(src._hasMask); +} + +api::GetBucketDiffCommand::Entry get_diff_entry(const protobuf::MetaDiffEntry& src) { + api::GetBucketDiffCommand::Entry e; + e._timestamp = src.timestamp(); + e._gid = get_global_id(src.gid()); + e._headerSize = src.header_size(); + e._bodySize = src.body_size(); + e._flags = src.flags(); + e._hasMask = src.presence_mask(); + return e; +} + +void fill_proto_meta_diff(::google::protobuf::RepeatedPtrField<protobuf::MetaDiffEntry>& dest, + const std::vector<api::GetBucketDiffCommand::Entry>& src) { + for (const auto& diff_entry : src) { + set_diff_entry(*dest.Add(), diff_entry); + } +} + +void fill_api_meta_diff(std::vector<api::GetBucketDiffCommand::Entry>& dest, + const ::google::protobuf::RepeatedPtrField<protobuf::MetaDiffEntry>& src) { + // FIXME GetBucketDiffReply ctor copies the diff from the request for some reason + // TODO verify this isn't actually used anywhere and remove this "feature". + dest.clear(); + dest.reserve(src.size()); + for (const auto& diff_entry : src) { + dest.emplace_back(get_diff_entry(diff_entry)); + } +} + +} // anonymous namespace + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::GetBucketDiffCommand& msg) const { + encode_bucket_request<protobuf::GetBucketDiffRequest>(buf, msg, [&](auto& req) { + set_merge_nodes(*req.mutable_nodes(), msg.getNodes()); + req.set_max_timestamp(msg.getMaxTimestamp()); + fill_proto_meta_diff(*req.mutable_diff(), msg.getDiff()); + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::GetBucketDiffReply& msg) const { + encode_bucket_response<protobuf::GetBucketDiffResponse>(buf, msg, [&](auto& res) { + fill_proto_meta_diff(*res.mutable_diff(), msg.getDiff()); + }); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeGetBucketDiffCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::GetBucketDiffRequest>(buf, [&](auto& req, auto& bucket) { + auto nodes = get_merge_nodes(req.nodes()); + auto cmd = std::make_unique<api::GetBucketDiffCommand>(bucket, std::move(nodes), req.max_timestamp()); + fill_api_meta_diff(cmd->getDiff(), req.diff()); + return cmd; + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeGetBucketDiffReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_response<protobuf::GetBucketDiffResponse>(buf, [&](auto& res) { + auto reply = std::make_unique<api::GetBucketDiffReply>(static_cast<const api::GetBucketDiffCommand&>(cmd)); + fill_api_meta_diff(reply->getDiff(), res.diff()); + return reply; + }); +} + +// ----------------------------------------------------------------- +// ApplyBucketDiff +// ----------------------------------------------------------------- + +namespace { + +void fill_api_apply_diff_vector(std::vector<api::ApplyBucketDiffCommand::Entry>& diff, + const ::google::protobuf::RepeatedPtrField<protobuf::ApplyDiffEntry>& src) +{ + // We use the same approach as the legacy protocols here in that we pre-reserve and + // directly write into the vector. This avoids having to ensure all buffer management is movable. + size_t n_entries = src.size(); + diff.resize(n_entries); + for (size_t i = 0; i < n_entries; ++i) { + auto& proto_entry = src.Get(i); + auto& dest = diff[i]; + dest._entry = get_diff_entry(proto_entry.entry_meta()); + dest._docName = proto_entry.document_id(); + // TODO consider making buffers std::strings instead to avoid explicit zeroing-on-resize overhead + dest._headerBlob.resize(proto_entry.header_blob().size()); + memcpy(dest._headerBlob.data(), proto_entry.header_blob().data(), proto_entry.header_blob().size()); + dest._bodyBlob.resize(proto_entry.body_blob().size()); + memcpy(dest._bodyBlob.data(), proto_entry.body_blob().data(), proto_entry.body_blob().size()); + } +} + +void fill_proto_apply_diff_vector(::google::protobuf::RepeatedPtrField<protobuf::ApplyDiffEntry>& dest, + const std::vector<api::ApplyBucketDiffCommand::Entry>& src) +{ + dest.Reserve(src.size()); + for (const auto& entry : src) { + auto* proto_entry = dest.Add(); + set_diff_entry(*proto_entry->mutable_entry_meta(), entry._entry); + proto_entry->set_document_id(entry._docName.data(), entry._docName.size()); + proto_entry->set_header_blob(entry._headerBlob.data(), entry._headerBlob.size()); + proto_entry->set_body_blob(entry._bodyBlob.data(), entry._bodyBlob.size()); + } +} + +} // anonymous namespace + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::ApplyBucketDiffCommand& msg) const { + encode_bucket_request<protobuf::ApplyBucketDiffRequest>(buf, msg, [&](auto& req) { + set_merge_nodes(*req.mutable_nodes(), msg.getNodes()); + req.set_max_buffer_size(msg.getMaxBufferSize()); + fill_proto_apply_diff_vector(*req.mutable_entries(), msg.getDiff()); + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::ApplyBucketDiffReply& msg) const { + encode_bucket_response<protobuf::ApplyBucketDiffResponse>(buf, msg, [&](auto& res) { + fill_proto_apply_diff_vector(*res.mutable_entries(), msg.getDiff()); + }); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeApplyBucketDiffCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::ApplyBucketDiffRequest>(buf, [&](auto& req, auto& bucket) { + auto nodes = get_merge_nodes(req.nodes()); + auto cmd = std::make_unique<api::ApplyBucketDiffCommand>(bucket, std::move(nodes), req.max_buffer_size()); + fill_api_apply_diff_vector(cmd->getDiff(), req.entries()); + return cmd; + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeApplyBucketDiffReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_response<protobuf::ApplyBucketDiffResponse>(buf, [&](auto& res) { + auto reply = std::make_unique<api::ApplyBucketDiffReply>(static_cast<const api::ApplyBucketDiffCommand&>(cmd)); + fill_api_apply_diff_vector(reply->getDiff(), res.entries()); + return reply; + }); +} + +// ----------------------------------------------------------------- +// RequestBucketInfo +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RequestBucketInfoCommand& msg) const { + encode_request<protobuf::RequestBucketInfoRequest>(buf, msg, [&](auto& req) { + set_bucket_space(*req.mutable_bucket_space(), msg.getBucketSpace()); + auto& buckets = msg.getBuckets(); + if (!buckets.empty()) { + auto* proto_buckets = req.mutable_explicit_bucket_set(); + for (const auto& b : buckets) { + set_bucket_id(*proto_buckets->add_bucket_ids(), b); + } + } else { + auto* all_buckets = req.mutable_all_buckets(); + auto cluster_state = msg.getSystemState().toString(); + all_buckets->set_distributor_index(msg.getDistributor()); + all_buckets->set_cluster_state(cluster_state.data(), cluster_state.size()); + all_buckets->set_distribution_hash(msg.getDistributionHash().data(), msg.getDistributionHash().size()); + } + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RequestBucketInfoReply& msg) const { + encode_response<protobuf::RequestBucketInfoResponse>(buf, msg, [&](auto& res) { + auto* proto_info = res.mutable_bucket_infos(); + proto_info->Reserve(msg.getBucketInfo().size()); + for (const auto& entry : msg.getBucketInfo()) { + auto* bucket_and_info = proto_info->Add(); + bucket_and_info->set_raw_bucket_id(entry._bucketId.getRawId()); + set_bucket_info(*bucket_and_info->mutable_bucket_info(), entry._info); + } + }); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeRequestBucketInfoCommand(BBuf& buf) const { + return decode_request<protobuf::RequestBucketInfoRequest>(buf, [&](auto& req) { + auto bucket_space = get_bucket_space(req.bucket_space()); + if (req.has_explicit_bucket_set()) { + const uint32_t n_buckets = req.explicit_bucket_set().bucket_ids_size(); + std::vector<document::BucketId> buckets(n_buckets); + const auto& proto_buckets = req.explicit_bucket_set().bucket_ids(); + for (uint32_t i = 0; i < n_buckets; ++i) { + buckets[i] = get_bucket_id(proto_buckets.Get(i)); + } + return std::make_unique<api::RequestBucketInfoCommand>(bucket_space, std::move(buckets)); + } else if (req.has_all_buckets()) { + const auto& all_req = req.all_buckets(); + return std::make_unique<api::RequestBucketInfoCommand>( + bucket_space, all_req.distributor_index(), + lib::ClusterState(all_req.cluster_state()), all_req.distribution_hash()); + } else { + throw vespalib::IllegalArgumentException("RequestBucketInfo does not have any applicable fields set"); + } + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeRequestBucketInfoReply(const SCmd& cmd, BBuf& buf) const { + return decode_response<protobuf::RequestBucketInfoResponse>(buf, [&](auto& res) { + auto reply = std::make_unique<api::RequestBucketInfoReply>(static_cast<const api::RequestBucketInfoCommand&>(cmd)); + auto& dest_entries = reply->getBucketInfo(); + uint32_t n_entries = res.bucket_infos_size(); + dest_entries.resize(n_entries); + for (uint32_t i = 0; i < n_entries; ++i) { + const auto& proto_entry = res.bucket_infos(i); + dest_entries[i]._bucketId = document::BucketId(proto_entry.raw_bucket_id()); + dest_entries[i]._info = get_bucket_info(proto_entry.bucket_info()); + } + return reply; + }); +} + +// ----------------------------------------------------------------- +// NotifyBucketChange +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::NotifyBucketChangeCommand& msg) const { + encode_bucket_request<protobuf::NotifyBucketChangeRequest>(buf, msg, [&](auto& req) { + set_bucket_info(*req.mutable_bucket_info(), msg.getBucketInfo()); + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::NotifyBucketChangeReply& msg) const { + encode_response<protobuf::NotifyBucketChangeResponse>(buf, msg, no_op_encode); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeNotifyBucketChangeCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::NotifyBucketChangeRequest>(buf, [&](auto& req, auto& bucket) { + auto bucket_info = get_bucket_info(req.bucket_info()); + return std::make_unique<api::NotifyBucketChangeCommand>(bucket, bucket_info); + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeNotifyBucketChangeReply(const SCmd& cmd, BBuf& buf) const { + return decode_response<protobuf::NotifyBucketChangeResponse>(buf, [&]([[maybe_unused]] auto& res) { + return std::make_unique<api::NotifyBucketChangeReply>(static_cast<const api::NotifyBucketChangeCommand&>(cmd)); + }); +} + +// ----------------------------------------------------------------- +// SplitBucket +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::SplitBucketCommand& msg) const { + encode_bucket_request<protobuf::SplitBucketRequest>(buf, msg, [&](auto& req) { + req.set_min_split_bits(msg.getMinSplitBits()); + req.set_max_split_bits(msg.getMaxSplitBits()); + req.set_min_byte_size(msg.getMinByteSize()); + req.set_min_doc_count(msg.getMinDocCount()); + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::SplitBucketReply& msg) const { + encode_bucket_response<protobuf::SplitBucketResponse>(buf, msg, [&](auto& res) { + for (const auto& split_info : msg.getSplitInfo()) { + auto* proto_info = res.add_split_info(); + proto_info->set_raw_bucket_id(split_info.first.getRawId()); + set_bucket_info(*proto_info->mutable_bucket_info(), split_info.second); + } + }); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeSplitBucketCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::SplitBucketRequest>(buf, [&](auto& req, auto& bucket) { + auto cmd = std::make_unique<api::SplitBucketCommand>(bucket); + cmd->setMinSplitBits(static_cast<uint8_t>(req.min_split_bits())); + cmd->setMaxSplitBits(static_cast<uint8_t>(req.max_split_bits())); + cmd->setMinByteSize(req.min_byte_size()); + cmd->setMinDocCount(req.min_doc_count()); + return cmd; + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeSplitBucketReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_response<protobuf::SplitBucketResponse>(buf, [&](auto& res) { + auto reply = std::make_unique<api::SplitBucketReply>(static_cast<const api::SplitBucketCommand&>(cmd)); + auto& dest_info = reply->getSplitInfo(); + dest_info.reserve(res.split_info_size()); + for (const auto& proto_info : res.split_info()) { + dest_info.emplace_back(document::BucketId(proto_info.raw_bucket_id()), + get_bucket_info(proto_info.bucket_info())); + } + return reply; + }); +} + +// ----------------------------------------------------------------- +// JoinBuckets +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::JoinBucketsCommand& msg) const { + encode_bucket_request<protobuf::JoinBucketsRequest>(buf, msg, [&](auto& req) { + for (const auto& source : msg.getSourceBuckets()) { + set_bucket_id(*req.add_source_buckets(), source); + } + req.set_min_join_bits(msg.getMinJoinBits()); + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::JoinBucketsReply& msg) const { + encode_bucket_info_response<protobuf::JoinBucketsResponse>(buf, msg, no_op_encode); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeJoinBucketsCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::JoinBucketsRequest>(buf, [&](auto& req, auto& bucket) { + auto cmd = std::make_unique<api::JoinBucketsCommand>(bucket); + auto& entries = cmd->getSourceBuckets(); + for (const auto& proto_bucket : req.source_buckets()) { + entries.emplace_back(get_bucket_id(proto_bucket)); + } + cmd->setMinJoinBits(static_cast<uint8_t>(req.min_join_bits())); + return cmd; + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeJoinBucketsReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_info_response<protobuf::JoinBucketsResponse>(buf, [&]([[maybe_unused]] auto& res) { + return std::make_unique<api::JoinBucketsReply>(static_cast<const api::JoinBucketsCommand&>(cmd)); + }); +} + +// ----------------------------------------------------------------- +// SetBucketState +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::SetBucketStateCommand& msg) const { + encode_bucket_request<protobuf::SetBucketStateRequest>(buf, msg, [&](auto& req) { + auto state = (msg.getState() == api::SetBucketStateCommand::BUCKET_STATE::ACTIVE + ? protobuf::SetBucketStateRequest_BucketState_Active + : protobuf::SetBucketStateRequest_BucketState_Inactive); + req.set_state(state); + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::SetBucketStateReply& msg) const { + // SetBucketStateReply is _technically_ a BucketInfoReply, but the legacy protocol impls + // do _not_ encode bucket info as part of the wire format (and it's not used on the distributor), + // so we follow that here and only encode remapping information. + encode_bucket_response<protobuf::SetBucketStateResponse>(buf, msg, no_op_encode); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeSetBucketStateCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::SetBucketStateRequest>(buf, [&](auto& req, auto& bucket) { + auto state = (req.state() == protobuf::SetBucketStateRequest_BucketState_Active + ? api::SetBucketStateCommand::BUCKET_STATE::ACTIVE + : api::SetBucketStateCommand::BUCKET_STATE::INACTIVE); + return std::make_unique<api::SetBucketStateCommand>(bucket, state); + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeSetBucketStateReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_response<protobuf::SetBucketStateResponse>(buf, [&]([[maybe_unused]] auto& res) { + return std::make_unique<api::SetBucketStateReply>(static_cast<const api::SetBucketStateCommand&>(cmd)); + }); +} + +// ----------------------------------------------------------------- +// CreateVisitor +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::CreateVisitorCommand& msg) const { + encode_request<protobuf::CreateVisitorRequest>(buf, msg, [&](auto& req) { + set_bucket_space(*req.mutable_bucket_space(), msg.getBucketSpace()); + for (const auto& bucket : msg.getBuckets()) { + set_bucket_id(*req.add_buckets(), bucket); + } + + auto* ctrl_meta = req.mutable_control_meta(); + ctrl_meta->set_library_name(msg.getLibraryName().data(), msg.getLibraryName().size()); + ctrl_meta->set_instance_id(msg.getInstanceId().data(), msg.getInstanceId().size()); + ctrl_meta->set_visitor_command_id(msg.getVisitorCmdId()); + ctrl_meta->set_control_destination(msg.getControlDestination().data(), msg.getControlDestination().size()); + ctrl_meta->set_data_destination(msg.getDataDestination().data(), msg.getDataDestination().size()); + ctrl_meta->set_queue_timeout(msg.getQueueTimeout()); + ctrl_meta->set_max_pending_reply_count(msg.getMaximumPendingReplyCount()); + ctrl_meta->set_max_buckets_per_visitor(msg.getMaxBucketsPerVisitor()); + + auto* constraints = req.mutable_constraints(); + constraints->set_document_selection(msg.getDocumentSelection().data(), msg.getDocumentSelection().size()); + constraints->set_from_time_usec(msg.getFromTime()); + constraints->set_to_time_usec(msg.getToTime()); + constraints->set_visit_inconsistent_buckets(msg.visitInconsistentBuckets()); + constraints->set_visit_removes(msg.visitRemoves()); + constraints->set_field_set(msg.getFieldSet().data(), msg.getFieldSet().size()); + + for (const auto& param : msg.getParameters()) { + auto* proto_param = req.add_client_parameters(); + proto_param->set_key(param.first.data(), param.first.size()); + proto_param->set_value(param.second.data(), param.second.size()); + } + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::CreateVisitorReply& msg) const { + encode_response<protobuf::CreateVisitorResponse>(buf, msg, [&](auto& res) { + auto& stats = msg.getVisitorStatistics(); + auto* proto_stats = res.mutable_visitor_statistics(); + proto_stats->set_buckets_visited(stats.getBucketsVisited()); + proto_stats->set_documents_visited(stats.getDocumentsVisited()); + proto_stats->set_bytes_visited(stats.getBytesVisited()); + proto_stats->set_documents_returned(stats.getDocumentsReturned()); + proto_stats->set_bytes_returned(stats.getBytesReturned()); + proto_stats->set_second_pass_documents_returned(stats.getSecondPassDocumentsReturned()); + proto_stats->set_second_pass_bytes_returned(stats.getSecondPassBytesReturned()); + }); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeCreateVisitorCommand(BBuf& buf) const { + return decode_request<protobuf::CreateVisitorRequest>(buf, [&](auto& req) { + auto bucket_space = get_bucket_space(req.bucket_space()); + auto& ctrl_meta = req.control_meta(); + auto& constraints = req.constraints(); + auto cmd = std::make_unique<api::CreateVisitorCommand>(bucket_space, ctrl_meta.library_name(), + ctrl_meta.instance_id(), constraints.document_selection()); + for (const auto& proto_bucket : req.buckets()) { + cmd->getBuckets().emplace_back(get_bucket_id(proto_bucket)); + } + + cmd->setVisitorCmdId(ctrl_meta.visitor_command_id()); + cmd->setControlDestination(ctrl_meta.control_destination()); + cmd->setDataDestination(ctrl_meta.data_destination()); + cmd->setMaximumPendingReplyCount(ctrl_meta.max_pending_reply_count()); + cmd->setQueueTimeout(ctrl_meta.queue_timeout()); + cmd->setMaxBucketsPerVisitor(ctrl_meta.max_buckets_per_visitor()); + cmd->setVisitorDispatcherVersion(50); // FIXME this magic number is lifted verbatim from the 5.1 protocol impl + + for (const auto& proto_param : req.client_parameters()) { + cmd->getParameters().set(proto_param.key(), proto_param.value()); + } + + cmd->setFromTime(constraints.from_time_usec()); + cmd->setToTime(constraints.to_time_usec()); + cmd->setVisitRemoves(constraints.visit_removes()); + cmd->setFieldSet(constraints.field_set()); + cmd->setVisitInconsistentBuckets(constraints.visit_inconsistent_buckets()); + return cmd; + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeCreateVisitorReply(const SCmd& cmd, BBuf& buf) const { + return decode_response<protobuf::CreateVisitorResponse>(buf, [&](auto& res) { + auto reply = std::make_unique<api::CreateVisitorReply>(static_cast<const api::CreateVisitorCommand&>(cmd)); + vdslib::VisitorStatistics vs; + const auto& proto_stats = res.visitor_statistics(); + vs.setBucketsVisited(proto_stats.buckets_visited()); + vs.setDocumentsVisited(proto_stats.documents_visited()); + vs.setBytesVisited(proto_stats.bytes_visited()); + vs.setDocumentsReturned(proto_stats.documents_returned()); + vs.setBytesReturned(proto_stats.bytes_returned()); + vs.setSecondPassDocumentsReturned(proto_stats.second_pass_documents_returned()); + vs.setSecondPassBytesReturned(proto_stats.second_pass_bytes_returned()); + reply->setVisitorStatistics(vs); + return reply; + }); +} + +// ----------------------------------------------------------------- +// DestroyVisitor +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::DestroyVisitorCommand& msg) const { + encode_request<protobuf::DestroyVisitorRequest>(buf, msg, [&](auto& req) { + req.set_instance_id(msg.getInstanceId().data(), msg.getInstanceId().size()); + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::DestroyVisitorReply& msg) const { + encode_response<protobuf::DestroyVisitorResponse>(buf, msg, no_op_encode); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeDestroyVisitorCommand(BBuf& buf) const { + return decode_request<protobuf::DestroyVisitorRequest>(buf, [&](auto& req) { + return std::make_unique<api::DestroyVisitorCommand>(req.instance_id()); + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeDestroyVisitorReply(const SCmd& cmd, BBuf& buf) const { + return decode_response<protobuf::DestroyVisitorResponse>(buf, [&]([[maybe_unused]] auto& res) { + return std::make_unique<api::DestroyVisitorReply>(static_cast<const api::DestroyVisitorCommand&>(cmd)); + }); +} + +} // storage::mbusprot diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.h new file mode 100644 index 00000000000..f3499150278 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.h @@ -0,0 +1,146 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "protocolserialization.h" +#include <vespa/documentapi/loadtypes/loadtypeset.h> + +namespace storage { +namespace mbusprot { + +/** + * Protocol serialization version that uses Protocol Buffers for all its binary + * encoding and decoding. + */ +class ProtocolSerialization7 : public ProtocolSerialization { + const std::shared_ptr<const document::DocumentTypeRepo> _repo; + const documentapi::LoadTypeSet& _load_types; +public: + ProtocolSerialization7(std::shared_ptr<const document::DocumentTypeRepo> repo, + const documentapi::LoadTypeSet& load_types); + + const document::DocumentTypeRepo& type_repo() const { return *_repo; } + + // Put + void onEncode(GBBuf&, const api::PutCommand&) const override; + void onEncode(GBBuf&, const api::PutReply&) const override; + SCmd::UP onDecodePutCommand(BBuf&) const override; + SRep::UP onDecodePutReply(const SCmd&, BBuf&) const override; + + // Update + void onEncode(GBBuf&, const api::UpdateCommand&) const override; + void onEncode(GBBuf&, const api::UpdateReply&) const override; + SCmd::UP onDecodeUpdateCommand(BBuf&) const override; + SRep::UP onDecodeUpdateReply(const SCmd&, BBuf&) const override; + + // Remove + void onEncode(GBBuf&, const api::RemoveCommand&) const override; + void onEncode(GBBuf&, const api::RemoveReply&) const override; + SCmd::UP onDecodeRemoveCommand(BBuf&) const override; + SRep::UP onDecodeRemoveReply(const SCmd&, BBuf&) const override; + + // Get + void onEncode(GBBuf&, const api::GetCommand&) const override; + void onEncode(GBBuf&, const api::GetReply&) const override; + SCmd::UP onDecodeGetCommand(BBuf&) const override; + SRep::UP onDecodeGetReply(const SCmd&, BBuf&) const override; + + // Revert - TODO this is deprecated, no? + void onEncode(GBBuf&, const api::RevertCommand&) const override; + void onEncode(GBBuf&, const api::RevertReply&) const override; + SCmd::UP onDecodeRevertCommand(BBuf&) const override; + SRep::UP onDecodeRevertReply(const SCmd&, BBuf&) const override; + + // DeleteBucket + void onEncode(GBBuf&, const api::DeleteBucketCommand&) const override; + void onEncode(GBBuf&, const api::DeleteBucketReply&) const override; + SCmd::UP onDecodeDeleteBucketCommand(BBuf&) const override; + SRep::UP onDecodeDeleteBucketReply(const SCmd&, BBuf&) const override; + + // CreateBucket + void onEncode(GBBuf&, const api::CreateBucketCommand&) const override; + void onEncode(GBBuf&, const api::CreateBucketReply&) const override; + SCmd::UP onDecodeCreateBucketCommand(BBuf&) const override; + SRep::UP onDecodeCreateBucketReply(const SCmd&, BBuf&) const override; + + // MergeBucket + void onEncode(GBBuf&, const api::MergeBucketCommand&) const override; + void onEncode(GBBuf&, const api::MergeBucketReply&) const override; + SCmd::UP onDecodeMergeBucketCommand(BBuf&) const override; + SRep::UP onDecodeMergeBucketReply(const SCmd&, BBuf&) const override; + + // GetBucketDiff + void onEncode(GBBuf&, const api::GetBucketDiffCommand&) const override; + void onEncode(GBBuf&, const api::GetBucketDiffReply&) const override; + SCmd::UP onDecodeGetBucketDiffCommand(BBuf&) const override; + SRep::UP onDecodeGetBucketDiffReply(const SCmd&, BBuf&) const override; + + // ApplyBucketDiff + void onEncode(GBBuf&, const api::ApplyBucketDiffCommand&) const override; + void onEncode(GBBuf&, const api::ApplyBucketDiffReply&) const override; + SCmd::UP onDecodeApplyBucketDiffCommand(BBuf&) const override; + SRep::UP onDecodeApplyBucketDiffReply(const SCmd&, BBuf&) const override; + + // RequestBucketInfo + void onEncode(GBBuf&, const api::RequestBucketInfoCommand&) const override; + void onEncode(GBBuf&, const api::RequestBucketInfoReply&) const override; + SCmd::UP onDecodeRequestBucketInfoCommand(BBuf&) const override; + SRep::UP onDecodeRequestBucketInfoReply(const SCmd&, BBuf&) const override; + + // NotifyBucketChange + void onEncode(GBBuf&, const api::NotifyBucketChangeCommand&) const override; + void onEncode(GBBuf&, const api::NotifyBucketChangeReply&) const override; + SCmd::UP onDecodeNotifyBucketChangeCommand(BBuf&) const override; + SRep::UP onDecodeNotifyBucketChangeReply(const SCmd&, BBuf&) const override; + + // SplitBucket + void onEncode(GBBuf&, const api::SplitBucketCommand&) const override; + void onEncode(GBBuf&, const api::SplitBucketReply&) const override; + SCmd::UP onDecodeSplitBucketCommand(BBuf&) const override; + SRep::UP onDecodeSplitBucketReply(const SCmd&, BBuf&) const override; + + // JoinBuckets + void onEncode(GBBuf&, const api::JoinBucketsCommand&) const override; + void onEncode(GBBuf&, const api::JoinBucketsReply&) const override; + SCmd::UP onDecodeJoinBucketsCommand(BBuf&) const override; + SRep::UP onDecodeJoinBucketsReply(const SCmd&, BBuf&) const override; + + // SetBucketState + void onEncode(GBBuf&, const api::SetBucketStateCommand&) const override; + void onEncode(GBBuf&, const api::SetBucketStateReply&) const override; + SCmd::UP onDecodeSetBucketStateCommand(BBuf&) const override; + SRep::UP onDecodeSetBucketStateReply(const SCmd&, BBuf&) const override; + + // CreateVisitor + void onEncode(GBBuf&, const api::CreateVisitorCommand&) const override; + void onEncode(GBBuf&, const api::CreateVisitorReply&) const override; + SCmd::UP onDecodeCreateVisitorCommand(BBuf&) const override; + SRep::UP onDecodeCreateVisitorReply(const SCmd&, BBuf&) const override; + + // DestroyVisitor + void onEncode(GBBuf&, const api::DestroyVisitorCommand&) const override; + void onEncode(GBBuf&, const api::DestroyVisitorReply&) const override; + SCmd::UP onDecodeDestroyVisitorCommand(BBuf&) const override; + SRep::UP onDecodeDestroyVisitorReply(const SCmd&, BBuf&) const override; + + // RemoveLocation + void onEncode(GBBuf&, const api::RemoveLocationCommand&) const override; + void onEncode(GBBuf&, const api::RemoveLocationReply&) const override; + SCmd::UP onDecodeRemoveLocationCommand(BBuf&) const override; + SRep::UP onDecodeRemoveLocationReply(const SCmd&, BBuf&) const override; + +private: + template <typename ProtobufType, typename Func> + std::unique_ptr<api::StorageCommand> decode_request(document::ByteBuffer& in_buf, Func&& f) const; + template <typename ProtobufType, typename Func> + std::unique_ptr<api::StorageReply> decode_response(document::ByteBuffer& in_buf, Func&& f) const; + template <typename ProtobufType, typename Func> + std::unique_ptr<api::StorageCommand> decode_bucket_request(document::ByteBuffer& in_buf, Func&& f) const; + template <typename ProtobufType, typename Func> + std::unique_ptr<api::StorageReply> decode_bucket_response(document::ByteBuffer& in_buf, Func&& f) const; + template <typename ProtobufType, typename Func> + std::unique_ptr<api::StorageReply> decode_bucket_info_response(document::ByteBuffer& in_buf, Func&& f) const; +}; + +} +} diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp index 7e6be0a84f5..33fcc29db11 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp @@ -20,7 +20,8 @@ StorageProtocol::StorageProtocol(const std::shared_ptr<const document::DocumentT : _serializer5_0(repo, loadTypes), _serializer5_1(repo, loadTypes), _serializer5_2(repo, loadTypes), - _serializer6_0(repo, loadTypes) + _serializer6_0(repo, loadTypes), + _serializer7_0(repo, loadTypes) { } @@ -33,6 +34,7 @@ StorageProtocol::createPolicy(const mbus::string&, const mbus::string&) const } namespace { + vespalib::Version version7_0(7, 41, 19); vespalib::Version version6_0(6, 240, 0); vespalib::Version version5_2(5, 93, 30); vespalib::Version version5_1(5, 1, 0); @@ -106,8 +108,10 @@ StorageProtocol::encode(const vespalib::Version& version, } else { if (version < version6_0) { return encodeMessage(_serializer5_2, routable, message, version5_2, version); - } else { + } else if (version < version7_0) { return encodeMessage(_serializer6_0, routable, message, version6_0, version); + } else { + return encodeMessage(_serializer7_0, routable, message, version7_0, version); } } @@ -180,8 +184,10 @@ StorageProtocol::decode(const vespalib::Version & version, } else { if (version < version6_0) { return decodeMessage(_serializer5_2, data, type, version5_2, version); - } else { + } else if (version < version7_0) { return decodeMessage(_serializer6_0, data, type, version6_0, version); + } else { + return decodeMessage(_serializer7_0, data, type, version7_0, version); } } } catch (std::exception & e) { diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h index 1acd7c9675f..67ea121c340 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h +++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h @@ -3,6 +3,7 @@ #include "protocolserialization5_2.h" #include "protocolserialization6_0.h" +#include "protocolserialization7.h" #include <vespa/messagebus/iprotocol.h> namespace storage::mbusprot { @@ -28,6 +29,7 @@ private: ProtocolSerialization5_1 _serializer5_1; ProtocolSerialization5_2 _serializer5_2; ProtocolSerialization6_0 _serializer6_0; + ProtocolSerialization7 _serializer7_0; }; } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java index 433fee69e4e..8e820a89eb1 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClient.java @@ -9,23 +9,27 @@ import java.util.concurrent.atomic.AtomicInteger; /** * API for feeding document operations (add, removes or updates) to one or many Vespa clusters. - * Use the factory to configure and set up an instance of this API. + * Use the factory to configure and set up an instance of this. Instances are expensive - create one instance of this + * and use it for all feed operations (from multiple threads, if desired) for the duration of your client runtime. * The feedclient does automatic error recovery and reconnects to hosts when connections die. * * A {@link FeedClientFactory} is provided to instantiate Sessions. * * See com.yahoo.text.Text.stripInvalidCharacters(String) to remove invalid characters from string fields before feeding * + * Instances of this are multithread safe. + * * @author dybis * @see FeedClientFactory */ public interface FeedClient extends AutoCloseable { /** - * Streams a document to cluster(s). If the pipeline and buffers are full, this call will be blocking. - * Documents might time out before they are sent. Failed documents are not retried. - * Don't call stream() after close is called. - * + * Issues a document operation to the configured cluster(s). + * If the pipeline and buffers are full, this call will be blocking, ensuring that operations are not + * produced faster than the can be handled. Transient failured are retried internally by this client. + * Exactly one callback will always be received for each (completed) call to this. + * * @param documentId the document id of the document. * @param documentData the document data as JSON or XML (as specified when using the factory to create the API) */ @@ -34,10 +38,11 @@ public interface FeedClient extends AutoCloseable { } /** - * Streams a document to cluster(s). If the pipeline and buffers are full, this call will be blocking. - * Documents might time out before they are sent. Failed documents are not retried. - * Don't call stream() after close is called. - * + * Issues a document operation to the configured cluster(s). + * If the pipeline and buffers are full, this call will be blocking, ensuring that operations are not + * produced faster than the can be handled. Transient failured are retried internally by this client. + * Exactly one callback will always be received for each (completed) call to this. + * * @param documentId the document id of the document. * @param documentData the document data as JSON or XML (as specified when using the factory to create the API) * @param context a context object which will be accessible in the result of the callback, or null if none @@ -47,9 +52,10 @@ public interface FeedClient extends AutoCloseable { } /** - * Streams a document to cluster(s). If the pipeline and buffers are full, this call will be blocking. - * Documents might time out before they are sent. Failed documents are not retried. - * Don't call stream() after close is called. + * Issues a document operation to the configured cluster(s). + * If the pipeline and buffers are full, this call will be blocking, ensuring that operations are not + * produced faster than the can be handled. Transient failured are retried internally by this client. + * Exactly one callback will always be received for each (completed) call to this. * * @param documentId the document id of the document. * @param operationId the id to use for this operation, or null to let the client decide an operation id. @@ -61,30 +67,6 @@ public interface FeedClient extends AutoCloseable { void stream(String documentId, String operationId, CharSequence documentData, Object context); /** - * This callback is executed when new results are arriving or an error occur. - * Don't do any heavy lifting in this thread (no IO, disk, or heavy CPU usage). - * This call back will run in a different thread than your main program so use e.g. - * AtomicInteger for counters and follow general guides for thread-safe programming. - * There is an example implementation in class SimpleLoggerResultCallback. - */ - interface ResultCallback { - - void onCompletion(String docId, Result documentResult); - - /** - * Called with an exception whenever an endpoint specific error occurs during feeding. - * The error may or may not be transient - the operation will in both cases be retried until it's successful. - * This callback is intended for application level monitoring (logging, metrics, altering etc). - * Document specific errors will be reported back through {@link #onCompletion(String, Result)}. - * - * @see FeedEndpointException - * @param exception An exception specifying endpoint and cause. See {@link FeedEndpointException} for details. - */ - default void onEndpointException(FeedEndpointException exception) {} - - } - - /** * Waits for all results to arrive and closes the FeedClient. Don't call any other method after calling close(). * Does not throw any exceptions. */ @@ -128,4 +110,32 @@ public interface FeedClient extends AutoCloseable { } } + /** + * This callback is executed when new results are arriving or an error occur. + * Don't do any heavy lifting in this thread (no IO, disk, or heavy CPU usage). + * This call back will run in a different thread than your main program so use e.g. + * AtomicInteger for counters and follow general guides for thread-safe programming. + * There is an example implementation in class SimpleLoggerResultCallback. + */ + interface ResultCallback { + + /** + * This callback is always called exactly once for each feed operation passed to the client + * instance, whether or not it was successful. + */ + void onCompletion(String docId, Result documentResult); + + /** + * Called with an exception whenever an endpoint specific error occurs during feeding. + * The error may or may not be transient - the operation will in both cases be retried until it's successful. + * This callback is intended for application level monitoring (logging, metrics, altering etc). + * Document specific errors will be reported back through {@link #onCompletion(String, Result)}. + * + * @see FeedEndpointException + * @param exception An exception specifying endpoint and cause. See {@link FeedEndpointException} for details. + */ + default void onEndpointException(FeedEndpointException exception) {} + + } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClientFactory.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClientFactory.java index 6095134b7a2..4d50905da7b 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClientFactory.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/FeedClientFactory.java @@ -5,7 +5,9 @@ package com.yahoo.vespa.http.client; import com.yahoo.vespa.http.client.config.SessionParams; import com.yahoo.vespa.http.client.core.api.FeedClientImpl; -import static com.yahoo.vespa.http.client.SessionFactory.createTimeoutExecutor; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadFactory; /** * Factory for creating FeedClient. @@ -15,7 +17,7 @@ import static com.yahoo.vespa.http.client.SessionFactory.createTimeoutExecutor; public class FeedClientFactory { /** - * Creates a FeedClient. + * Creates a FeedClient. Call this sparingly: Feed clients are expensive and should be as long-lived as possible. * * @param sessionParams parameters for connection, hosts, cluster configurations and more. * @param resultCallback on each result, this callback is called. @@ -25,4 +27,31 @@ public class FeedClientFactory { return new FeedClientImpl(sessionParams, resultCallback, createTimeoutExecutor()); } + static ScheduledThreadPoolExecutor createTimeoutExecutor() { + ScheduledThreadPoolExecutor timeoutExecutor; + timeoutExecutor = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("timeout-")); + timeoutExecutor.setRemoveOnCancelPolicy(true); + timeoutExecutor.setContinueExistingPeriodicTasksAfterShutdownPolicy(false); + timeoutExecutor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); + return timeoutExecutor; + } + + private static class DaemonThreadFactory implements ThreadFactory { + + private final ThreadFactory defaultThreadFactory = Executors.defaultThreadFactory(); + private final String prefix; + + private DaemonThreadFactory(String prefix) { + this.prefix = prefix; + } + + @Override + public Thread newThread(Runnable runnable) { + Thread t = defaultThreadFactory.newThread(runnable); + t.setDaemon(true); + t.setName(prefix + t.getName()); + return t; + } + } + } diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Session.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Session.java index de39eabd3ee..3089717c260 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Session.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/Session.java @@ -15,7 +15,9 @@ import java.util.concurrent.BlockingQueue; * * @author Einar M R Rosenvinge * @see SessionFactory + * @deprecated use either FeedClient or SyncFeedClient // TODO: Remove on Vespa 8 */ +@Deprecated public interface Session extends AutoCloseable { /** diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SessionFactory.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SessionFactory.java index 2e47e45dff0..b03a2541cd0 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SessionFactory.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/SessionFactory.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.http.client; import com.yahoo.vespa.http.client.config.Cluster; import com.yahoo.vespa.http.client.config.Endpoint; import com.yahoo.vespa.http.client.config.SessionParams; -import com.yahoo.vespa.http.client.core.api.SessionImpl; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledThreadPoolExecutor; @@ -14,7 +13,9 @@ import java.util.concurrent.ThreadFactory; * Factory for creating {@link Session} instances. * * @author Einar M R Rosenvinge + * @deprecated use either FeedClient or SyncFeedClient // TODO: Remove on Vespa 8 */ +@Deprecated public final class SessionFactory { /** @@ -29,7 +30,7 @@ public final class SessionFactory { @SuppressWarnings("deprecation") static Session createInternal(SessionParams params) { - return new SessionImpl(params, createTimeoutExecutor()); + return new com.yahoo.vespa.http.client.core.api.SessionImpl(params, createTimeoutExecutor()); } static ScheduledThreadPoolExecutor createTimeoutExecutor() { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java index bd187ea3371..f503190864b 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java @@ -16,10 +16,7 @@ import java.util.Objects; import java.util.concurrent.TimeUnit; /** - * Parameters given to a {@link com.yahoo.vespa.http.client.SessionFactory} - * when creating {@link com.yahoo.vespa.http.client.Session}s. The parameters - * contained in this class are related to the connections from the node running - * the Session to the Vespa clusters. + * Connection level parameters. * This class is immutable * and has no public constructor - to instantiate one, use a {@link Builder}. * @@ -31,9 +28,9 @@ public final class ConnectionParams { * Builder for {@link ConnectionParams}. */ public static final class Builder { + private SSLContext sslContext = null; private HostnameVerifier hostnameVerifier = SSLConnectionSocketFactory.getDefaultHostnameVerifier(); - private long connectionTimeout = TimeUnit.SECONDS.toMillis(60); private final Multimap<String, String> headers = ArrayListMultimap.create(); private final Map<String, HeaderProvider> headerProviders = new HashMap<>(); private int numPersistentConnectionsPerEndpoint = 1; @@ -225,7 +222,6 @@ public final class ConnectionParams { return new ConnectionParams( sslContext, hostnameVerifier, - connectionTimeout, headers, headerProviders, numPersistentConnectionsPerEndpoint, @@ -280,7 +276,6 @@ public final class ConnectionParams { } private final SSLContext sslContext; private final HostnameVerifier hostnameVerifier; - private final long connectionTimeout; private final Multimap<String, String> headers = ArrayListMultimap.create(); private final Map<String, HeaderProvider> headerProviders = new HashMap<>(); private final int numPersistentConnectionsPerEndpoint; @@ -297,7 +292,6 @@ public final class ConnectionParams { private ConnectionParams( SSLContext sslContext, HostnameVerifier hostnameVerifier, - long connectionTimeout, Multimap<String, String> headers, Map<String, HeaderProvider> headerProviders, int numPersistentConnectionsPerEndpoint, @@ -312,7 +306,6 @@ public final class ConnectionParams { boolean printTraceToStdErr) { this.sslContext = sslContext; this.hostnameVerifier = hostnameVerifier; - this.connectionTimeout = connectionTimeout; this.headers.putAll(headers); this.headerProviders.putAll(headerProviders); this.numPersistentConnectionsPerEndpoint = numPersistentConnectionsPerEndpoint; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/FeedParams.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/FeedParams.java index 30f1ad11d3a..008f3b63a89 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/FeedParams.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/FeedParams.java @@ -6,8 +6,7 @@ import com.google.common.annotations.Beta; import java.util.concurrent.TimeUnit; /** - * Parameters given to a {@link com.yahoo.vespa.http.client.SessionFactory} - * when creating {@link com.yahoo.vespa.http.client.Session}s. This class is immutable + * Feed level parameters. This class is immutable * and has no public constructor - to instantiate one, use a {@link Builder}. * @author Einar M R Rosenvinge @@ -25,8 +24,8 @@ public final class FeedParams { public boolean getSilentUpgrade() { return silentUpgrade; } /** - * Enumeration of data formats that are acceptable by the OutputStream - * returned by {@link com.yahoo.vespa.http.client.Session#stream(CharSequence)}. + * Enumeration of data formats that are acceptable by the + * {@link com.yahoo.vespa.http.client.FeedClient} methods. */ public enum DataFormat { /** UTF-8-encoded XML. Preamble is not necessary. */ @@ -117,9 +116,7 @@ public final class FeedParams { * * Note that the TOTAL timeout of any one operation in this API would be * {@link #getServerTimeout(java.util.concurrent.TimeUnit)} + - * {@link #getClientTimeout(java.util.concurrent.TimeUnit)}, - * after which {@link com.yahoo.vespa.http.client.Session#results()} is guaranteed - * to produce a Result. + * {@link #getClientTimeout(java.util.concurrent.TimeUnit)}. * * @param serverTimeout timeout value * @param unit unit of timeout value @@ -135,14 +132,13 @@ public final class FeedParams { /** * Sets the client-side timeout for each operation. If BOTH the server-side - * timeout AND this timeout has passed, {@link com.yahoo.vespa.http.client.Session} + * timeout AND this timeout has passed, the {@link com.yahoo.vespa.http.client.FeedClient} * will synthesize a {@link com.yahoo.vespa.http.client.Result}. * * Note that the TOTAL timeout of any one operation in this API would be * {@link #getServerTimeout(java.util.concurrent.TimeUnit)} + * {@link #getClientTimeout(java.util.concurrent.TimeUnit)}, - * after which {@link com.yahoo.vespa.http.client.Session#results()} is guaranteed - * to produce a Result. + * after which a result callback is guaranteed to be made. * * @param clientTimeout timeout value * @param unit unit of timeout value diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/SessionParams.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/SessionParams.java index daa6eecb823..48fd21e2b1f 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/SessionParams.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/SessionParams.java @@ -8,12 +8,12 @@ import java.util.LinkedList; import java.util.List; /** - * Parameters given to a {@link com.yahoo.vespa.http.client.SessionFactory} - * when creating {@link com.yahoo.vespa.http.client.Session}s. This class is immutable + * Parameters given to a {@link com.yahoo.vespa.http.client.FeedClientFactory} + * when creating {@link com.yahoo.vespa.http.client.FeedClient}s. This class is immutable * and has no public constructor - to instantiate one, use a {@link Builder}. * * @author Einar M R Rosenvinge - * @see com.yahoo.vespa.http.client.SessionFactory + * @see com.yahoo.vespa.http.client.FeedClientFactory * @see Builder */ public final class SessionParams { @@ -85,8 +85,7 @@ public final class SessionParams { /** * Sets the maximum number of document operations to hold in memory, waiting to be - * sent to Vespa. When this threshold is reached, {@link java.io.OutputStream#close()} will block, - * see {@link com.yahoo.vespa.http.client.Session#stream(CharSequence)}. + * sent to Vespa. When this threshold is reached, {@link java.io.OutputStream#close()} will block. * * @param clientQueueSize the maximum number of document operations to hold in memory. * @return pointer to builder. @@ -111,7 +110,7 @@ public final class SessionParams { } /** - * Instantiates a {@link SessionParams} that can be given to a {@link com.yahoo.vespa.http.client.SessionFactory}. + * Instantiates a {@link SessionParams} that can be given to a {@link com.yahoo.vespa.http.client.FeedClientFactory}. * * @return a SessionParams object with the parameters of this Builder */ @@ -133,6 +132,7 @@ public final class SessionParams { return throttlerMinSize; } } + private final List<Cluster> clusters; private final FeedParams feedParams; private final ConnectionParams connectionParams; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/SessionImpl.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/SessionImpl.java index c3b5d9912de..a5c97351347 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/SessionImpl.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/api/SessionImpl.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.http.client.core.api; import com.yahoo.vespa.http.client.FeedClient; import com.yahoo.vespa.http.client.Result; -import com.yahoo.vespa.http.client.Session; import com.yahoo.vespa.http.client.config.SessionParams; import com.yahoo.vespa.http.client.core.ThrottlePolicy; import com.yahoo.vespa.http.client.core.operationProcessor.IncompleteResultsThrottler; @@ -16,8 +15,11 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; /** * This class wires up the Session API using MultiClusterHandler and MultiClusterSessionOutputStream. + * + * @deprecated */ -public class SessionImpl implements Session { +@Deprecated // TODO: Remove on Vespa 8 +public class SessionImpl implements com.yahoo.vespa.http.client.Session { private final OperationProcessor operationProcessor; private final BlockingQueue<Result> resultQueue = new LinkedBlockingQueue<>(); diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java index f9357ab16d8..5289a7a562a 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/core/communication/ApacheGatewayConnection.java @@ -361,12 +361,12 @@ class ApacheGatewayConnection implements GatewayConnection { @Override public void handshake() throws ServerResponseException, IOException { - final boolean useCompression = false; - final boolean drain = false; - final boolean handshake = true; + boolean useCompression = false; + boolean drain = false; + boolean handshake = true; HttpPost httpPost = createPost(drain, useCompression, handshake); - final String oldSessionID = sessionId; + String oldSessionID = sessionId; sessionId = null; try (InputStream stream = executePost(httpPost)) { if (oldSessionID != null && !oldSessionID.equals(sessionId)) { diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/package-info.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/package-info.java index b14c2ffa4cc..c54c19a2eb1 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/package-info.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/package-info.java @@ -1,10 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** * Programmatic API for feeding to Vespa clusters independently of the - * cluster configuration. {@link com.yahoo.vespa.http.client.Session} - * is the central interface which is used to interact with a cluster. - * Use {@link com.yahoo.vespa.http.client.SessionFactory} to - * instantiate a {@link com.yahoo.vespa.http.client.Session}. + * cluster configuration. * * NOTE: This is a PUBLIC API, but not annotated as such because this is not a bundle and * we don't want to introduce Vespa dependencies. diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/FeedClientTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/FeedClientTest.java index 33afc759f59..88deaa07e12 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/FeedClientTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/FeedClientTest.java @@ -34,13 +34,14 @@ public class FeedClientTest { .build()) .build(); final AtomicInteger resultsReceived = new AtomicInteger(0); + FeedClient.ResultCallback resultCallback = (docId, documentResult) -> { assertTrue(documentResult.isSuccess()); assertEquals(DOCID, docId); resultsReceived.incrementAndGet(); }; - FeedClient feedClient = new FeedClientImpl(sessionParams, resultCallback, SessionFactory.createTimeoutExecutor()); + FeedClient feedClient = new FeedClientImpl(sessionParams, resultCallback, FeedClientFactory.createTimeoutExecutor()); @Test public void testStreamAndClose() { diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/QueueBoundsTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/QueueBoundsTest.java index 357296eb789..08c9299dcde 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/QueueBoundsTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/QueueBoundsTest.java @@ -6,7 +6,6 @@ import com.yahoo.vespa.http.client.config.ConnectionParams; import com.yahoo.vespa.http.client.config.Endpoint; import com.yahoo.vespa.http.client.config.FeedParams; import com.yahoo.vespa.http.client.config.SessionParams; -import com.yahoo.vespa.http.client.core.api.SessionImpl; import com.yahoo.vespa.http.client.handlers.V3MockParsingRequestHandler; import org.junit.Test; @@ -33,6 +32,7 @@ import static org.junit.Assert.fail; * * @author Einar M R Rosenvinge */ +@SuppressWarnings("deprecation") public class QueueBoundsTest extends TestOnCiBuildingSystemOnly { public static final List<TestDocument> documents; @@ -64,7 +64,7 @@ public class QueueBoundsTest extends TestOnCiBuildingSystemOnly { mockXmlParsingRequestHandler.setScenario(V3MockParsingRequestHandler.Scenario.DONT_ACCEPT_VERSION); try (Server server = new Server(mockXmlParsingRequestHandler, 0); Session session = - new SessionImpl( + new com.yahoo.vespa.http.client.core.api.SessionImpl( new SessionParams.Builder() .addCluster(new Cluster.Builder() .addEndpoint(Endpoint.create("localhost", server.getPort(), false)) @@ -108,7 +108,7 @@ public class QueueBoundsTest extends TestOnCiBuildingSystemOnly { try (Server serverA = new Server(new V3MockParsingRequestHandler("A"), 0); Server serverB = new Server(slowHandler, 0); - SessionImpl session = new SessionImpl( + com.yahoo.vespa.http.client.core.api.SessionImpl session = new com.yahoo.vespa.http.client.core.api.SessionImpl( new SessionParams.Builder() .addCluster(new Cluster.Builder() .addEndpoint(Endpoint.create("localhost", serverA.getPort(), false)) @@ -195,7 +195,7 @@ public class QueueBoundsTest extends TestOnCiBuildingSystemOnly { mockXmlParsingRequestHandler.setScenario(V3MockParsingRequestHandler.Scenario.DONT_ACCEPT_VERSION); try (Server server = new Server(mockXmlParsingRequestHandler, 0); Session session = - new SessionImpl( + new com.yahoo.vespa.http.client.core.api.SessionImpl( new SessionParams.Builder() .addCluster(new Cluster.Builder() .addEndpoint(Endpoint.create("localhost", server.getPort(), false)) @@ -277,7 +277,7 @@ public class QueueBoundsTest extends TestOnCiBuildingSystemOnly { } } - private void assertIncompleteResultQueueSize(SessionImpl session, int size, long timeout, TimeUnit timeUnit) throws InterruptedException { + private void assertIncompleteResultQueueSize(com.yahoo.vespa.http.client.core.api.SessionImpl session, int size, long timeout, TimeUnit timeUnit) throws InterruptedException { long timeoutMs = TimeUnit.MILLISECONDS.convert(timeout, timeUnit); long waitTimeMs = 0; while (true) { diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestUtils.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestUtils.java index 8827c863024..b44385e11ff 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestUtils.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/TestUtils.java @@ -13,10 +13,11 @@ import java.util.zip.GZIPInputStream; import static org.junit.Assert.assertNull; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.20 + * @author Einar M R Rosenvinge */ +@SuppressWarnings("deprecation") public class TestUtils { + public static void writeDocuments(Session session, List<TestDocument> documents) throws IOException { for (TestDocument document : documents) { writeDocument(session, document); @@ -57,4 +58,5 @@ public class TestUtils { } return rawContent.toString(); } + } diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/V3HttpAPIMultiClusterTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/V3HttpAPIMultiClusterTest.java index 88f5496d929..8c4b24f74c2 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/V3HttpAPIMultiClusterTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/V3HttpAPIMultiClusterTest.java @@ -23,9 +23,10 @@ import static org.junit.Assert.fail; /** * Only runs on screwdriver to save time! - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - * @since 5.1.27 + * + * @author Einar M R Rosenvinge */ +@SuppressWarnings("deprecation") public class V3HttpAPIMultiClusterTest extends TestOnCiBuildingSystemOnly { private void writeDocuments(Session session) throws IOException { diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/V3HttpAPITest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/V3HttpAPITest.java index 594a49e7d91..ad54bbc6a90 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/V3HttpAPITest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/V3HttpAPITest.java @@ -28,6 +28,7 @@ import static org.junit.Assert.assertThat; * * @author Einar M R Rosenvinge */ +@SuppressWarnings("deprecation") public class V3HttpAPITest extends TestOnCiBuildingSystemOnly { public static final List<TestDocument> documents; diff --git a/vespa_feed_perf/pom.xml b/vespa_feed_perf/pom.xml index c6cf0cb9edf..a4edd9cda2c 100644 --- a/vespa_feed_perf/pom.xml +++ b/vespa_feed_perf/pom.xml @@ -34,11 +34,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-core</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>component</artifactId> <version>${project.version}</version> diff --git a/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/FeederParams.java b/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/FeederParams.java index e7738d92818..ffe1eb42e3e 100644 --- a/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/FeederParams.java +++ b/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/FeederParams.java @@ -7,100 +7,115 @@ import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.InputStream; +import java.io.OutputStream; import java.io.PrintStream; /** * @author Simon Thoresen Hult */ -public class FeederParams { +class FeederParams { + enum DumpFormat {JSON, VESPA}; private InputStream stdIn = System.in; private PrintStream stdErr = System.err; private PrintStream stdOut = System.out; private Route route = Route.parse("default"); private String configId = "client"; + private OutputStream dumpStream = null; + private DumpFormat dumpFormat = DumpFormat.JSON; private boolean serialTransferEnabled = false; private int numDispatchThreads = 1; - public InputStream getStdIn() { + InputStream getStdIn() { return stdIn; } - public FeederParams setStdIn(InputStream stdIn) { + FeederParams setStdIn(InputStream stdIn) { this.stdIn = stdIn; return this; } - public PrintStream getStdErr() { + PrintStream getStdErr() { return stdErr; } - public FeederParams setStdErr(PrintStream stdErr) { + FeederParams setStdErr(PrintStream stdErr) { this.stdErr = stdErr; return this; } - public PrintStream getStdOut() { + PrintStream getStdOut() { return stdOut; } - public FeederParams setStdOut(PrintStream stdOut) { + FeederParams setStdOut(PrintStream stdOut) { this.stdOut = stdOut; return this; } - public Route getRoute() { + Route getRoute() { return route; } + OutputStream getDumpStream() { return dumpStream; } + FeederParams setDumpStream(OutputStream dumpStream) { + this.dumpStream = dumpStream; + return this; + } - public FeederParams setRoute(Route route) { - this.route = new Route(route); + DumpFormat getDumpFormat() { return dumpFormat; } + FeederParams setDumpFormat(DumpFormat dumpFormat) { + this.dumpFormat = dumpFormat; return this; } - public String getConfigId() { + String getConfigId() { return configId; } - public FeederParams setConfigId(String configId) { + FeederParams setConfigId(String configId) { this.configId = configId; return this; } - public boolean isSerialTransferEnabled() { + boolean isSerialTransferEnabled() { return serialTransferEnabled; } - public FeederParams setSerialTransfer(boolean serial) { + FeederParams setSerialTransfer(boolean serial) { this.serialTransferEnabled = serial; return this; } - public int getNumDispatchThreads() { return numDispatchThreads; } + int getNumDispatchThreads() { return numDispatchThreads; } - public FeederParams parseArgs(String... args) throws ParseException { + FeederParams parseArgs(String... args) throws ParseException, FileNotFoundException { Options opts = new Options(); opts.addOption("s", "serial", false, "use serial transfer mode, at most 1 pending operation"); - opts.addOption("n", "numthreads", true, "Number of clients for sending messages."); + opts.addOption("n", "numthreads", true, "Number of clients for sending messages. Anything, but 1 will bypass sequencing by document id."); + opts.addOption("r", "route", true, "Route for sending messages. default is 'default'...."); + opts.addOption("o", "output", true, "File to write to. Extensions gives format (.xml, .json, .vespa) json will be produced if no extension."); CommandLine cmd = new DefaultParser().parse(opts, args); - serialTransferEnabled = cmd.hasOption("s"); + serialTransferEnabled = cmd.hasOption('s'); if (cmd.hasOption('n')) { numDispatchThreads = Integer.valueOf(cmd.getOptionValue('n').trim()); } - route = newRoute(cmd.getArgs()); - return this; - } - - private static Route newRoute(String... args) { - if (args.length == 0) { - return Route.parse("default"); + if (cmd.hasOption('r')) { + route = Route.parse(cmd.getOptionValue('r').trim()); } - StringBuilder out = new StringBuilder(); - for (String arg : args) { - out.append(arg).append(' '); + if (cmd.hasOption('o')) { + String fileName = cmd.getOptionValue('o').trim(); + dumpStream = new FileOutputStream(new File(fileName)); + if (fileName.endsWith(".vespa")) { + dumpFormat = DumpFormat.VESPA; + } } - return Route.parse(out.toString()); + + return this; } + } diff --git a/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java b/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java index e55cd27f7da..dbb109aab0a 100644 --- a/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java +++ b/vespa_feed_perf/src/main/java/com/yahoo/vespa/feed/perf/SimpleFeeder.java @@ -2,12 +2,23 @@ package com.yahoo.vespa.feed.perf; import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentTypeManager; +import com.yahoo.document.DocumentUpdate; +import com.yahoo.document.json.JsonFeedReader; +import com.yahoo.document.json.JsonWriter; +import com.yahoo.document.serialization.DocumentDeserializer; +import com.yahoo.document.serialization.DocumentDeserializerFactory; +import com.yahoo.document.serialization.DocumentSerializer; +import com.yahoo.document.serialization.DocumentSerializerFactory; +import com.yahoo.document.serialization.DocumentWriter; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; +import com.yahoo.io.GrowableByteBuffer; import com.yahoo.messagebus.Error; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.MessageBusParams; @@ -19,11 +30,18 @@ import com.yahoo.messagebus.SourceSessionParams; import com.yahoo.messagebus.StaticThrottlePolicy; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; import com.yahoo.messagebus.routing.Route; +import com.yahoo.vespaxmlparser.FeedReader; +import com.yahoo.vespaxmlparser.FeedOperation; +import com.yahoo.vespaxmlparser.RemoveFeedOperation; import com.yahoo.vespaxmlparser.VespaXMLFeedReader; +import net.jpountz.xxhash.XXHashFactory; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.io.PrintStream; +import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.concurrent.ExecutorService; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; @@ -41,9 +59,7 @@ public class SimpleFeeder implements ReplyHandler { private final DocumentTypeManager docTypeMgr = new DocumentTypeManager(); private final InputStream in; private final PrintStream out; - private final PrintStream err; private final RPCMessageBus mbus; - private final Route route; private final SourceSession session; private final long startTime = System.currentTimeMillis(); private final AtomicReference<Throwable> failure = new AtomicReference<>(null); @@ -54,36 +70,260 @@ public class SimpleFeeder implements ReplyHandler { private long nextReport = startTime + REPORT_INTERVAL; private long sumLatency = 0; private final int numThreads; + private final Destination destination; public static void main(String[] args) throws Throwable { new SimpleFeeder(new FeederParams().parseArgs(args)).run().close(); } - SimpleFeeder(FeederParams params) { - this.in = params.getStdIn(); - this.out = params.getStdOut(); - this.err = params.getStdErr(); - this.route = params.getRoute(); - this.numThreads = params.getNumDispatchThreads(); - this.mbus = newMessageBus(docTypeMgr, params.getConfigId()); - this.session = newSession(mbus, this, params.isSerialTransferEnabled()); - this.docTypeMgr.configure(params.getConfigId()); - } - - private void sendOperation(VespaXMLFeedReader.Operation op) { - Message msg = newMessage(op); - if (msg == null) { - err.println("ignoring operation; " + op.getType()); - return; + private interface Destination { + void send(FeedOperation op); + void close() throws Exception; + } + + private static class MbusDestination implements Destination { + private final PrintStream err; + private final Route route; + private final SourceSession session; + private final AtomicReference<Throwable> failure; + MbusDestination(SourceSession session, Route route, AtomicReference<Throwable> failure, PrintStream err) { + this.route = route; + this.err = err; + this.session = session; + this.failure = failure; + } + public void send(FeedOperation op) { + Message msg = newMessage(op); + if (msg == null) { + err.println("ignoring operation; " + op.getType()); + return; + } + msg.setContext(System.currentTimeMillis()); + msg.setRoute(route); + try { + Error err = session.sendBlocking(msg).getError(); + if (err != null) { + failure.set(new IOException(err.toString())); + } + } catch (InterruptedException e) {} + } + public void close() throws Exception { + session.destroy(); + } + } + + private static class JsonDestination implements Destination { + private final OutputStream outputStream; + private final DocumentWriter writer; + private final AtomicLong numReplies; + private final AtomicReference<Throwable> failure; + private boolean isFirst = true; + JsonDestination(OutputStream outputStream, AtomicReference<Throwable> failure, AtomicLong numReplies) { + this.outputStream = outputStream; + writer = new JsonWriter(outputStream); + this.numReplies = numReplies; + this.failure = failure; + try { + outputStream.write('['); + outputStream.write('\n'); + } catch (IOException e) { + failure.set(e); + } + } + public void send(FeedOperation op) { + if (op.getType() == FeedOperation.Type.DOCUMENT) { + if (!isFirst) { + try { + outputStream.write(','); + outputStream.write('\n'); + } catch (IOException e) { + failure.set(e); + } + } else { + isFirst = false; + } + writer.write(op.getDocument()); + } + numReplies.incrementAndGet(); + } + public void close() throws Exception { + outputStream.write('\n'); + outputStream.write(']'); + outputStream.close(); + } + + } + + static private final int NONE = 0; + static private final int DOCUMENT = 1; + static private final int UPDATE = 2; + static private final int REMOVE = 3; + private static class VespaV1Destination implements Destination { + private final OutputStream outputStream; + GrowableByteBuffer buffer = new GrowableByteBuffer(16384); + ByteBuffer header = ByteBuffer.allocate(16); + private final AtomicLong numReplies; + private final AtomicReference<Throwable> failure; + VespaV1Destination(OutputStream outputStream, AtomicReference<Throwable> failure, AtomicLong numReplies) { + this.outputStream = outputStream; + this.numReplies = numReplies; + this.failure = failure; + try { + outputStream.write('V'); + outputStream.write('1'); + } catch (IOException e) { + failure.set(e); + } + } + public void send(FeedOperation op) { + DocumentSerializer writer = DocumentSerializerFactory.createHead(buffer); + int type = NONE; + if (op.getType() == FeedOperation.Type.DOCUMENT) { + writer.write(op.getDocument()); + type = DOCUMENT; + } else if (op.getType() == FeedOperation.Type.UPDATE) { + writer.write(op.getDocumentUpdate()); + type = UPDATE; + } else if (op.getType() == FeedOperation.Type.REMOVE) { + writer.write(op.getRemove()); + type = REMOVE; + } + int sz = buffer.position(); + long hash = hash(buffer.array(), 0, sz); + try { + + header.putInt(sz); + header.putInt(type); + header.putLong(hash); + outputStream.write(header.array(), 0, header.position()); + outputStream.write(buffer.array(), 0, buffer.position()); + header.clear(); + buffer.clear(); + } catch (IOException e) { + failure.set(e); + } + numReplies.incrementAndGet(); + } + public void close() throws Exception { + outputStream.close(); + } + static long hash(byte [] buf, int offset, int length) { + return XXHashFactory.fastestJavaInstance().hash64().hash(buf, offset, length, 0); + } + } + + private static int readExact(InputStream in, byte [] buf) throws IOException { + return in.readNBytes(buf, 0, buf.length); + } + + static class VespaV1FeedReader implements FeedReader { + private final InputStream in; + private final DocumentTypeManager mgr; + private final byte[] prefix = new byte[16]; + VespaV1FeedReader(InputStream in, DocumentTypeManager mgr) throws IOException { + this.in = in; + this.mgr = mgr; + byte [] header = new byte[2]; + int read = readExact(in, header); + if ((read != header.length) || (header[0] != 'V') || (header[1] != '1')) { + throw new IllegalArgumentException("Invalid Header " + Arrays.toString(header)); + } + } + + class LazyDocumentOperation extends FeedOperation { + private final DocumentDeserializer deserializer; + LazyDocumentOperation(DocumentDeserializer deserializer) { + super(Type.DOCUMENT); + this.deserializer = deserializer; + } + + @Override + public Document getDocument() { + return new Document(deserializer); + } } - msg.setContext(System.currentTimeMillis()); - msg.setRoute(route); - try { - Error err = session.sendBlocking(msg).getError(); - if (err != null) { - failure.set(new IOException(err.toString())); + class LazyUpdateOperation extends FeedOperation { + private final DocumentDeserializer deserializer; + LazyUpdateOperation(DocumentDeserializer deserializer) { + super(Type.UPDATE); + this.deserializer = deserializer; + } + + @Override + public DocumentUpdate getDocumentUpdate() { + return new DocumentUpdate(deserializer); + } + } + @Override + public FeedOperation read() throws Exception { + int read = readExact(in, prefix); + if (read != prefix.length) { + return FeedOperation.INVALID; + } + ByteBuffer header = ByteBuffer.wrap(prefix); + int sz = header.getInt(); + int type = header.getInt(); + long hash = header.getLong(); + byte [] blob = new byte[sz]; + read = readExact(in, blob); + if (read != blob.length) { + throw new IllegalArgumentException("Underflow, failed reading " + blob.length + "bytes. Got " + read); + } + long computedHash = VespaV1Destination.hash(blob, 0, blob.length); + if (computedHash != hash) { + throw new IllegalArgumentException("Hash mismatch, expected " + hash + ", got " + computedHash); } - } catch (InterruptedException e) {} + DocumentDeserializer deser = DocumentDeserializerFactory.createHead(mgr, GrowableByteBuffer.wrap(blob)); + if (type == DOCUMENT) { + return new LazyDocumentOperation(deser); + } else if (type == UPDATE) { + return new LazyUpdateOperation(deser); + } else if (type == REMOVE) { + return new RemoveFeedOperation(new DocumentId(deser)); + } else { + throw new IllegalArgumentException("Unknown operation " + type); + } + } + } + + private Destination createDumper(FeederParams params) { + if (params.getDumpFormat() == FeederParams.DumpFormat.VESPA) { + return new VespaV1Destination(params.getDumpStream(), failure, numReplies); + } + return new JsonDestination(params.getDumpStream(), failure, numReplies); + } + SimpleFeeder(FeederParams params) { + in = params.getStdIn(); + out = params.getStdOut(); + numThreads = params.getNumDispatchThreads(); + mbus = newMessageBus(docTypeMgr, params.getConfigId()); + session = newSession(mbus, this, params.isSerialTransferEnabled()); + docTypeMgr.configure(params.getConfigId()); + destination = (params.getDumpStream() != null) + ? createDumper(params) + : new MbusDestination(session, params.getRoute(), failure, params.getStdErr()); + } + + private void sendOperation(FeedOperation op) { + destination.send(op); + } + + SourceSession getSourceSession() { return session; } + private FeedReader createFeedReader() throws Exception { + in.mark(8); + byte [] b = new byte[2]; + int numRead = readExact(in, b); + in.reset(); + if (numRead != b.length) { + throw new IllegalArgumentException("Need to read " + b.length + " bytes to detect format. Got " + numRead + " bytes."); + } + if (b[0] == '[') { + return new JsonFeedReader(in, docTypeMgr); + } else if ((b[0] == 'V') && (b[1] == '1')) { + return new VespaV1FeedReader(in, docTypeMgr); + } else { + return new VespaXMLFeedReader(in, docTypeMgr); + } } SimpleFeeder run() throws Throwable { @@ -93,14 +333,13 @@ public class SimpleFeeder implements ReplyHandler { ThreadFactoryFactory.getDaemonThreadFactory("perf-feeder"), new ThreadPoolExecutor.CallerRunsPolicy()) : null; - VespaXMLFeedReader reader = new VespaXMLFeedReader(in, docTypeMgr); + FeedReader reader = createFeedReader(); printHeader(); long numMessages = 0; while (failure.get() == null) { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - reader.read(op); - if (op.getType() == VespaXMLFeedReader.OperationType.INVALID) { + FeedOperation op = reader.read(); + if (op.getType() == FeedOperation.Type.INVALID) { break; } if (executor != null) { @@ -120,12 +359,12 @@ public class SimpleFeeder implements ReplyHandler { return this; } - void close() { - session.destroy(); + void close() throws Exception { + destination.close(); mbus.destroy(); } - private Message newMessage(VespaXMLFeedReader.Operation op) { + private static Message newMessage(FeedOperation op) { switch (op.getType()) { case DOCUMENT: { PutDocumentMessage message = new PutDocumentMessage(new DocumentPut(op.getDocument())); diff --git a/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/FeederParamsTest.java b/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/FeederParamsTest.java index f08e494a717..ab1eb27e416 100644 --- a/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/FeederParamsTest.java +++ b/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/FeederParamsTest.java @@ -4,14 +4,19 @@ package com.yahoo.vespa.feed.perf; import com.yahoo.messagebus.routing.Route; import org.apache.commons.cli.ParseException; import org.junit.Test; -import org.mockito.Mockito; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; @@ -19,28 +24,26 @@ import static org.junit.Assert.assertTrue; * @author Simon Thoresen Hult */ public class FeederParamsTest { + static final String TESTFILE_JSON = "test.json"; + static final String TESTFILE_VESPA = "test.vespa"; + static final String TESTFILE_UNKNOWN = "test.xyz"; @Test public void requireThatAccessorsWork() { FeederParams params = new FeederParams(); - InputStream stdIn = Mockito.mock(InputStream.class); + InputStream stdIn = new ByteArrayInputStream(new byte[1]); params.setStdIn(stdIn); assertSame(stdIn, params.getStdIn()); - PrintStream stdErr = Mockito.mock(PrintStream.class); + PrintStream stdErr = new PrintStream(new ByteArrayOutputStream()); params.setStdErr(stdErr); assertSame(stdErr, params.getStdErr()); - PrintStream stdOut = Mockito.mock(PrintStream.class); + PrintStream stdOut = new PrintStream(new ByteArrayOutputStream()); params.setStdOut(stdOut); assertSame(stdOut, params.getStdOut()); - Route route = Route.parse("my_route"); - params.setRoute(route); - assertEquals(route, params.getRoute()); - assertNotSame(route, params.getRoute()); - params.setConfigId("my_config_id"); assertEquals("my_config_id", params.getConfigId()); @@ -62,7 +65,7 @@ public class FeederParamsTest { } @Test - public void requireThatSerialTransferOptionIsParsed() throws ParseException { + public void requireThatSerialTransferOptionIsParsed() throws ParseException, FileNotFoundException { assertTrue(new FeederParams().parseArgs("-s").isSerialTransferEnabled()); assertTrue(new FeederParams().parseArgs("foo", "-s").isSerialTransferEnabled()); assertTrue(new FeederParams().parseArgs("-s", "foo").isSerialTransferEnabled()); @@ -72,23 +75,45 @@ public class FeederParamsTest { } @Test - public void requireThatArgumentsAreParsedAsRoute() throws ParseException { - assertEquals(Route.parse("foo bar"), new FeederParams().parseArgs("foo", "bar").getRoute()); - assertEquals(Route.parse("foo bar"), new FeederParams().parseArgs("-s", "foo", "bar").getRoute()); - assertEquals(Route.parse("foo bar"), new FeederParams().parseArgs("foo", "-s", "bar").getRoute()); - assertEquals(Route.parse("foo bar"), new FeederParams().parseArgs("foo", "bar", "-s").getRoute()); + public void requireThatArgumentsAreParsedAsRoute() throws ParseException, FileNotFoundException { + assertEquals(Route.parse("foo bar"), new FeederParams().parseArgs("-r foo bar").getRoute()); + assertEquals(Route.parse("foo bar"), new FeederParams().parseArgs("--route","foo bar").getRoute()); } @Test - public void requireThatRouteIsAnOptionalArgument() throws ParseException { + public void requireThatRouteIsAnOptionalArgument() throws ParseException, FileNotFoundException { assertEquals(Route.parse("default"), new FeederParams().parseArgs().getRoute()); assertEquals(Route.parse("default"), new FeederParams().parseArgs("-s").getRoute()); } @Test - public void requireThatNumThreadsAreParsed() throws ParseException { + public void requireThatNumThreadsAreParsed() throws ParseException, FileNotFoundException { assertEquals(1, new FeederParams().getNumDispatchThreads()); assertEquals(17, new FeederParams().parseArgs("-n 17").getNumDispatchThreads()); } + @Test + public void requireThatDumpStreamAreParsed() throws ParseException, IOException { + assertNull(new FeederParams().getDumpStream()); + + FeederParams p = new FeederParams().parseArgs("-o " + TESTFILE_JSON); + assertNotNull(p.getDumpStream()); + assertEquals(FeederParams.DumpFormat.JSON, p.getDumpFormat()); + p.getDumpStream().close(); + + p = new FeederParams().parseArgs("-o " + TESTFILE_VESPA); + assertNotNull(p.getDumpStream()); + assertEquals(FeederParams.DumpFormat.VESPA, p.getDumpFormat()); + p.getDumpStream().close(); + + p = new FeederParams().parseArgs("-o " + TESTFILE_UNKNOWN); + assertNotNull(p.getDumpStream()); + assertEquals(FeederParams.DumpFormat.JSON, p.getDumpFormat()); + p.getDumpStream().close(); + + assertTrue(new File(TESTFILE_JSON).delete()); + assertTrue(new File(TESTFILE_VESPA).delete()); + assertTrue(new File(TESTFILE_UNKNOWN).delete()); + } + } diff --git a/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/SimpleFeederTest.java b/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/SimpleFeederTest.java index 89e52eeee19..1c2cac3bcee 100644 --- a/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/SimpleFeederTest.java +++ b/vespa_feed_perf/src/test/java/com/yahoo/vespa/feed/perf/SimpleFeederTest.java @@ -11,7 +11,6 @@ import com.yahoo.messagebus.ErrorCode; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.MessageHandler; import com.yahoo.messagebus.Reply; -import com.yahoo.messagebus.SourceSession; import com.yahoo.messagebus.StaticThrottlePolicy; import com.yahoo.messagebus.ThrottlePolicy; import org.junit.Test; @@ -19,6 +18,7 @@ import org.junit.Test; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.PrintStream; import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; @@ -36,7 +36,7 @@ public class SimpleFeederTest { private static final String CONFIG_DIR = "target/test-classes/"; @Test - public void requireThatFeederWorks() throws Throwable { + public void requireThatXMLFeederWorks() throws Throwable { assertFeed("<vespafeed>" + " <document documenttype='simple' documentid='doc:scheme:0'>" + " <my_str>foo</my_str>" + @@ -61,6 +61,137 @@ public class SimpleFeederTest { } @Test + public void requireThatXML2JsonFeederWorks() throws Throwable { + ByteArrayOutputStream dump = new ByteArrayOutputStream(); + assertFeed(new FeederParams().setDumpStream(dump), + "<vespafeed>" + + " <document documenttype='simple' documentid='id:simple:simple::0'>" + + " <my_str>foo</my_str>" + + " </document>" + + " <update documenttype='simple' documentid='id:simple:simple::1'>" + + " <assign field='my_str'>bar</assign>" + + " </update>" + + " <remove documenttype='simple' documentid='id:simple:simple::2'/>" + + "</vespafeed>", + new MessageHandler() { + + @Override + public void handleMessage(Message msg) { + Reply reply = ((DocumentMessage)msg).createReply(); + reply.swapState(msg); + reply.popHandler().handleReply(reply); + } + }, + "", + "(.+\n)+" + + "\\s*\\d+,\\s*3,.+\n"); + assertEquals(58, dump.size()); + assertEquals("[\n{\"id\":\"id:simple:simple::0\",\"fields\":{\"my_str\":\"foo\"}}\n]", dump.toString()); + } + + @Test + public void requireThatDualPutXML2JsonFeederWorks() throws Throwable { + ByteArrayOutputStream dump = new ByteArrayOutputStream(); + assertFeed(new FeederParams().setDumpStream(dump), + "<vespafeed>" + + " <document documenttype='simple' documentid='id:simple:simple::0'>" + + " <my_str>foo</my_str>" + + " </document>" + + " <document documenttype='simple' documentid='id:simple:simple::1'>" + + " <my_str>bar</my_str>" + + " </document>" + + " <remove documenttype='simple' documentid='id:simple:simple::2'/>" + + "</vespafeed>", + new MessageHandler() { + + @Override + public void handleMessage(Message msg) { + Reply reply = ((DocumentMessage)msg).createReply(); + reply.swapState(msg); + reply.popHandler().handleReply(reply); + } + }, + "", + "(.+\n)+" + + "\\s*\\d+,\\s*3,.+\n"); + assertEquals(115, dump.size()); + assertEquals("[\n{\"id\":\"id:simple:simple::0\",\"fields\":{\"my_str\":\"foo\"}},\n {\"id\":\"id:simple:simple::1\",\"fields\":{\"my_str\":\"bar\"}}\n]", dump.toString()); + assertFeed(dump.toString(), + new MessageHandler() { + @Override + public void handleMessage(Message msg) { + Reply reply = ((DocumentMessage)msg).createReply(); + reply.swapState(msg); + reply.popHandler().handleReply(reply); + } + }, + "", + "(.+\n)+" + + "\\s*\\d+,\\s*2,.+\n"); + } + + @Test + public void requireThatDualPutXML2VespaFeederWorks() throws Throwable { + ByteArrayOutputStream dump = new ByteArrayOutputStream(); + assertFeed(new FeederParams().setDumpStream(dump).setDumpFormat(FeederParams.DumpFormat.VESPA), + "<vespafeed>" + + " <document documenttype='simple' documentid='id:simple:simple::0'>" + + " <my_str>foo</my_str>" + + " </document>" + + " <document documenttype='simple' documentid='id:simple:simple::1'>" + + " <my_str>bar</my_str>" + + " </document>" + + " <remove documenttype='simple' documentid='id:simple:simple::2'/>" + + "</vespafeed>", + new MessageHandler() { + + @Override + public void handleMessage(Message msg) { + Reply reply = ((DocumentMessage)msg).createReply(); + reply.swapState(msg); + reply.popHandler().handleReply(reply); + } + }, + "", + "(.+\n)+" + + "\\s*\\d+,\\s*3,.+\n"); + assertEquals(178, dump.size()); + assertFeed(new ByteArrayInputStream(dump.toByteArray()), + new MessageHandler() { + @Override + public void handleMessage(Message msg) { + Reply reply = ((DocumentMessage)msg).createReply(); + reply.swapState(msg); + reply.popHandler().handleReply(reply); + } + }, + "", + "(.+\n)+" + + "\\s*\\d+,\\s*3,.+\n"); + } + + @Test + public void requireThatJsonFeederWorks() throws Throwable { + assertFeed("[" + + " { \"put\": \"id:simple:simple::0\", \"fields\": { \"my_str\":\"foo\"}}," + + " { \"update\": \"id:simple:simple::1\", \"fields\": { \"my_str\": { \"assign\":\"bar\"}}}," + + " { \"remove\": \"id:simple:simple::2\"}" + + "]", + new MessageHandler() { + + @Override + public void handleMessage(Message msg) { + Reply reply = ((DocumentMessage)msg).createReply(); + reply.swapState(msg); + reply.popHandler().handleReply(reply); + } + }, + "", + "(.+\n)+" + + "\\s*\\d+,\\s*3,.+\n"); + } + + @Test public void requireThatParseFailuresThrowInMainThread() throws Throwable { TestDriver driver = new TestDriver(new FeederParams(), "<vespafeed>" + @@ -84,7 +215,7 @@ public class SimpleFeederTest { " <document documenttype='simple' documentid='doc:scheme:0'/>" + "</vespafeed>", null); - getSourceSession(driver).close(); + driver.feeder.getSourceSession().close(); try { driver.run(); fail(); @@ -135,12 +266,8 @@ public class SimpleFeederTest { assertTrue(driver.close()); } - private static SourceSession getSourceSession(TestDriver driver) { - return (SourceSession)getField(driver.feeder, "session"); - } - private static ThrottlePolicy getThrottlePolicy(TestDriver driver) { - return (ThrottlePolicy)getField(getSourceSession(driver), "throttlePolicy"); + return (ThrottlePolicy)getField(driver.feeder.getSourceSession(), "throttlePolicy"); } private static Object getField(Object obj, String fieldName) { @@ -153,9 +280,17 @@ public class SimpleFeederTest { } } - private static void assertFeed(String in, MessageHandler validator, String expectedErr, String expectedOut) - throws Throwable { - TestDriver driver = new TestDriver(new FeederParams(), in, validator); + private static void assertFeed(String in, MessageHandler validator, String expectedErr, String expectedOut) throws Throwable { + assertFeed(new FeederParams(), in, validator, expectedErr, expectedOut); + } + private static void assertFeed(InputStream in, MessageHandler validator, String expectedErr, String expectedOut) throws Throwable { + assertFeed(new FeederParams(), in, validator, expectedErr, expectedOut); + } + private static void assertFeed(FeederParams params, String in, MessageHandler validator, String expectedErr, String expectedOut) throws Throwable { + assertFeed(params, new ByteArrayInputStream(in.getBytes(StandardCharsets.UTF_8)), validator, expectedErr, expectedOut); + } + private static void assertFeed(FeederParams params, InputStream in, MessageHandler validator, String expectedErr, String expectedOut) throws Throwable { + TestDriver driver = new TestDriver(params, in, validator); driver.run(); assertMatches(expectedErr, new String(driver.err.toByteArray(), StandardCharsets.UTF_8)); assertMatches(expectedOut, new String(driver.out.toByteArray(), StandardCharsets.UTF_8)); @@ -175,12 +310,14 @@ public class SimpleFeederTest { final SimpleFeeder feeder; final SimpleServer server; - public TestDriver(FeederParams params, String in, MessageHandler validator) - throws IOException, ListenFailedException { + TestDriver(FeederParams params, String in, MessageHandler validator) throws IOException, ListenFailedException { + this(params, new ByteArrayInputStream(in.getBytes(StandardCharsets.UTF_8)), validator); + } + TestDriver(FeederParams params, InputStream in, MessageHandler validator) throws IOException, ListenFailedException { server = new SimpleServer(CONFIG_DIR, validator); feeder = new SimpleFeeder(params.setConfigId("dir:" + CONFIG_DIR) .setStdErr(new PrintStream(err)) - .setStdIn(new ByteArrayInputStream(in.getBytes(StandardCharsets.UTF_8))) + .setStdIn(in) .setStdOut(new PrintStream(out))); } @@ -188,7 +325,7 @@ public class SimpleFeederTest { feeder.run(); } - boolean close() { + boolean close() throws Exception { feeder.close(); server.close(); return true; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java index cfa77455f41..a6fdcb10a00 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandler.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document.restapi; -import com.yahoo.vespaxmlparser.VespaXMLFeedReader; +import com.yahoo.vespaxmlparser.FeedOperation; import java.util.Optional; @@ -90,9 +90,9 @@ public interface OperationHandler { VisitResult visit(RestUri restUri, String documentSelection, VisitOptions options) throws RestApiException; - void put(RestUri restUri, VespaXMLFeedReader.Operation data, Optional<String> route) throws RestApiException; + void put(RestUri restUri, FeedOperation data, Optional<String> route) throws RestApiException; - void update(RestUri restUri, VespaXMLFeedReader.Operation data, Optional<String> route) throws RestApiException; + void update(RestUri restUri, FeedOperation data, Optional<String> route) throws RestApiException; void delete(RestUri restUri, String condition, Optional<String> route) throws RestApiException; diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java index bfc4a611a5e..b9bbe4f792e 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java @@ -25,6 +25,7 @@ import com.yahoo.messagebus.StaticThrottlePolicy; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.vdslib.VisitorOrdering; import com.yahoo.vespaclient.ClusterDef; +import com.yahoo.vespaxmlparser.FeedOperation; import com.yahoo.vespaxmlparser.VespaXMLFeedReader; import com.yahoo.yolean.concurrent.ConcurrentResourcePool; import com.yahoo.yolean.concurrent.ResourceFactory; @@ -201,7 +202,7 @@ public class OperationHandlerImpl implements OperationHandler { } @Override - public void put(RestUri restUri, VespaXMLFeedReader.Operation data, Optional<String> route) throws RestApiException { + public void put(RestUri restUri, FeedOperation data, Optional<String> route) throws RestApiException { SyncSession syncSession = syncSessions.alloc(); Response response; try { @@ -225,7 +226,7 @@ public class OperationHandlerImpl implements OperationHandler { } @Override - public void update(RestUri restUri, VespaXMLFeedReader.Operation data, Optional<String> route) throws RestApiException { + public void update(RestUri restUri, FeedOperation data, Optional<String> route) throws RestApiException { SyncSession syncSession = syncSessions.alloc(); Response response; try { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java index 880ea2102ab..873b0569553 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java @@ -33,6 +33,8 @@ import com.yahoo.vespa.config.content.LoadTypeConfig; import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; import com.yahoo.vespaclient.ClusterDef; import com.yahoo.vespaclient.ClusterList; +import com.yahoo.vespaxmlparser.DocumentFeedOperation; +import com.yahoo.vespaxmlparser.FeedOperation; import com.yahoo.vespaxmlparser.VespaXMLFeedReader; import java.io.IOException; @@ -236,23 +238,21 @@ public class RestApi extends LoggingRequestHandler { return new Response(200, resultJson, Optional.of(restUri)); } - private VespaXMLFeedReader.Operation createPutOperation(HttpRequest request, String id, String condition) { - final VespaXMLFeedReader.Operation operationPut = - singleDocumentParser.parsePut(request.getData(), id); + private FeedOperation createPutOperation(HttpRequest request, String id, String condition) { + FeedOperation put = singleDocumentParser.parsePut(request.getData(), id); if (condition != null && ! condition.isEmpty()) { - operationPut.setCondition(new TestAndSetCondition(condition)); + return new DocumentFeedOperation(put.getDocument(), new TestAndSetCondition(condition)); } - return operationPut; + return put; } - private VespaXMLFeedReader.Operation createUpdateOperation(HttpRequest request, String id, String condition, Optional<Boolean> create) { - final VespaXMLFeedReader.Operation operationUpdate = - singleDocumentParser.parseUpdate(request.getData(), id); + private FeedOperation createUpdateOperation(HttpRequest request, String id, String condition, Optional<Boolean> create) { + FeedOperation update = singleDocumentParser.parseUpdate(request.getData(), id); if (condition != null && ! condition.isEmpty()) { - operationUpdate.getDocumentUpdate().setCondition(new TestAndSetCondition(condition)); + update.getDocumentUpdate().setCondition(new TestAndSetCondition(condition)); } - create.ifPresent(c -> operationUpdate.getDocumentUpdate().setCreateIfNonExistent(c)); - return operationUpdate; + create.ifPresent(c -> update.getDocumentUpdate().setCreateIfNonExistent(c)); + return update; } private HttpResponse handleGet(RestUri restUri, HttpRequest request) throws RestApiException { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java index ac2c9515d71..947fcb637fb 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java @@ -17,6 +17,7 @@ import com.yahoo.net.HostName; import com.yahoo.vespa.http.client.core.ErrorCode; import com.yahoo.vespa.http.client.core.Headers; import com.yahoo.vespa.http.client.core.OperationStatus; +import com.yahoo.vespaxmlparser.FeedOperation; import com.yahoo.vespaxmlparser.VespaXMLFeedReader; import com.yahoo.yolean.Exceptions; @@ -274,7 +275,7 @@ class ClientFeederV3 { /** Returns the next message in the stream, or null if none */ protected DocumentOperationMessageV3 getNextMessage( String operationId, InputStream requestInputStream, FeederSettings settings) throws Exception { - VespaXMLFeedReader.Operation operation = streamReaderV3.getNextOperation(requestInputStream, settings); + FeedOperation operation = streamReaderV3.getNextOperation(requestInputStream, settings); // This is a bit hard to set up while testing, so we accept that things are not perfect. if (sourceSession.getResource().session() != null) { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java index ca2fdd6b329..6df6bba5a59 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/DocumentOperationMessageV3.java @@ -9,11 +9,7 @@ import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; import com.yahoo.jdisc.Metric; import com.yahoo.messagebus.Message; -import com.yahoo.messagebus.routing.ErrorDirective; -import com.yahoo.messagebus.routing.Hop; -import com.yahoo.messagebus.routing.Route; -import com.yahoo.vespaxmlparser.VespaXMLFeedReader; -import com.yahoo.yolean.Exceptions; +import com.yahoo.vespaxmlparser.FeedOperation; /** * Keeps an operation with its message. @@ -40,18 +36,7 @@ class DocumentOperationMessageV3 { return operationId; } - static DocumentOperationMessageV3 newErrorMessage(String operationId, Exception exception) { - Message feedErrorMessageV3 = new FeedErrorMessage(operationId); - DocumentOperationMessageV3 msg = new DocumentOperationMessageV3(operationId, feedErrorMessageV3); - Hop hop = new Hop(); - hop.addDirective(new ErrorDirective(Exceptions.toMessageString(exception))); - Route route = new Route(); - route.addHop(hop); - feedErrorMessageV3.setRoute(route); - return msg; - } - - static DocumentOperationMessageV3 newUpdateMessage(VespaXMLFeedReader.Operation op, String operationId) { + private static DocumentOperationMessageV3 newUpdateMessage(FeedOperation op, String operationId) { DocumentUpdate update = op.getDocumentUpdate(); update.setCondition(op.getCondition()); Message msg = new UpdateDocumentMessage(update); @@ -60,7 +45,7 @@ class DocumentOperationMessageV3 { return new DocumentOperationMessageV3(id, msg); } - static DocumentOperationMessageV3 newRemoveMessage(VespaXMLFeedReader.Operation op, String operationId) { + static DocumentOperationMessageV3 newRemoveMessage(FeedOperation op, String operationId) { DocumentRemove remove = new DocumentRemove(op.getRemove()); remove.setCondition(op.getCondition()); Message msg = new RemoveDocumentMessage(remove); @@ -69,7 +54,7 @@ class DocumentOperationMessageV3 { return new DocumentOperationMessageV3(id, msg); } - static DocumentOperationMessageV3 newPutMessage(VespaXMLFeedReader.Operation op, String operationId) { + private static DocumentOperationMessageV3 newPutMessage(FeedOperation op, String operationId) { DocumentPut put = new DocumentPut(op.getDocument()); put.setCondition(op.getCondition()); Message msg = new PutDocumentMessage(put); @@ -78,7 +63,7 @@ class DocumentOperationMessageV3 { return new DocumentOperationMessageV3(id, msg); } - static DocumentOperationMessageV3 create(VespaXMLFeedReader.Operation operation, String operationId, Metric metric) { + static DocumentOperationMessageV3 create(FeedOperation operation, String operationId, Metric metric) { switch (operation.getType()) { case DOCUMENT: metric.add(MetricNames.NUM_PUTS, 1, null /*metricContext*/); diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java index 00290c4fb09..69f810ad4c7 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/StreamReaderV3.java @@ -5,8 +5,8 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.document.DocumentTypeManager; import com.yahoo.vespa.http.client.core.Encoder; import com.yahoo.vespa.http.server.util.ByteLimitedInputStream; +import com.yahoo.vespaxmlparser.FeedOperation; import com.yahoo.vespaxmlparser.FeedReader; -import com.yahoo.vespaxmlparser.VespaXMLFeedReader; import java.io.IOException; import java.io.InputStream; @@ -30,15 +30,14 @@ public class StreamReaderV3 { this.docTypeManager = docTypeManager; } - public VespaXMLFeedReader.Operation getNextOperation( - InputStream requestInputStream, FeederSettings settings) throws Exception { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); + public FeedOperation getNextOperation(InputStream requestInputStream, FeederSettings settings) throws Exception { + FeedOperation op = null; int length = readByteLength(requestInputStream); try (InputStream limitedInputStream = new ByteLimitedInputStream(requestInputStream, length)){ FeedReader reader = feedReaderFactory.createReader(limitedInputStream, docTypeManager, settings.dataFormat); - reader.read(op); + op = reader.read(); } return op; } diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java index fbaa7f86bd0..1e982c7b700 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/MockedOperationHandler.java @@ -5,6 +5,7 @@ import com.yahoo.document.restapi.OperationHandler; import com.yahoo.document.restapi.Response; import com.yahoo.document.restapi.RestApiException; import com.yahoo.document.restapi.RestUri; +import com.yahoo.vespaxmlparser.FeedOperation; import com.yahoo.vespaxmlparser.VespaXMLFeedReader; import java.util.Optional; @@ -31,13 +32,13 @@ public class MockedOperationHandler implements OperationHandler { @Override @SuppressWarnings("deprecation") - public void put(RestUri restUri, VespaXMLFeedReader.Operation data, Optional<String> route) throws RestApiException { + public void put(RestUri restUri, FeedOperation data, Optional<String> route) throws RestApiException { log.append("PUT: " + data.getDocument().getId()); log.append(data.getDocument().getBody().toString()); } @Override - public void update(RestUri restUri, VespaXMLFeedReader.Operation data, Optional<String> route) throws RestApiException { + public void update(RestUri restUri, FeedOperation data, Optional<String> route) throws RestApiException { log.append("UPDATE: " + data.getDocumentUpdate().getId()); log.append(data.getDocumentUpdate().fieldUpdates().toString()); if (data.getDocumentUpdate().getCreateIfNonExistent()) { diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java index 3c758114ecf..fd9655fb838 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/FeedTesterV3.java @@ -11,7 +11,6 @@ import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; import com.yahoo.documentapi.metrics.DocumentApiMetrics; -import com.yahoo.feedhandler.NullFeedMetric; import com.yahoo.jdisc.ReferencedResource; import com.yahoo.messagebus.SourceSessionParams; import com.yahoo.messagebus.shared.SharedSourceSession; diff --git a/vespaclient-core/src/main/java/com/yahoo/feedhandler/NullFeedMetric.java b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/NullFeedMetric.java index 2940c0fcc44..4777c6c7b99 100644 --- a/vespaclient-core/src/main/java/com/yahoo/feedhandler/NullFeedMetric.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/feedhandler/v3/NullFeedMetric.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.feedhandler; +package com.yahoo.feedhandler.v3; import com.yahoo.jdisc.Metric; import java.util.Map; diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V3CongestionTestCase.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V3CongestionTestCase.java index 106dd71b83c..04b66480a82 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V3CongestionTestCase.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespa/http/server/V3CongestionTestCase.java @@ -18,8 +18,8 @@ import com.yahoo.messagebus.shared.SharedMessageBus; import com.yahoo.messagebus.shared.SharedSourceSession; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.vespa.http.client.core.Headers; +import com.yahoo.vespaxmlparser.FeedOperation; import com.yahoo.vespaxmlparser.MockFeedReaderFactory; -import com.yahoo.vespaxmlparser.VespaXMLFeedReader; import org.junit.Before; import org.junit.Test; @@ -45,8 +45,7 @@ public class V3CongestionTestCase { ClientFeederWithMocks(ReferencedResource<SharedSourceSession> sourceSession, FeedReaderFactory feedReaderFactory, DocumentTypeManager docTypeManager, String clientId, Metric metric, ReplyHandler feedReplyHandler, AtomicInteger threadsAvailableForFeeding) { super(sourceSession, feedReaderFactory, docTypeManager, clientId, metric, feedReplyHandler, threadsAvailableForFeeding); // The operation to return from the client feeder. - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - docOp = DocumentOperationMessageV3.newRemoveMessage(op, "operation id"); + docOp = DocumentOperationMessageV3.newRemoveMessage(FeedOperation.INVALID, "operation id"); } diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockReader.java b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockReader.java index b399b1197da..eabbb2dab20 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockReader.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/vespaxmlparser/MockReader.java @@ -7,7 +7,6 @@ import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentUpdate; import com.yahoo.vespa.http.server.MetaStream; import com.yahoo.vespa.http.server.util.ByteLimitedInputStream; -import com.yahoo.vespaxmlparser.VespaXMLFeedReader.Operation; import java.io.InputStream; import java.lang.reflect.Field; @@ -45,9 +44,9 @@ public class MockReader implements FeedReader { } @Override - public void read(Operation operation) throws Exception { + public FeedOperation read() throws Exception { if (finished) { - return; + return FeedOperation.INVALID; } byte whatToDo = stream.getNextOperation(); @@ -55,19 +54,14 @@ public class MockReader implements FeedReader { DocumentType docType = new DocumentType("banana"); switch (whatToDo) { case 0: - finished = true; - break; + return FeedOperation.INVALID; case 1: - Document doc = new Document(docType, id); - operation.setDocument(doc); - break; + return new DocumentFeedOperation(new Document(docType, id)); case 2: - operation.setRemove(id); - break; + return new RemoveFeedOperation(id); case 3: - operation.setDocumentUpdate(new DocumentUpdate(docType, id)); - break; - case 4: + return new DocumentUpdateFeedOperation(new DocumentUpdate(docType, id)); + default: throw new RuntimeException("boom"); } } diff --git a/vespaclient-core/pom.xml b/vespaclient-core/pom.xml index 693af71e0bc..98d324b062e 100644 --- a/vespaclient-core/pom.xml +++ b/vespaclient-core/pom.xml @@ -19,11 +19,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>metrics</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>container-dev</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/vespaclient-core/src/main/java/com/yahoo/clientmetrics/ClientMetrics.java b/vespaclient-core/src/main/java/com/yahoo/clientmetrics/ClientMetrics.java index a94d006675f..ff54f3d4063 100755 --- a/vespaclient-core/src/main/java/com/yahoo/clientmetrics/ClientMetrics.java +++ b/vespaclient-core/src/main/java/com/yahoo/clientmetrics/ClientMetrics.java @@ -1,39 +1,21 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.clientmetrics; -import com.yahoo.messagebus.Reply; -import com.yahoo.metrics.*; -import com.yahoo.text.XMLWriter; - -import java.io.IOException; -import java.io.OutputStream; -import java.io.PrintStream; -import java.text.DecimalFormat; -import java.text.NumberFormat; -import java.util.ArrayList; -import java.util.List; +import java.util.HashMap; +import java.util.Map; /** * @author thomasg */ public class ClientMetrics { - MetricSet topSet; - SumMetric sum; - List<String> routes = new ArrayList<String>(); + Map<String, RouteMetricSet> routes = new HashMap<>(); public ClientMetrics() { - topSet = new SimpleMetricSet("routes", "", "", null); - sum = new SumMetric("total", "", "Messages sent to all routes", topSet); - } - public MetricSet getMetricSet() { - return topSet; } public void addRouteMetricSet(RouteMetricSet metric) { - topSet.registerMetric(metric); - sum.addMetricToSum(metric); - routes.add(metric.getRoute()); + routes.put(metric.getRoute(), metric); } } diff --git a/vespaclient-core/src/main/java/com/yahoo/clientmetrics/MessageTypeMetricSet.java b/vespaclient-core/src/main/java/com/yahoo/clientmetrics/MessageTypeMetricSet.java index 731a08513e9..45b1a11b4af 100644 --- a/vespaclient-core/src/main/java/com/yahoo/clientmetrics/MessageTypeMetricSet.java +++ b/vespaclient-core/src/main/java/com/yahoo/clientmetrics/MessageTypeMetricSet.java @@ -6,61 +6,27 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.messagebus.Reply; import com.yahoo.concurrent.SystemTimer; import com.yahoo.messagebus.Error; -import com.yahoo.metrics.CountMetric; -import com.yahoo.metrics.Metric; -import com.yahoo.metrics.MetricSet; -import com.yahoo.metrics.SimpleMetricSet; -import com.yahoo.metrics.SumMetric; -import com.yahoo.metrics.ValueMetric; +import java.util.HashMap; +import java.util.Map; import java.util.stream.Stream; /** * @author thomasg */ -public class MessageTypeMetricSet extends MetricSet { - ValueMetric<Long> latency; - CountMetric count; - CountMetric ignored; - - SumMetric errorSum; - MetricSet errors; - String msgName; - - class ErrorMetric extends CountMetric { - ErrorMetric(String name, MetricSet owner) { - super(name, "", "Number of errors of type " + name, owner); - } - - ErrorMetric(ErrorMetric other, CopyType copyType, MetricSet owner) { - super(other, copyType, owner); - } - - @Override - public String getXMLTag() { - return "error"; - } - - @Override - public Metric clone(CopyType type, MetricSet owner, boolean includeUnused) { - return new ErrorMetric(this, type, owner); - } - - } - - public MessageTypeMetricSet(String msgName, MetricSet owner) { - super(msgName.toLowerCase(), "", "", owner); +public class MessageTypeMetricSet { + public long latency_total; + public long latency_min = Long.MAX_VALUE; + public long latency_max = Long.MIN_VALUE; + public long count = 0; + public long ignored = 0; + public long errorCount = 0; + private final Map<String, Long> errorCounts = new HashMap<>(); + + private final String msgName; + + public MessageTypeMetricSet(String msgName) { this.msgName = msgName; - latency = new ValueMetric<Long>("latency", "", "Latency (in ms)", this).averageMetric().createAverageOnJoin(); - count = new CountMetric("count", "", "Number received", this); - ignored = new CountMetric("ignored", "", "Number ignored due to no matching document routing selectors", this); - errors = new SimpleMetricSet("errors", "", "The errors returned", this); - errorSum = new SumMetric("total", "", "Total number of errors", errors); - } - - public MessageTypeMetricSet(MessageTypeMetricSet source, CopyType copyType, MetricSet owner, boolean includeUnused) { - super(source, copyType, owner, includeUnused); - msgName = source.msgName; } public String getMessageName() { @@ -76,30 +42,30 @@ public class MessageTypeMetricSet extends MetricSet { } private void updateFailureMetrics(Reply r) { + errorCount++; String error = DocumentProtocol.getErrorName(r.getError(0).getCode()); - CountMetric s = (CountMetric)errors.getMetric(error); + Long s = errorCounts.get(error); if (s == null) { - s = new ErrorMetric(error, errors); - errorSum.addMetricToSum(s); + errorCounts.put(error, 1L); + } else { + errorCounts.put(error, s+1); } - s.inc(); } private void updateSuccessMetrics(Reply r) { if (!(r instanceof DocumentIgnoredReply)) { if (r.getMessage().getTimeReceived() != 0) { - latency.addValue(SystemTimer.INSTANCE.milliTime() - r.getMessage().getTimeReceived()); + long latency = (SystemTimer.INSTANCE.milliTime() - r.getMessage().getTimeReceived()); + latency_max = Math.max(latency_max, latency); + latency_min = Math.min(latency_min, latency); + latency_total += latency; } - count.inc(); + count++; } else { - ignored.inc(); + ignored++; } } - @Override - public Metric clone(CopyType type, MetricSet owner, boolean includeUnused) - { return new MessageTypeMetricSet(this, type, owner, includeUnused); } - /** * Returns true if every error in a stream is a test and set condition failed */ diff --git a/vespaclient-core/src/main/java/com/yahoo/clientmetrics/RouteMetricSet.java b/vespaclient-core/src/main/java/com/yahoo/clientmetrics/RouteMetricSet.java index c5469e8a4f1..21efd53d7a5 100644 --- a/vespaclient-core/src/main/java/com/yahoo/clientmetrics/RouteMetricSet.java +++ b/vespaclient-core/src/main/java/com/yahoo/clientmetrics/RouteMetricSet.java @@ -2,9 +2,6 @@ package com.yahoo.clientmetrics; import com.yahoo.messagebus.Reply; -import com.yahoo.metrics.Metric; -import com.yahoo.metrics.MetricSet; -import com.yahoo.metrics.SumMetric; import java.util.HashMap; import java.util.Map; @@ -12,9 +9,9 @@ import java.util.Map; /** * @author thomasg */ -public class RouteMetricSet extends MetricSet { +public class RouteMetricSet { - private final SumMetric sum; + private final String route; private final ProgressCallback callback; private final Map<Integer, MessageTypeMetricSet> typeMap = new HashMap<>(); @@ -24,28 +21,17 @@ public class RouteMetricSet extends MetricSet { } public RouteMetricSet(String route, ProgressCallback callback) { - super(route, "", "Messages sent to the named route", null); - sum = new SumMetric("total", "", "All kinds of messages sent to the given route", this); + this.route = route; this.callback = callback; } - @Override - public String getXMLTag() { - return "route"; - } - - private RouteMetricSet(RouteMetricSet source, CopyType copyType, MetricSet owner, boolean includeUnused) { - super(source, copyType, owner, includeUnused); - sum = null; - callback = null; - } + public Map<Integer, MessageTypeMetricSet> getMetrics() { return typeMap; } public void addReply(Reply r) { MessageTypeMetricSet type = typeMap.get(r.getMessage().getType()); if (type == null) { String msgName = r.getMessage().getClass().getSimpleName().replace("Message", ""); - type = new MessageTypeMetricSet(msgName, this); - sum.addMetricToSum(type); + type = new MessageTypeMetricSet(msgName); typeMap.put(r.getMessage().getType(), type); } @@ -61,12 +47,7 @@ public class RouteMetricSet extends MetricSet { } } - @Override - public Metric clone(CopyType type, MetricSet owner, boolean includeUnused) { - return new RouteMetricSet(this, type, owner, includeUnused); - } - - String getRoute() { - return getName(); + public String getRoute() { + return route; } } diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/DummySessionFactory.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/DummySessionFactory.java index c644b551a79..4e9e17d0b5f 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/DummySessionFactory.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/DummySessionFactory.java @@ -3,7 +3,6 @@ package com.yahoo.feedapi; import com.yahoo.document.Document; import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; -import com.yahoo.jdisc.Metric; import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; import com.yahoo.messagebus.Message; @@ -47,7 +46,7 @@ public class DummySessionFactory implements SessionFactory { } @Override - public SendSession createSendSession(ReplyHandler r, Metric metric) { + public SendSession createSendSession(ReplyHandler r) { if (output != null) { return new DumpDocuments(output, r, this); } diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/FeedContext.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/FeedContext.java index 6bfd132a70f..2d35adfab75 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/FeedContext.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/FeedContext.java @@ -1,14 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.feedapi; -import com.yahoo.cloud.config.ClusterListConfig; -import com.yahoo.cloud.config.SlobroksConfig; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.jdisc.Metric; -import com.yahoo.vespa.config.content.LoadTypeConfig; import com.yahoo.document.DocumentTypeManager; import com.yahoo.clientmetrics.ClientMetrics; -import com.yahoo.vespaclient.config.FeederConfig; import java.util.Map; import java.util.TreeMap; @@ -19,26 +13,16 @@ public class FeedContext { private final MessagePropertyProcessor propertyProcessor; private final DocumentTypeManager docTypeManager; private final ClientMetrics metrics; - private final Metric metric; private Map<String, SharedSender> senders = new TreeMap<>(); public static final Object sync = new Object(); public static FeedContext instance = null; - public FeedContext(MessagePropertyProcessor propertyProcessor, SessionFactory factory, DocumentTypeManager manager, Metric metric) { + public FeedContext(MessagePropertyProcessor propertyProcessor, SessionFactory factory, DocumentTypeManager manager) { this.propertyProcessor = propertyProcessor; this.factory = factory; docTypeManager = manager; metrics = new ClientMetrics(); - this.metric = metric; - } - - public ClientMetrics getMetrics() { - return metrics; - } - - public Metric getMetricAPI() { - return metric; } private void shutdownSenders() { @@ -52,7 +36,7 @@ public class FeedContext { Map<String, SharedSender> newSenders = new TreeMap<>(); for (Map.Entry<String, SharedSender> sender : senders.entrySet()) { - newSenders.put(sender.getKey(), new SharedSender(sender.getKey(), factory, sender.getValue(), metric)); + newSenders.put(sender.getKey(), new SharedSender(sender.getKey(), factory, sender.getValue())); } shutdownSenders(); @@ -67,7 +51,7 @@ public class FeedContext { SharedSender sender = senders.get(route); if (sender == null) { - sender = new SharedSender(route, factory, null, metric); + sender = new SharedSender(route, factory, null); senders.put(route, sender); metrics.addRouteMetricSet(sender.getMetrics()); } @@ -83,37 +67,4 @@ public class FeedContext { return docTypeManager; } - public static FeedContext getInstance(FeederConfig feederConfig, - LoadTypeConfig loadTypeConfig, - DocumentmanagerConfig documentmanagerConfig, - SlobroksConfig slobroksConfig, - Metric metric) { - synchronized (sync) { - try { - if (instance == null) { - MessagePropertyProcessor proc = new MessagePropertyProcessor(feederConfig, loadTypeConfig); - - if (System.getProperty("vespa.local", "false").equals("true")) { - // Use injected configs when running from Application. This means we cannot reconfigure - MessageBusSessionFactory mbusFactory = new MessageBusSessionFactory(proc, documentmanagerConfig, slobroksConfig); - instance = new FeedContext(proc, mbusFactory, mbusFactory.getAccess().getDocumentTypeManager(), metric); - } - else { - // Don't send configs to messagebus to make it self-subscribe instead as this instance - // survives reconfig :-/ - // This code will die soon ... - MessageBusSessionFactory mbusFactory = new MessageBusSessionFactory(proc, null, null); - instance = new FeedContext(proc, mbusFactory, mbusFactory.getAccess().getDocumentTypeManager(), metric); - } - } else { - instance.getPropertyProcessor().configure(feederConfig, loadTypeConfig); - } - - return instance; - } catch (Exception e) { - throw new RuntimeException(e); - } - } - } - } diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/Feeder.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/Feeder.java index e354cba141d..19d074a0ead 100644 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/Feeder.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/Feeder.java @@ -10,8 +10,8 @@ import java.util.List; import javax.xml.stream.XMLStreamException; import com.yahoo.document.DocumentTypeManager; +import com.yahoo.vespaxmlparser.FeedOperation; import com.yahoo.vespaxmlparser.FeedReader; -import com.yahoo.vespaxmlparser.VespaXMLFeedReader; /** * Base class for unpacking document operation streams and pushing to feed @@ -80,14 +80,13 @@ public abstract class Feeder { while (!sender.isAborted()) { try { - VespaXMLFeedReader.Operation op = new VespaXMLFeedReader.Operation(); - reader.read(op); + FeedOperation op = reader.read(); if (createIfNonExistent && op.getDocumentUpdate() != null) { op.getDocumentUpdate().setCreateIfNonExistent(true); } // Done feeding. - if (op.getType() == VespaXMLFeedReader.OperationType.INVALID) { + if (op.getType() == FeedOperation.Type.INVALID) { break; } else { sender.sendOperation(op); diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/FeederOptions.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/FeederOptions.java index dc78f037534..ecfd3c9eded 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/FeederOptions.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/FeederOptions.java @@ -2,9 +2,11 @@ package com.yahoo.feedapi; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.messagebus.*; +import com.yahoo.messagebus.DynamicThrottlePolicy; +import com.yahoo.messagebus.RateThrottlingPolicy; +import com.yahoo.messagebus.SourceSessionParams; +import com.yahoo.messagebus.StaticThrottlePolicy; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; -import com.yahoo.messagebus.routing.RetryTransientErrorsPolicy; import com.yahoo.vespaclient.config.FeederConfig; @@ -19,61 +21,28 @@ public class FeederOptions { private boolean abortOnDocumentError = true; private boolean abortOnSendError = true; private boolean retryEnabled = true; - private double retryDelay = 1; private double timeout = 60; private int maxPendingBytes = 0; private int maxPendingDocs = 0; private double maxFeedRate = 0.0; - private String documentManagerConfigId = "client"; - private String idPrefix = ""; private String route = "default"; - private String routingConfigId; - private String slobrokConfigId; private int traceLevel; private int mbusPort; private DocumentProtocol.Priority priority = DocumentProtocol.Priority.NORMAL_3; - private boolean priorityExplicitlySet = false; private String docprocChain = ""; /** Constructs an options object with all default values. */ - public FeederOptions() { + FeederOptions() { // empty } - /** - * Implements the copy constructor. - * - * @param src The options to copy. - */ - public FeederOptions(FeederOptions src) { - abortOnDocumentError = src.abortOnDocumentError; - abortOnSendError = src.abortOnSendError; - retryEnabled = src.retryEnabled; - retryDelay = src.retryDelay; - timeout = src.timeout; - maxPendingBytes = src.maxPendingBytes; - maxPendingDocs = src.maxPendingDocs; - maxFeedRate = src.maxFeedRate; - documentManagerConfigId = src.documentManagerConfigId; - idPrefix = src.idPrefix; - route = src.route; - routingConfigId = src.routingConfigId; - slobrokConfigId = src.slobrokConfigId; - traceLevel = src.traceLevel; - mbusPort = src.mbusPort; - priority = src.priority; - docprocChain = src.docprocChain; - } - /** Constructor that sets values from config. */ - public FeederOptions(FeederConfig config) { + FeederOptions(FeederConfig config) { setAbortOnDocumentError(config.abortondocumenterror()); setAbortOnSendError(config.abortonsenderror()); - setIdPrefix(config.idprefix()); setMaxPendingBytes(config.maxpendingbytes()); setMaxPendingDocs(config.maxpendingdocs()); setRetryEnabled(config.retryenabled()); - setRetryDelay(config.retrydelay()); setRoute(config.route()); setTimeout(config.timeout()); setTraceLevel(config.tracelevel()); @@ -82,31 +51,18 @@ public class FeederOptions { setMaxFeedRate(config.maxfeedrate()); } - public void setMaxFeedRate(double feedRate) { + void setMaxFeedRate(double feedRate) { maxFeedRate = feedRate; } - - public double getMaxFeedRate() { - return maxFeedRate; - } - - public boolean getRetryEnabled() { + boolean getRetryEnabled() { return retryEnabled; } - public void setRetryEnabled(boolean retryEnabled) { + private void setRetryEnabled(boolean retryEnabled) { this.retryEnabled = retryEnabled; } - public double getRetryDelay() { - return retryDelay; - } - - public void setRetryDelay(double retryDelay) { - this.retryDelay = retryDelay; - } - public double getTimeout() { return timeout; } @@ -115,46 +71,30 @@ public class FeederOptions { this.timeout = timeout; } - public int getMaxPendingBytes() { - return maxPendingBytes; - } - - public void setMaxPendingBytes(int maxPendingBytes) { + private void setMaxPendingBytes(int maxPendingBytes) { this.maxPendingBytes = maxPendingBytes; } - public int getMaxPendingDocs() { - return maxPendingDocs; - } - - public void setMaxPendingDocs(int maxPendingDocs) { + private void setMaxPendingDocs(int maxPendingDocs) { this.maxPendingDocs = maxPendingDocs; } - public boolean abortOnDocumentError() { + boolean abortOnDocumentError() { return abortOnDocumentError; } - public void setAbortOnDocumentError(boolean abortOnDocumentError) { + void setAbortOnDocumentError(boolean abortOnDocumentError) { this.abortOnDocumentError = abortOnDocumentError; } - public boolean abortOnSendError() { + boolean abortOnSendError() { return abortOnSendError; } - public void setAbortOnSendError(boolean abortOnSendError) { + private void setAbortOnSendError(boolean abortOnSendError) { this.abortOnSendError = abortOnSendError; } - public String getIdPrefix() { - return idPrefix; - } - - public void setIdPrefix(String idPrefix) { - this.idPrefix = idPrefix; - } - public void setRoute(String route) { this.route = route; } @@ -167,35 +107,7 @@ public class FeederOptions { return priority; } - public boolean isPriorityExplicitlySet() { - return priorityExplicitlySet; - } - - public String getSlobrokConfigId() { - return slobrokConfigId; - } - - public void setSlobrokConfigId(String slobrokConfigId) { - this.slobrokConfigId = slobrokConfigId; - } - - public String getRoutingConfigId() { - return routingConfigId; - } - - public void setRoutingConfigId(String routingConfigId) { - this.routingConfigId = routingConfigId; - } - - public String getDocumentManagerConfigId() { - return documentManagerConfigId; - } - - public void setDocumentManagerConfigId(String documentManagerConfigId) { - this.documentManagerConfigId = documentManagerConfigId; - } - - public int getTraceLevel() { + int getTraceLevel() { return traceLevel; } @@ -203,24 +115,19 @@ public class FeederOptions { this.traceLevel = traceLevel; } - public int getMessageBusPort() { - return mbusPort; - } - - public void setMessageBusPort(int mbusPort) { + private void setMessageBusPort(int mbusPort) { this.mbusPort = mbusPort; } public void setPriority(DocumentProtocol.Priority priority) { this.priority = priority; - this.priorityExplicitlySet = true; } - public String getDocprocChain() { + String getDocprocChain() { return docprocChain; } - public void setDocprocChain(String chain) { + private void setDocprocChain(String chain) { docprocChain = chain; } @@ -228,7 +135,7 @@ public class FeederOptions { * Creates a source session params object with parameters set as these options * dictate. */ - public SourceSessionParams toSourceSessionParams() { + SourceSessionParams toSourceSessionParams() { SourceSessionParams params = new SourceSessionParams(); StaticThrottlePolicy policy; @@ -252,7 +159,7 @@ public class FeederOptions { return params; } - public RPCNetworkParams getNetworkParams() { + RPCNetworkParams getNetworkParams() { try { RPCNetworkParams networkParams = new RPCNetworkParams(); if (mbusPort != -1) { @@ -271,19 +178,13 @@ public class FeederOptions { "abortOnDocumentError=" + abortOnDocumentError + ", abortOnSendError=" + abortOnSendError + ", retryEnabled=" + retryEnabled + - ", retryDelay=" + retryDelay + ", timeout=" + timeout + ", maxPendingBytes=" + maxPendingBytes + ", maxPendingDocs=" + maxPendingDocs + - ", documentManagerConfigId='" + documentManagerConfigId + '\'' + - ", idPrefix='" + idPrefix + '\'' + ", route='" + route + '\'' + - ", routingConfigId='" + routingConfigId + '\'' + - ", slobrokConfigId='" + slobrokConfigId + '\'' + ", traceLevel=" + traceLevel + ", mbusPort=" + mbusPort + ", priority=" + priority.name() + - ", priorityExplicitlySet=" + priorityExplicitlySet + ", docprocChain='" + docprocChain + '\'' + '}'; } @@ -301,24 +202,12 @@ public class FeederOptions { if (maxPendingDocs != that.maxPendingDocs) return false; if (maxFeedRate != that.maxFeedRate) return false; if (mbusPort != that.mbusPort) return false; - if (priorityExplicitlySet != that.priorityExplicitlySet) return false; - if (Double.compare(that.retryDelay, retryDelay) != 0) return false; if (retryEnabled != that.retryEnabled) return false; if (Double.compare(that.timeout, timeout) != 0) return false; if (traceLevel != that.traceLevel) return false; if (docprocChain != null ? !docprocChain.equals(that.docprocChain) : that.docprocChain != null) return false; - if (documentManagerConfigId != null ? !documentManagerConfigId.equals(that.documentManagerConfigId) : that.documentManagerConfigId != null) { - return false; - } - if (idPrefix != null ? !idPrefix.equals(that.idPrefix) : that.idPrefix != null) return false; if (priority != that.priority) return false; if (route != null ? !route.equals(that.route) : that.route != null) return false; - if (routingConfigId != null ? !routingConfigId.equals(that.routingConfigId) : that.routingConfigId != null) { - return false; - } - if (slobrokConfigId != null ? !slobrokConfigId.equals(that.slobrokConfigId) : that.slobrokConfigId != null) { - return false; - } return true; } @@ -330,22 +219,15 @@ public class FeederOptions { result = (abortOnDocumentError ? 1 : 0); result = 31 * result + (abortOnSendError ? 1 : 0); result = 31 * result + (retryEnabled ? 1 : 0); - temp = retryDelay != +0.0d ? Double.doubleToLongBits(retryDelay) : 0L; - result = 31 * result + (int) (temp ^ (temp >>> 32)); temp = timeout != +0.0d ? Double.doubleToLongBits(timeout) : 0L; result = 31 * result + (int) (temp ^ (temp >>> 32)); result = 31 * result + maxPendingBytes; result = 31 * result + maxPendingDocs; result = 31 * result + ((int)(maxFeedRate * 1000)); - result = 31 * result + (documentManagerConfigId != null ? documentManagerConfigId.hashCode() : 0); - result = 31 * result + (idPrefix != null ? idPrefix.hashCode() : 0); result = 31 * result + (route != null ? route.hashCode() : 0); - result = 31 * result + (routingConfigId != null ? routingConfigId.hashCode() : 0); - result = 31 * result + (slobrokConfigId != null ? slobrokConfigId.hashCode() : 0); result = 31 * result + traceLevel; result = 31 * result + mbusPort; result = 31 * result + (priority != null ? priority.hashCode() : 0); - result = 31 * result + (priorityExplicitlySet ? 1 : 0); result = 31 * result + (docprocChain != null ? docprocChain.hashCode() : 0); return result; } diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessageBusSessionFactory.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessageBusSessionFactory.java index 12a4ecde493..5e52da23c12 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessageBusSessionFactory.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessageBusSessionFactory.java @@ -5,34 +5,21 @@ import com.yahoo.cloud.config.SlobroksConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusParams; -import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; -import com.yahoo.jdisc.Metric; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.ReplyHandler; import com.yahoo.messagebus.SourceSession; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; -import java.util.Collections; - public class MessageBusSessionFactory implements SessionFactory { private final MessageBusDocumentAccess access; private final MessagePropertyProcessor processor; - private interface Metrics { - String NUM_OPERATIONS = "num_operations"; - String NUM_PUTS = "num_puts"; - String NUM_REMOVES = "num_removes"; - String NUM_UPDATES = "num_updates"; - } - public MessageBusSessionFactory(MessagePropertyProcessor processor) { this(processor, null, null); } - public MessageBusSessionFactory(MessagePropertyProcessor processor, + private MessageBusSessionFactory(MessagePropertyProcessor processor, DocumentmanagerConfig documentmanagerConfig, SlobroksConfig slobroksConfig) { this.processor = processor; @@ -53,10 +40,9 @@ public class MessageBusSessionFactory implements SessionFactory { } @Override - public synchronized SendSession createSendSession(ReplyHandler handler, Metric metric) { + public synchronized SendSession createSendSession(ReplyHandler handler) { return new SourceSessionWrapper( - access.getMessageBus().createSourceSession(handler, processor.getFeederOptions().toSourceSessionParams()), - metric); + access.getMessageBus().createSourceSession(handler, processor.getFeederOptions().toSourceSessionParams())); } public void shutDown() { @@ -66,18 +52,13 @@ public class MessageBusSessionFactory implements SessionFactory { private class SourceSessionWrapper extends SendSession { private final SourceSession session; - private final Metric metric; - private final Metric.Context context; - private SourceSessionWrapper(SourceSession session, Metric metric) { + private SourceSessionWrapper(SourceSession session) { this.session = session; - this.metric = metric; - this.context = metric.createContext(Collections.<String, String>emptyMap()); } @Override protected com.yahoo.messagebus.Result onSend(Message m, boolean blockIfQueueFull) throws InterruptedException { - updateCounters(m); if (blockIfQueueFull) { return session.sendBlocking(m); } else { @@ -85,18 +66,6 @@ public class MessageBusSessionFactory implements SessionFactory { } } - private void updateCounters(Message m) { - metric.add(Metrics.NUM_OPERATIONS, 1, context); - - if (m instanceof PutDocumentMessage) { - metric.add(Metrics.NUM_PUTS, 1, context); - } else if (m instanceof RemoveDocumentMessage) { - metric.add(Metrics.NUM_REMOVES, 1, context); - } else if (m instanceof UpdateDocumentMessage) { - metric.add(Metrics.NUM_UPDATES, 1, context); - } - } - @Override public void close() { session.close(); diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/SessionFactory.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/SessionFactory.java index 6dce2b6f315..52583052ddf 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/SessionFactory.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/SessionFactory.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.feedapi; -import com.yahoo.jdisc.Metric; import com.yahoo.messagebus.ReplyHandler; /** @@ -17,5 +16,5 @@ public interface SessionFactory { * @param handler A replyhandler to callback when receiving replies from messagebus * @return The session to use for sending messages. */ - SendSession createSendSession(ReplyHandler handler, Metric metric); + SendSession createSendSession(ReplyHandler handler); } diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java index 52d21897a15..4fcbcf4d634 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java @@ -2,7 +2,6 @@ package com.yahoo.feedapi; import com.yahoo.concurrent.SystemTimer; -import com.yahoo.jdisc.Metric; import com.yahoo.log.LogLevel; import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Message; @@ -31,8 +30,8 @@ public class SharedSender implements ReplyHandler { * Creates a new shared sender. * If oldsender != null, we copy that status information from that sender. */ - SharedSender(String route, SessionFactory factory, SharedSender oldSender, Metric metric) { - sender = (factory != null) ? factory.createSendSession(this, metric) : null; + SharedSender(String route, SessionFactory factory, SharedSender oldSender) { + sender = (factory != null) ? factory.createSendSession(this) : null; metrics = (oldSender != null) ? oldSender.metrics : new RouteMetricSet(route, null); } diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/SingleSender.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/SingleSender.java index 9d0c740789e..d78e3a62302 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/SingleSender.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/SingleSender.java @@ -19,12 +19,10 @@ public class SingleSender implements SimpleFeedAccess { private final SharedSender.ResultCallback owner; private final SharedSender sender; private final List<MessageProcessor> messageProcessors = new ArrayList<>(); - private boolean blockingQueue; - public SingleSender(SharedSender.ResultCallback owner, SharedSender sender, boolean blockingQueue) { + public SingleSender(SharedSender.ResultCallback owner, SharedSender sender) { this.owner = owner; this.sender = sender; - this.blockingQueue = blockingQueue; } @Override @@ -86,7 +84,7 @@ public class SingleSender implements SimpleFeedAccess { * @param m The message to send */ public void send(Message m) { - sender.send(processMessage(m), owner, blockingQueue); + sender.send(processMessage(m), owner, true); } public void done() { diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/VespaFeedSender.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/VespaFeedSender.java index b441e81a829..a1666a83856 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/VespaFeedSender.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/VespaFeedSender.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.feedapi; -import com.yahoo.vespaxmlparser.VespaXMLFeedReader; +import com.yahoo.vespaxmlparser.FeedOperation; /** * Wrapper class for SimpleFeedAccess to send various XML operations. @@ -18,7 +18,7 @@ public class VespaFeedSender { return sender.isAborted(); } - public void sendOperation(VespaXMLFeedReader.Operation op) { + public void sendOperation(FeedOperation op) { switch (op.getType()) { case DOCUMENT: sender.put(op.getDocument(), op.getCondition()); diff --git a/vespaclient-core/src/main/java/com/yahoo/feedhandler/FeedResponse.java b/vespaclient-core/src/main/java/com/yahoo/feedhandler/FeedResponse.java index 4bf1267201a..58cceb2d8ee 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedhandler/FeedResponse.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedhandler/FeedResponse.java @@ -2,8 +2,6 @@ package com.yahoo.feedhandler; import com.yahoo.clientmetrics.RouteMetricSet; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.VespaHeaders; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; @@ -14,18 +12,13 @@ import com.yahoo.messagebus.ErrorCode; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.Reply; import com.yahoo.search.result.ErrorMessage; -import com.yahoo.text.Utf8String; -import com.yahoo.text.XMLWriter; -import java.io.IOException; -import java.io.OutputStream; -import java.io.OutputStreamWriter; import java.util.ArrayList; import java.util.List; import java.util.logging.Logger; import java.util.stream.Stream; -public final class FeedResponse extends HttpResponse implements SharedSender.ResultCallback { +public final class FeedResponse implements SharedSender.ResultCallback { private final static Logger log = Logger.getLogger(FeedResponse.class.getName()); private final List<ErrorMessage> errorMessages = new ArrayList<>(); @@ -37,7 +30,6 @@ public final class FeedResponse extends HttpResponse implements SharedSender.Res private final SharedSender.Pending pendingNumber = new SharedSender.Pending(); FeedResponse(RouteMetricSet metrics) { - super(com.yahoo.jdisc.http.HttpResponse.Status.OK); this.metrics = metrics; } @@ -49,44 +41,6 @@ public final class FeedResponse extends HttpResponse implements SharedSender.Res abortOnError = abort; } - @Override - public void render(OutputStream outputStream) throws IOException { - if ( ! errorMessages.isEmpty()) - setStatus(VespaHeaders.getStatus(false, errorMessages.get(0), errorMessages.iterator())); - - XMLWriter writer = new XMLWriter(new OutputStreamWriter(outputStream)); - writer.openTag("result"); - - if (metrics != null) { - metrics.printXml(writer, 0, 0); - } - if (traces.length() > 0) { - writer.openTag("trace"); - writer.append(traces); - writer.closeTag(); - } - if (!errors.isEmpty()) { - writer.openTag("errors"); - writer.attribute(new Utf8String("count"), errors.size()); - - for (int i = 0; i < errors.size() && i < 10; ++i) { - writer.openTag("error"); - writer.attribute(new Utf8String("message"), errors.get(i)); - writer.closeTag(); - } - writer.closeTag(); - } - - writer.closeTag(); - writer.flush(); - outputStream.close(); - } - - @Override - public java.lang.String getContentType() { - return "application/xml"; - } - private String prettyPrint(Message m) { if (m instanceof PutDocumentMessage) { return "PUT[" + ((PutDocumentMessage)m).getDocumentPut().getDocument().getId() + "] "; diff --git a/vespaclient-core/src/main/java/com/yahoo/feedhandler/MetricResponse.java b/vespaclient-core/src/main/java/com/yahoo/feedhandler/MetricResponse.java deleted file mode 100644 index 4947032e649..00000000000 --- a/vespaclient-core/src/main/java/com/yahoo/feedhandler/MetricResponse.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.feedhandler; - -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.metrics.MetricSet; -import com.yahoo.text.XMLWriter; - -import java.io.IOException; -import java.io.OutputStream; -import java.io.OutputStreamWriter; - -/** - * Response that generates metric output like a status page. - */ -public final class MetricResponse extends HttpResponse { - - MetricSet set; - - MetricResponse(MetricSet set) { - super(com.yahoo.jdisc.http.HttpResponse.Status.OK); - this.set = set; - } - - @Override - public void render(OutputStream stream) throws IOException { - XMLWriter writer = new XMLWriter(new OutputStreamWriter(stream)); - writer.openTag("status"); - set.printXml(writer, 0, 2); - writer.closeTag(); - writer.flush(); - } - - @Override - public java.lang.String getContentType() { - return "application/xml"; - } - -} diff --git a/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java b/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java index 4587c84f9dc..892f3763805 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandler.java @@ -1,16 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.feedhandler; -import com.google.inject.Inject; import com.yahoo.clientmetrics.RouteMetricSet; -import com.yahoo.cloud.config.ClusterListConfig; -import com.yahoo.cloud.config.SlobroksConfig; -import com.yahoo.container.jdisc.EmptyResponse; import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.protect.Error; -import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.feedapi.DocprocMessageProcessor; import com.yahoo.feedapi.FeedContext; import com.yahoo.feedapi.Feeder; import com.yahoo.feedapi.JsonFeeder; @@ -18,13 +11,8 @@ import com.yahoo.feedapi.MessagePropertyProcessor; import com.yahoo.feedapi.SimpleFeedAccess; import com.yahoo.feedapi.SingleSender; import com.yahoo.feedapi.XMLFeeder; -import com.yahoo.jdisc.Metric; -import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.vespaclient.config.FeederConfig; import java.util.List; -import java.util.concurrent.Executor; -import java.util.concurrent.atomic.AtomicInteger; /** @@ -37,89 +25,47 @@ public final class VespaFeedHandler extends VespaFeedHandlerBase { public static final String JSON_INPUT = "jsonInput"; - private final AtomicInteger busyThreads = new AtomicInteger(0); - private final int maxBusyThreads; - - @SuppressWarnings("unused") - @Inject - public VespaFeedHandler(FeederConfig feederConfig, - LoadTypeConfig loadTypeConfig, - DocumentmanagerConfig documentmanagerConfig, - SlobroksConfig slobroksConfig, - Executor executor, - Metric metric) { - super(feederConfig, loadTypeConfig, documentmanagerConfig, slobroksConfig, executor, metric); - this.maxBusyThreads = feederConfig.maxbusythreads(); - } - - private VespaFeedHandler(FeedContext context, Executor executor) { - super(context, executor); - this.maxBusyThreads = 32; + private VespaFeedHandler(FeedContext context) { + super(context); } - public static VespaFeedHandler createFromContext(FeedContext context, Executor executor) { - return new VespaFeedHandler(context, executor); + public static VespaFeedHandler createFromContext(FeedContext context) { + return new VespaFeedHandler(context); } - @Override - public HttpResponse handle(HttpRequest request) { - return handle(request, null, 1); - } - - public HttpResponse handle(HttpRequest request, RouteMetricSet.ProgressCallback callback, int numThreads) { - if (request.getProperty("status") != null) { - return new MetricResponse(context.getMetrics().getMetricSet()); - } - try { - int busy = busyThreads.incrementAndGet(); - if (busy > maxBusyThreads) - return new EmptyResponse(com.yahoo.jdisc.http.HttpResponse.Status.SERVICE_UNAVAILABLE); + public FeedResponse handle(HttpRequest request, RouteMetricSet.ProgressCallback callback, int numThreads) { + MessagePropertyProcessor.PropertySetter properties = getPropertyProcessor().buildPropertySetter(request); - boolean asynchronous = request.getBooleanProperty("asynchronous"); + String route = properties.getRoute().toString(); + FeedResponse response = new FeedResponse(new RouteMetricSet(route, callback)); - MessagePropertyProcessor.PropertySetter properties = getPropertyProcessor().buildPropertySetter(request); + SingleSender sender = new SingleSender(response, getSharedSender(route)); + sender.addMessageProcessor(properties); + ThreadedFeedAccess feedAccess = new ThreadedFeedAccess(numThreads, sender); + Feeder feeder = createFeeder(feedAccess, request); + feeder.setAbortOnDocumentError(properties.getAbortOnDocumentError()); + feeder.setCreateIfNonExistent(properties.getCreateIfNonExistent()); + response.setAbortOnFeedError(properties.getAbortOnFeedError()); - String route = properties.getRoute().toString(); - FeedResponse response = new FeedResponse(new RouteMetricSet(route, callback)); - - SingleSender sender = new SingleSender(response, getSharedSender(route), !asynchronous); - sender.addMessageProcessor(properties); - sender.addMessageProcessor(new DocprocMessageProcessor(getDocprocChain(request), getDocprocServiceRegistry(request))); - ThreadedFeedAccess feedAccess = new ThreadedFeedAccess(numThreads, sender); - Feeder feeder = createFeeder(feedAccess, request); - feeder.setAbortOnDocumentError(properties.getAbortOnDocumentError()); - feeder.setCreateIfNonExistent(properties.getCreateIfNonExistent()); - response.setAbortOnFeedError(properties.getAbortOnFeedError()); - - List<String> errors = feeder.parse(); - for (String s : errors) { - response.addXMLParseError(s); - } - if (errors.size() > 0 && feeder instanceof XMLFeeder) { - response.addXMLParseError("If you are trying to feed JSON, set the Content-Type header to application/json."); - } - - sender.done(); - feedAccess.close(); + List<String> errors = feeder.parse(); + for (String s : errors) { + response.addXMLParseError(s); + } - if (asynchronous) { - return response; - } - long millis = getTimeoutMillis(request); - boolean completed = sender.waitForPending(millis); - if (!completed) { - response.addError(Error.TIMEOUT, "Timed out after " + millis + " ms waiting for responses"); - } - response.done(); - return response; - } finally { - busyThreads.decrementAndGet(); + sender.done(); + feedAccess.close(); + long millis = getTimeoutMillis(request); + boolean completed = sender.waitForPending(millis); + if (!completed) { + response.addError(Error.TIMEOUT, "Timed out after " + millis + " ms waiting for responses"); } + response.done(); + return response; + } private Feeder createFeeder(SimpleFeedAccess sender, HttpRequest request) { - String contentType = request.getHeader("Content-Type"); - if (Boolean.valueOf(request.getProperty(JSON_INPUT)) || (contentType != null && contentType.startsWith("application/json"))) { + if (Boolean.valueOf(request.getProperty(JSON_INPUT))) { return new JsonFeeder(getDocumentTypeManager(), sender, getRequestInputStream(request)); } else { return new XMLFeeder(getDocumentTypeManager(), sender, getRequestInputStream(request)); diff --git a/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerBase.java b/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerBase.java index 9dc81c31a0d..532f10663b9 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerBase.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedhandler/VespaFeedHandlerBase.java @@ -1,51 +1,26 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.feedhandler; -import com.google.inject.Inject; -import com.yahoo.clientmetrics.ClientMetrics; -import com.yahoo.cloud.config.SlobroksConfig; -import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.docproc.DocprocService; import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.feedapi.FeedContext; import com.yahoo.feedapi.MessagePropertyProcessor; import com.yahoo.feedapi.SharedSender; -import com.yahoo.jdisc.Metric; import com.yahoo.search.query.ParameterParser; -import com.yahoo.vespa.config.content.LoadTypeConfig; -import com.yahoo.vespaclient.config.FeederConfig; -import java.io.IOException; + import java.io.InputStream; -import java.util.concurrent.Executor; -import java.util.zip.GZIPInputStream; -public abstract class VespaFeedHandlerBase extends ThreadedHttpRequestHandler { +public abstract class VespaFeedHandlerBase { protected FeedContext context; private final long defaultTimeoutMillis; - @Inject - public VespaFeedHandlerBase(FeederConfig feederConfig, - LoadTypeConfig loadTypeConfig, - DocumentmanagerConfig documentmanagerConfig, - SlobroksConfig slobroksConfig, - Executor executor, - Metric metric) { - this(FeedContext.getInstance(feederConfig, loadTypeConfig, documentmanagerConfig, - slobroksConfig, metric), - executor, (long)feederConfig.timeout() * 1000); - } - - VespaFeedHandlerBase(FeedContext context, Executor executor) { - this(context, executor, context.getPropertyProcessor().getDefaultTimeoutMillis()); + VespaFeedHandlerBase(FeedContext context) { + this(context, context.getPropertyProcessor().getDefaultTimeoutMillis()); } - VespaFeedHandlerBase(FeedContext context, Executor executor, long defaultTimeoutMillis) { - super(executor, context.getMetricAPI()); + private VespaFeedHandlerBase(FeedContext context, long defaultTimeoutMillis) { this.context = context; this.defaultTimeoutMillis = defaultTimeoutMillis; } @@ -54,14 +29,6 @@ public abstract class VespaFeedHandlerBase extends ThreadedHttpRequestHandler { return context.getSharedSender(route); } - DocprocService getDocprocChain(HttpRequest request) { - return context.getPropertyProcessor().getDocprocChain(request); - } - - ComponentRegistry<DocprocService> getDocprocServiceRegistry(HttpRequest request) { - return context.getPropertyProcessor().getDocprocServiceRegistry(request); - } - MessagePropertyProcessor getPropertyProcessor() { return context.getPropertyProcessor(); } @@ -73,25 +40,13 @@ public abstract class VespaFeedHandlerBase extends ThreadedHttpRequestHandler { * @throws IllegalArgumentException if GZIP stream creation failed */ InputStream getRequestInputStream(HttpRequest request) { - if ("gzip".equals(request.getHeader("Content-Encoding"))) { - try { - return new GZIPInputStream(request.getData()); - } catch (IOException e) { - throw new IllegalArgumentException("Failed to create GZIP input stream from content", e); - } - } else { - return request.getData(); - } + return request.getData(); } protected DocumentTypeManager getDocumentTypeManager() { return context.getDocumentTypeManager(); } - public ClientMetrics getMetrics() { - return context.getMetrics(); - } - protected long getTimeoutMillis(HttpRequest request) { return ParameterParser.asMilliSeconds(request.getProperty("timeout"), defaultTimeoutMillis); } diff --git a/vespaclient-core/src/test/java/com/yahoo/feedapi/FeederOptionsTestCase.java b/vespaclient-core/src/test/java/com/yahoo/feedapi/FeederOptionsTestCase.java index 4dfde6f1d41..cb33b71424d 100644 --- a/vespaclient-core/src/test/java/com/yahoo/feedapi/FeederOptionsTestCase.java +++ b/vespaclient-core/src/test/java/com/yahoo/feedapi/FeederOptionsTestCase.java @@ -1,9 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.feedapi; -import static org.junit.Assert.*; import org.junit.Test; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + /** * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> */ @@ -27,26 +29,6 @@ public class FeederOptionsTestCase { assertTrue(f2.equals(f1)); assertTrue(f1.hashCode() == f2.hashCode()); - f1.setRoutingConfigId("blabla"); - assertFalse(f1.equals(f2)); - assertFalse(f2.equals(f1)); - assertFalse(f1.hashCode() == f2.hashCode()); - - f2.setRoutingConfigId("blabla"); - assertTrue(f1.equals(f2)); - assertTrue(f2.equals(f1)); - assertTrue(f1.hashCode() == f2.hashCode()); - - f1.setRetryDelay(5000); - assertFalse(f1.equals(f2)); - assertFalse(f2.equals(f1)); - assertFalse(f1.hashCode() == f2.hashCode()); - - f2.setRetryDelay(5000); - assertTrue(f1.equals(f2)); - assertTrue(f2.equals(f1)); - assertTrue(f1.hashCode() == f2.hashCode()); - f1.setRoute("all roads lead to rome"); assertFalse(f1.equals(f2)); assertFalse(f2.equals(f1)); diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/BenchmarkProgressPrinter.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/BenchmarkProgressPrinter.java index 80c7ccb113f..66934b35adc 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/BenchmarkProgressPrinter.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/BenchmarkProgressPrinter.java @@ -4,9 +4,6 @@ package com.yahoo.vespafeeder; import com.yahoo.clientmetrics.MessageTypeMetricSet; import com.yahoo.clientmetrics.RouteMetricSet; import com.yahoo.concurrent.Timer; -import com.yahoo.metrics.Metric; -import com.yahoo.metrics.MetricSet; -import com.yahoo.metrics.MetricVisitor; import java.io.PrintStream; @@ -24,51 +21,22 @@ public class BenchmarkProgressPrinter implements RouteMetricSet.ProgressCallback this.startTime = timer.milliTime(); } - class PrintVisitor extends MetricVisitor { - private final PrintStream out; - - PrintVisitor(PrintStream out) { - this.out = out; - } - - @Override - public boolean visitMetricSet(MetricSet set, boolean autoGenerated) { - if (set instanceof MessageTypeMetricSet && set.getName().equals("total")) { - Metric m = set.getMetric("latency"); - Metric count = set.getMetric("count"); - Metric err = set.getMetric("errors.total"); - - long okCount = 0, errCount = 0, minLatency = 0, maxLatency = 0, avgLatency = 0; - - if (m != null) { - minLatency = m.getLongValue("min"); - maxLatency = m.getLongValue("max"); - avgLatency = m.getLongValue("average"); - } - if (count != null) { - okCount = count.getLongValue("count"); - } - - if (err != null) { - errCount = err.getLongValue("count"); - } - long timeUsed = timer.milliTime() - startTime; - out.println(timeUsed + ", " + okCount + ", " + errCount + ", " + minLatency + ", " + maxLatency + ", " + avgLatency); - } - return true; + private void printMetrics(PrintStream out, RouteMetricSet metrics) { + for (MessageTypeMetricSet m : metrics.getMetrics().values()) { + long timeUsed = timer.milliTime() - startTime; + out.println(timeUsed + ", " + m.count + ", " + m.errorCount + ", " + m.latency_min + ", " + m.latency_max + ", " + m.latency_total/Long.max(1L, m.count)); } } @Override public void onProgress(RouteMetricSet metrics) { - //metrics.visit(new PrintVisitor(output), false); } @Override public void done(RouteMetricSet metrics) { try { - output.println("# Time used, num ok, num error, min latency, max latency, average latency"); - metrics.visit(new PrintVisitor(output), false); + output.println("# Time used, num ok, num error, min latency, max latency, average latency"); + printMetrics(output, metrics); } catch (Exception e) { e.printStackTrace(); } diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/ProgressPrinter.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/ProgressPrinter.java index 3b1ab38737d..84fe1691e4b 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/ProgressPrinter.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/ProgressPrinter.java @@ -4,12 +4,7 @@ package com.yahoo.vespafeeder; import com.yahoo.clientmetrics.MessageTypeMetricSet; import com.yahoo.clientmetrics.RouteMetricSet; import com.yahoo.concurrent.Timer; -import com.yahoo.metrics.Metric; -import com.yahoo.metrics.MetricSet; -import com.yahoo.metrics.MetricVisitor; -import com.yahoo.metrics.SumMetric; -import java.io.IOException; import java.io.PrintStream; import java.math.RoundingMode; import java.text.NumberFormat; @@ -24,8 +19,16 @@ public class ProgressPrinter implements RouteMetricSet.ProgressCallback { private long lastVerboseProgress = 0; final Timer timer; final PrintStream output; + final NumberFormat format; public ProgressPrinter(Timer timer, PrintStream output) { + format = NumberFormat.getNumberInstance(Locale.US); + format.setMaximumFractionDigits(2); + format.setMinimumFractionDigits(2); + format.setMinimumIntegerDigits(1); + format.setParseIntegerOnly(false); + format.setRoundingMode(RoundingMode.HALF_UP); + format.setGroupingUsed(false); this.timer = timer; this.output = output; @@ -34,58 +37,15 @@ public class ProgressPrinter implements RouteMetricSet.ProgressCallback { lastVerboseProgress = startTime; } - class PrintVisitor extends MetricVisitor { - final PrintStream out; - final NumberFormat format; - - PrintVisitor(PrintStream out) { - this.out = out; - format = NumberFormat.getNumberInstance(Locale.US); - format.setMaximumFractionDigits(2); - format.setMinimumFractionDigits(2); - format.setMinimumIntegerDigits(1); - format.setParseIntegerOnly(false); - format.setRoundingMode(RoundingMode.HALF_UP); - format.setGroupingUsed(false); - } - - @Override - public boolean visitMetricSet(MetricSet set, boolean autoGenerated) { - if (set instanceof MessageTypeMetricSet && !set.getName().equals("total")) { - Metric m = set.getMetric("latency"); - Metric count = set.getMetric("count"); - Metric err = set.getMetric("errors.total"); - - long okCount = 0, errCount = 0, ignored = 0; - long minLatency = 0, maxLatency = 0, avgLatency = 0; - - if (m != null) { - minLatency = m.getLongValue("min"); - maxLatency = m.getLongValue("max"); - avgLatency = m.getLongValue("average"); - } - if (count != null) { - okCount = count.getLongValue("count"); - } - Metric ignoredMetric = set.getMetric("ignored"); - if (ignoredMetric != null) { - ignored = ignoredMetric.getLongValue("count"); - } - - if (err != null) { - errCount = err.getLongValue("count"); - } - - long timeSinceStart = timer.milliTime() - startTime; - - out.println(((MessageTypeMetricSet)set).getMessageName() + ":\t" + - "ok: " + okCount + - " msgs/sec: " + format.format((double)okCount * 1000 / timeSinceStart) + - " failed: " + errCount + - " ignored: " + ignored + - " latency(min, max, avg): " + minLatency + ", " + maxLatency + ", " + avgLatency); - } - return true; + private void printMetrics(PrintStream out, RouteMetricSet metrics) { + for (MessageTypeMetricSet m : metrics.getMetrics().values()) { + long timeSinceStart = timer.milliTime() - startTime; + out.println(m.getMessageName() + ":\t" + + "ok: " + m.count + + " msgs/sec: " + format.format((double)m.count * 1000 / timeSinceStart) + + " failed: " + m.errorCount + + " ignored: " + m.ignored + + " latency(min, max, avg): " + m.latency_min + ", " + m.latency_max + ", " + m.latency_total/Long.max(1L, m.count)); } } @@ -98,25 +58,20 @@ public class ProgressPrinter implements RouteMetricSet.ProgressCallback { return dashes; } - public synchronized void renderStatusText(RouteMetricSet metrics, PrintStream stream) throws IOException { - String headline = "Messages sent to vespa (route " + metrics.getName() + ") :"; + public synchronized void renderStatusText(RouteMetricSet metrics, PrintStream stream) { + String headline = "Messages sent to vespa (route " + metrics.getRoute() + ") :"; stream.println(headline); stream.println(getDashes(headline.length())); - metrics.visit(new PrintVisitor(stream), false); + printMetrics(stream, metrics); } public long getOkMessageCount(RouteMetricSet metrics) { - SumMetric sum = (SumMetric)metrics.getMetric("total"); - - MetricSet ms = (MetricSet)sum.generateSum(); - if (ms != null) { - Metric latency = ms.getMetric("latency"); - if (latency != null) { - return latency.getLongValue("count"); - } + long count = 0; + for (MessageTypeMetricSet m : metrics.getMetrics().values()) { + count += m.count; } - return 0; + return count; } @Override diff --git a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java index 557caf21a89..100aba3a917 100755 --- a/vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespafeeder/VespaFeeder.java @@ -2,12 +2,10 @@ package com.yahoo.vespafeeder; import com.yahoo.clientmetrics.RouteMetricSet; -import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentTypeManagerConfigurer; import com.yahoo.feedapi.FeedContext; import com.yahoo.feedhandler.FeedResponse; -import com.yahoo.feedhandler.NullFeedMetric; import com.yahoo.feedhandler.VespaFeedHandler; import com.yahoo.log.LogSetup; import com.yahoo.concurrent.SystemTimer; @@ -20,14 +18,11 @@ import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; import java.util.List; -import java.util.concurrent.Executor; -import java.util.concurrent.Executors; public class VespaFeeder { - Arguments args; - DocumentTypeManager manager; - Executor threadPool = Executors.newCachedThreadPool(ThreadFactoryFactory.getThreadFactory("vespa-feeder")); + private final Arguments args; + private final DocumentTypeManager manager; public VespaFeeder(Arguments args, DocumentTypeManager manager) { this.args = args; @@ -37,7 +32,7 @@ public class VespaFeeder { public static class FeedErrorException extends Exception { String message; - public FeedErrorException(String message) { + FeedErrorException(String message) { this.message = message; } @@ -66,7 +61,7 @@ public class VespaFeeder { return new FeedErrorException(buffer.toString()); } - public RouteMetricSet.ProgressCallback createProgressCallback(PrintStream output) { + RouteMetricSet.ProgressCallback createProgressCallback(PrintStream output) { if ("benchmark".equals(args.getMode())) { return new BenchmarkProgressPrinter(SystemTimer.INSTANCE, output); } else { @@ -75,15 +70,15 @@ public class VespaFeeder { } void parseFiles(InputStream stdin, PrintStream output) throws Exception { - FeedContext context = new FeedContext(args.getPropertyProcessor(), args.getSessionFactory(), manager, new NullFeedMetric(true)); + FeedContext context = new FeedContext(args.getPropertyProcessor(), args.getSessionFactory(), manager); final BufferedInputStream input = new BufferedInputStream(stdin); - VespaFeedHandler handler = VespaFeedHandler.createFromContext(context, threadPool); + VespaFeedHandler handler = VespaFeedHandler.createFromContext(context); if (args.getFiles().isEmpty()) { InputStreamRequest req = new InputStreamRequest(input); setProperties(req, input); - FeedResponse response = (FeedResponse)handler.handle(req.toRequest(), createProgressCallback(output), args.getNumThreads()); + FeedResponse response = handler.handle(req.toRequest(), createProgressCallback(output), args.getNumThreads()); if ( ! response.isSuccess()) { throw renderErrors(response.getErrorList()); } @@ -101,7 +96,7 @@ public class VespaFeeder { final BufferedInputStream inputSnooper = new BufferedInputStream(new FileInputStream(fileName)); setProperties(req, inputSnooper); inputSnooper.close(); - FeedResponse response = (FeedResponse)handler.handle(req.toRequest(), createProgressCallback(output), args.getNumThreads()); + FeedResponse response = handler.handle(req.toRequest(), createProgressCallback(output), args.getNumThreads()); if (!response.isSuccess()) { throw renderErrors(response.getErrorList()); } diff --git a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/BenchmarkProgressPrinterTest.java b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/BenchmarkProgressPrinterTest.java index 69372e5b1f7..5545ffb7bd7 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/BenchmarkProgressPrinterTest.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/BenchmarkProgressPrinterTest.java @@ -71,7 +71,7 @@ public class BenchmarkProgressPrinterTest { String val = output.toString().split("\n")[1]; - String correctPattern = "62000,\\s*3,\\s*2,\\s*\\d+,\\s*\\d+,\\s*\\d+$"; + String correctPattern = "62000, \\d+, \\d+, \\d+, \\d+, \\d+$"; assertTrue("Value '" + val + "' does not match pattern '" + correctPattern + "'", val.matches(correctPattern)); } diff --git a/vespajlib/src/test/java/com/yahoo/time/TimeBudgetTest.java b/vespajlib/src/test/java/com/yahoo/time/TimeBudgetTest.java index f197cc34b32..b422c3b34fd 100644 --- a/vespajlib/src/test/java/com/yahoo/time/TimeBudgetTest.java +++ b/vespajlib/src/test/java/com/yahoo/time/TimeBudgetTest.java @@ -5,7 +5,6 @@ import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.test.ManualClock; import org.junit.Test; -import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.Optional; @@ -13,10 +12,8 @@ import java.util.Optional; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; public class TimeBudgetTest { - private final Clock clock = mock(Clock.class); @Test public void testBasics() { |