diff options
| author | Ola Aunrønning <olaa@verizonmedia.com> | 2021-02-03 11:08:25 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-02-03 11:08:25 +0100 |
| commit | 4b45a7684bdcbb1c1c044755c5d81a9a3b3e6326 (patch) | |
| tree | 25eb84905bee80a47e06a231342041e296d21c52 | |
| parent | 960022afa54672c004f37539452b6f68301a31a4 (diff) | |
| parent | 986dc62765dc2264ab07c2f4744d86d48e707193 (diff) | |
Merge branch 'master' into olaa/create-cloud-role
366 files changed, 4666 insertions, 3874 deletions
diff --git a/README.md b/README.md index a2b64f20fc3..3056f619982 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<!-- Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -[](https://vespa.ai) +[](https://vespa.ai) The big data serving engine - Store, search, rank and organize big data at user serving time. Vespa is an engine for low-latency computation over large data sets. diff --git a/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/BundleTest.java b/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/BundleTest.java index a46abce1dff..35a95ed3d89 100644 --- a/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/BundleTest.java +++ b/bundle-plugin-test/integration-test/src/test/java/com/yahoo/container/plugin/BundleTest.java @@ -91,7 +91,6 @@ public class BundleTest { // From SimpleSearcher assertThat(importPackage, containsString("com.yahoo.prelude.hitfield")); - assertThat(importPackage, containsString("org.json")); // From SimpleSearcher2 assertThat(importPackage, containsString("com.yahoo.processing")); diff --git a/bundle-plugin-test/test-bundles/main/pom.xml b/bundle-plugin-test/test-bundles/main/pom.xml index c9c9ea270eb..190e1c9d90f 100644 --- a/bundle-plugin-test/test-bundles/main/pom.xml +++ b/bundle-plugin-test/test-bundles/main/pom.xml @@ -16,11 +16,6 @@ <packaging>container-plugin</packaging> <dependencies> <dependency> - <groupId>org.json</groupId> - <artifactId>json</artifactId> - <scope>provided</scope> - </dependency> - <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>jrt</artifactId> <version>${project.version}</version> diff --git a/bundle-plugin-test/test-bundles/main/src/main/java/com/yahoo/test/SimpleSearcher.java b/bundle-plugin-test/test-bundles/main/src/main/java/com/yahoo/test/SimpleSearcher.java index dddca3f4d59..ae9644aa010 100644 --- a/bundle-plugin-test/test-bundles/main/src/main/java/com/yahoo/test/SimpleSearcher.java +++ b/bundle-plugin-test/test-bundles/main/src/main/java/com/yahoo/test/SimpleSearcher.java @@ -8,8 +8,6 @@ import com.yahoo.search.Searcher; import com.yahoo.search.result.Hit; import com.yahoo.search.searchchain.Execution; import com.yahoo.text.BooleanParser; -import org.json.JSONException; -import org.json.JSONObject; /** * A searcher adding a new hit. @@ -19,19 +17,13 @@ import org.json.JSONObject; public class SimpleSearcher extends Searcher { public Result search(Query query,Execution execution) { - try { BooleanParser.parseBoolean("true"); XMLString xmlString = new XMLString("<sampleXmlString/>"); Hit hit = new Hit("Hello world!"); - hit.setField("json", new JSONObject().put("price", 42).toString()); Result result = execution.search(query); result.hits().add(hit); return result; - - } catch (JSONException e) { - throw new RuntimeException(e); - } } } diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index a8655a82860..8bb7b75be89 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -34,7 +34,7 @@ <junit5.version>5.7.0</junit5.version> <junit5.platform.version>1.7.0</junit5.platform.version> <org.lz4.version>1.7.1</org.lz4.version> - <org.json.version>20090211</org.json.version> + <org.json.version>20090211</org.json.version><!-- TODO Vespa 8: remove as provided dependency --> <slf4j.version>1.7.5</slf4j.version> <tensorflow.version>1.12.0</tensorflow.version> <xml-apis.version>1.4.01</xml-apis.version> diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundle.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundle.java index 0ca4f5632a8..c4e61b1d3d0 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundle.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundle.java @@ -44,16 +44,30 @@ public class ClusterStateBundle { public static class FeedBlock { private final boolean blockFeedInCluster; private final String description; + private final Set<NodeResourceExhaustion> concreteExhaustions; public FeedBlock(boolean blockFeedInCluster, String description) { this.blockFeedInCluster = blockFeedInCluster; this.description = description; + this.concreteExhaustions = Collections.emptySet(); + } + + public FeedBlock(boolean blockFeedInCluster, String description, + Set<NodeResourceExhaustion> concreteExhaustions) + { + this.blockFeedInCluster = blockFeedInCluster; + this.description = description; + this.concreteExhaustions = concreteExhaustions; } public static FeedBlock blockedWithDescription(String desc) { return new FeedBlock(true, desc); } + public static FeedBlock blockedWith(String description, Set<NodeResourceExhaustion> concreteExhaustions) { + return new FeedBlock(true, description, concreteExhaustions); + } + public boolean blockFeedInCluster() { return blockFeedInCluster; } @@ -62,18 +76,31 @@ public class ClusterStateBundle { return description; } + public Set<NodeResourceExhaustion> getConcreteExhaustions() { + return concreteExhaustions; + } + + public boolean similarTo(FeedBlock other) { + // We check everything _but_ the description, as that includes current usage + // as floating point and we don't care about reporting changes in that. We do + // however care about reporting changes to the actual set of exhaustions. + return (blockFeedInCluster == other.blockFeedInCluster && + Objects.equals(concreteExhaustions, other.concreteExhaustions)); + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; FeedBlock feedBlock = (FeedBlock) o; - return (blockFeedInCluster == feedBlock.blockFeedInCluster && - Objects.equals(description, feedBlock.description)); + return blockFeedInCluster == feedBlock.blockFeedInCluster && + Objects.equals(description, feedBlock.description) && + Objects.equals(concreteExhaustions, feedBlock.concreteExhaustions); } @Override public int hashCode() { - return Objects.hash(blockFeedInCluster, description); + return Objects.hash(blockFeedInCluster, description, concreteExhaustions); } } @@ -229,6 +256,9 @@ public class ClusterStateBundle { if (clusterFeedIsBlocked() != other.clusterFeedIsBlocked()) { return false; } + if (clusterFeedIsBlocked() && !feedBlock.similarTo(other.feedBlock)) { + return false; + } // FIXME we currently treat mismatching bucket space sets as unchanged to avoid breaking some tests return derivedBucketSpaceStates.entrySet().stream() .allMatch(entry -> other.derivedBucketSpaceStates.getOrDefault(entry.getKey(), entry.getValue()) diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 6dc4b1e8015..e238303b58b 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -346,12 +346,11 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd return; } // TODO hysteresis to prevent oscillations! - // TODO also ensure we trigger if CC options have changed var calc = createResourceExhaustionCalculator(); // Important: nodeInfo contains the _current_ host info _prior_ to newHostInfo being applied. - boolean previouslyExhausted = !calc.enumerateNodeResourceExhaustions(nodeInfo).isEmpty(); - boolean nowExhausted = !calc.resourceExhaustionsFromHostInfo(nodeInfo.getNode(), newHostInfo).isEmpty(); - if (previouslyExhausted != nowExhausted) { + var previouslyExhausted = calc.enumerateNodeResourceExhaustions(nodeInfo); + var nowExhausted = calc.resourceExhaustionsFromHostInfo(nodeInfo, newHostInfo); + if (!previouslyExhausted.equals(nowExhausted)) { log.fine(() -> String.format("Triggering state recomputation due to change in cluster feed block: %s -> %s", previouslyExhausted, nowExhausted)); stateChangeHandler.setStateChangedFlag(); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeResourceExhaustion.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeResourceExhaustion.java index 609fea2b91e..79f04627073 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeResourceExhaustion.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeResourceExhaustion.java @@ -15,13 +15,16 @@ public class NodeResourceExhaustion { public final String resourceType; public final ResourceUsage resourceUsage; public final double limit; + public final String rpcAddress; public NodeResourceExhaustion(Node node, String resourceType, - ResourceUsage resourceUsage, double limit) { + ResourceUsage resourceUsage, double limit, + String rpcAddress) { this.node = node; this.resourceType = resourceType; this.resourceUsage = resourceUsage; this.limit = limit; + this.rpcAddress = rpcAddress; } @Override @@ -32,11 +35,12 @@ public class NodeResourceExhaustion { return Double.compare(that.limit, limit) == 0 && Objects.equals(node, that.node) && Objects.equals(resourceType, that.resourceType) && - Objects.equals(resourceUsage, that.resourceUsage); + Objects.equals(resourceUsage, that.resourceUsage) && + Objects.equals(rpcAddress, that.rpcAddress); } @Override public int hashCode() { - return Objects.hash(node, resourceType, resourceUsage, limit); + return Objects.hash(node, resourceType, resourceUsage, limit, rpcAddress); } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java index 80b8a6110f1..e2e61eb8ed0 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java @@ -1,14 +1,17 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; +import com.yahoo.jrt.Spec; import com.yahoo.vdslib.state.Node; import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; /** @@ -37,43 +40,65 @@ public class ResourceExhaustionCalculator { int maxDescriptions = 3; String description = exhaustions.stream() .limit(maxDescriptions) - .map(n -> String.format("%s on node %s (%.3g > %.3g)", - n.resourceType, n.node.getIndex(), - n.resourceUsage.getUsage(), n.limit)) + .map(ResourceExhaustionCalculator::formatNodeResourceExhaustion) .collect(Collectors.joining(", ")); if (exhaustions.size() > maxDescriptions) { description += String.format(" (... and %d more)", exhaustions.size() - maxDescriptions); } - return ClusterStateBundle.FeedBlock.blockedWithDescription(description); + // FIXME we currently will trigger a cluster state recomputation even if the number of + // exhaustions is greater than what is returned as part of the description. Though at + // that point, cluster state recomputations will be the least of your worries...! + return ClusterStateBundle.FeedBlock.blockedWith(description, exhaustions); } - public List<NodeResourceExhaustion> resourceExhaustionsFromHostInfo(Node node, HostInfo hostInfo) { - List<NodeResourceExhaustion> exceedingLimit = null; + private static String formatNodeResourceExhaustion(NodeResourceExhaustion n) { + return String.format("%s%s on node %d [%s] (%.3g > %.3g)", + n.resourceType, + (n.resourceUsage.getName() != null ? ":" + n.resourceUsage.getName() : ""), + n.node.getIndex(), + inferHostnameFromRpcAddress(n.rpcAddress), + n.resourceUsage.getUsage(), n.limit); + } + + private static String inferHostnameFromRpcAddress(String rpcAddress) { + if (rpcAddress == null) { + return "unknown hostname"; + } + var spec = new Spec(rpcAddress); + if (spec.malformed()) { + return "unknown hostname"; + } + return spec.host(); + } + + public Set<NodeResourceExhaustion> resourceExhaustionsFromHostInfo(NodeInfo nodeInfo, HostInfo hostInfo) { + Set<NodeResourceExhaustion> exceedingLimit = null; for (var usage : hostInfo.getContentNode().getResourceUsage().entrySet()) { double limit = feedBlockLimits.getOrDefault(usage.getKey(), 1.0); if (usage.getValue().getUsage() > limit) { if (exceedingLimit == null) { - exceedingLimit = new ArrayList<>(); + exceedingLimit = new LinkedHashSet<>(); } - exceedingLimit.add(new NodeResourceExhaustion(node, usage.getKey(), usage.getValue(), limit)); + exceedingLimit.add(new NodeResourceExhaustion(nodeInfo.getNode(), usage.getKey(), usage.getValue(), + limit, nodeInfo.getRpcAddress())); } } - return (exceedingLimit != null) ? exceedingLimit : Collections.emptyList(); + return (exceedingLimit != null) ? exceedingLimit : Collections.emptySet(); } - public List<NodeResourceExhaustion> enumerateNodeResourceExhaustions(NodeInfo nodeInfo) { + public Set<NodeResourceExhaustion> enumerateNodeResourceExhaustions(NodeInfo nodeInfo) { if (!nodeInfo.isStorage()) { - return Collections.emptyList(); + return Collections.emptySet(); } - return resourceExhaustionsFromHostInfo(nodeInfo.getNode(), nodeInfo.getHostInfo()); + return resourceExhaustionsFromHostInfo(nodeInfo, nodeInfo.getHostInfo()); } // Returns 0-n entries per content node in the cluster, where n is the number of exhausted // resource types on any given node. - public List<NodeResourceExhaustion> enumerateNodeResourceExhaustionsAcrossAllNodes(Collection<NodeInfo> nodeInfos) { + public Set<NodeResourceExhaustion> enumerateNodeResourceExhaustionsAcrossAllNodes(Collection<NodeInfo> nodeInfos) { return nodeInfos.stream() .flatMap(info -> enumerateNodeResourceExhaustions(info).stream()) - .collect(Collectors.toList()); + .collect(Collectors.toCollection(LinkedHashSet::new)); } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/ResourceUsage.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/ResourceUsage.java index e47ec5452a4..da0862d7de9 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/ResourceUsage.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/ResourceUsage.java @@ -8,12 +8,20 @@ import java.util.Objects; /** * Encapsulation of the usage levels for a particular resource type. The resource type * itself is not tracked in this class; this must be done on a higher level. + * + * Note: equality checks and hash code computations do NOT include the actual floating + * point usage! This is so sets of ResourceUsages are de-duplicated at the resource level + * regardless of the relative usage (all cases where these are compared is assumed to + * be when feed is blocked anyway, so just varying levels over the feed block limit). */ public class ResourceUsage { private final Double usage; + private final String name; - public ResourceUsage(@JsonProperty("usage") Double usage) { + public ResourceUsage(@JsonProperty("usage") Double usage, + @JsonProperty("name") String name) { this.usage = usage; + this.name = name; } /** Resource usage in [0, 1] */ @@ -21,16 +29,20 @@ public class ResourceUsage { return usage; } + public String getName() { + return name; + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; ResourceUsage that = (ResourceUsage) o; - return Objects.equals(usage, that.usage); + return Objects.equals(name, that.name); } @Override public int hashCode() { - return Objects.hash(usage); + return Objects.hash(name); } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java index 2ac7113741b..8dd2a9ca55c 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java @@ -17,10 +17,13 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.mapOf; +import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.setOf; import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.usage; import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.createResourceUsageJson; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -89,7 +92,7 @@ public class ClusterFeedBlockTest extends FleetControllerTest { return options; } - private void reportResourceUsageFromNode(int nodeIndex, Map<String, Double> resourceUsages) throws Exception { + private void reportResourceUsageFromNode(int nodeIndex, Set<FeedBlockUtil.UsageDetails> resourceUsages) throws Exception { String hostInfo = createResourceUsageJson(resourceUsages); communicator.setNodeState(new Node(NodeType.STORAGE, nodeIndex), new NodeState(NodeType.STORAGE, State.UP), hostInfo); ctrl.tick(); @@ -102,17 +105,17 @@ public class ClusterFeedBlockTest extends FleetControllerTest { assertFalse(ctrl.getClusterStateBundle().clusterFeedIsBlocked()); // Too much cheese in use, must block feed! - reportResourceUsageFromNode(1, mapOf(usage("cheese", 0.8), usage("wine", 0.3))); + reportResourceUsageFromNode(1, setOf(usage("cheese", 0.8), usage("wine", 0.3))); assertTrue(ctrl.getClusterStateBundle().clusterFeedIsBlocked()); // TODO check desc? // Wine usage has gone up too, we should remain blocked - reportResourceUsageFromNode(1, mapOf(usage("cheese", 0.8), usage("wine", 0.5))); + reportResourceUsageFromNode(1, setOf(usage("cheese", 0.8), usage("wine", 0.5))); assertTrue(ctrl.getClusterStateBundle().clusterFeedIsBlocked()); // TODO check desc? // Back to normal wine and cheese levels - reportResourceUsageFromNode(1, mapOf(usage("cheese", 0.6), usage("wine", 0.3))); + reportResourceUsageFromNode(1, setOf(usage("cheese", 0.6), usage("wine", 0.3))); assertFalse(ctrl.getClusterStateBundle().clusterFeedIsBlocked()); } @@ -121,7 +124,7 @@ public class ClusterFeedBlockTest extends FleetControllerTest { initialize(createOptions(mapOf(usage("cheese", 0.7), usage("wine", 0.4)))); assertFalse(ctrl.getClusterStateBundle().clusterFeedIsBlocked()); - reportResourceUsageFromNode(1, mapOf(usage("cheese", 0.8), usage("wine", 0.3))); + reportResourceUsageFromNode(1, setOf(usage("cheese", 0.8), usage("wine", 0.3))); assertTrue(ctrl.getClusterStateBundle().clusterFeedIsBlocked()); // Increase cheese allowance. Should now automatically unblock since reported usage is lower. @@ -131,4 +134,39 @@ public class ClusterFeedBlockTest extends FleetControllerTest { assertFalse(ctrl.getClusterStateBundle().clusterFeedIsBlocked()); } + @Test + public void cluster_feed_block_state_is_recomputed_when_resource_block_set_differs() throws Exception { + initialize(createOptions(mapOf(usage("cheese", 0.7), usage("wine", 0.4)))); + assertFalse(ctrl.getClusterStateBundle().clusterFeedIsBlocked()); + + reportResourceUsageFromNode(1, setOf(usage("cheese", 0.8), usage("wine", 0.3))); + var bundle = ctrl.getClusterStateBundle(); + assertTrue(bundle.clusterFeedIsBlocked()); + assertEquals("cheese on node 1 [unknown hostname] (0.800 > 0.700)", bundle.getFeedBlock().get().getDescription()); + + reportResourceUsageFromNode(1, setOf(usage("cheese", 0.8), usage("wine", 0.5))); + bundle = ctrl.getClusterStateBundle(); + assertTrue(bundle.clusterFeedIsBlocked()); + assertEquals("cheese on node 1 [unknown hostname] (0.800 > 0.700), " + + "wine on node 1 [unknown hostname] (0.500 > 0.400)", + bundle.getFeedBlock().get().getDescription()); + } + + @Test + public void cluster_feed_block_state_is_not_recomputed_when_only_resource_usage_levels_differ() throws Exception { + initialize(createOptions(mapOf(usage("cheese", 0.7), usage("wine", 0.4)))); + assertFalse(ctrl.getClusterStateBundle().clusterFeedIsBlocked()); + + reportResourceUsageFromNode(1, setOf(usage("cheese", 0.8), usage("wine", 0.3))); + var bundle = ctrl.getClusterStateBundle(); + assertTrue(bundle.clusterFeedIsBlocked()); + assertEquals("cheese on node 1 [unknown hostname] (0.800 > 0.700)", bundle.getFeedBlock().get().getDescription()); + + // 80% -> 90%, should not trigger new state. + reportResourceUsageFromNode(1, setOf(usage("cheese", 0.9), usage("wine", 0.4))); + bundle = ctrl.getClusterStateBundle(); + assertTrue(bundle.clusterFeedIsBlocked()); + assertEquals("cheese on node 1 [unknown hostname] (0.800 > 0.700)", bundle.getFeedBlock().get().getDescription()); + } + } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFixture.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFixture.java index a6cf10d4022..35cfd82f367 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFixture.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFixture.java @@ -150,7 +150,11 @@ public class ClusterFixture { } public ClusterFixture assignDummyRpcAddresses() { - cluster.getNodeInfo().forEach(ni -> ni.setRpcAddress("tcp/localhost:0")); + cluster.getNodeInfo().forEach(ni -> { + ni.setRpcAddress(String.format("tcp/%s.%d.local:0", + ni.isStorage() ? "storage" : "distributor", + ni.getNodeIndex())); + }); return this; } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundleTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundleTest.java index d2db47131bd..ceddf7cdcf3 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundleTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundleTest.java @@ -6,9 +6,14 @@ import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; +import com.yahoo.vespa.clustercontroller.core.hostinfo.ResourceUsage; import org.junit.Test; +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.Set; import java.util.function.Function; +import java.util.stream.Collectors; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; @@ -53,6 +58,12 @@ public class ClusterStateBundleTest { .deriveAndBuild(); } + private static ClusterStateBundle createTestBundleWithFeedBlock(String description, Set<NodeResourceExhaustion> concreteExhaustions) { + return createTestBundleBuilder(false) + .feedBlock(ClusterStateBundle.FeedBlock.blockedWith(description, concreteExhaustions)) + .deriveAndBuild(); + } + private static ClusterStateBundle createTestBundle() { return createTestBundle(true); } @@ -109,6 +120,29 @@ public class ClusterStateBundleTest { assertTrue(blockingBundle.similarTo(blockingBundleWithOtherDesc)); } + static NodeResourceExhaustion createDummyExhaustion(String type) { + return new NodeResourceExhaustion(new Node(NodeType.STORAGE, 1), type, new ResourceUsage(0.8, null), 0.7, "foo"); + } + + static Set<NodeResourceExhaustion> exhaustionsOf(String... types) { + return Arrays.stream(types) + .map(t -> createDummyExhaustion(t)) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + + @Test + public void similarity_test_considers_cluster_feed_block_concrete_exhaustion_set() { + var blockingBundleNoSet = createTestBundleWithFeedBlock("foo"); + var blockingBundleWithSet = createTestBundleWithFeedBlock("bar", exhaustionsOf("beer", "wine")); + var blockingBundleWithOtherSet = createTestBundleWithFeedBlock("bar", exhaustionsOf("beer", "soda")); + + assertTrue(blockingBundleNoSet.similarTo(blockingBundleNoSet)); + assertTrue(blockingBundleWithSet.similarTo(blockingBundleWithSet)); + assertFalse(blockingBundleWithSet.similarTo(blockingBundleWithOtherSet)); + assertFalse(blockingBundleNoSet.similarTo(blockingBundleWithSet)); + assertFalse(blockingBundleNoSet.similarTo(blockingBundleWithOtherSet)); + } + @Test public void feed_block_state_is_available() { var nonBlockingBundle = createTestBundle(false); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FeedBlockUtil.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FeedBlockUtil.java index e2894705352..a26f8791cda 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FeedBlockUtil.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FeedBlockUtil.java @@ -2,46 +2,78 @@ package com.yahoo.vespa.clustercontroller.core; import java.util.Arrays; +import java.util.LinkedHashSet; import java.util.Map; +import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; public class FeedBlockUtil { static class NodeAndUsages { public final int index; - public final Map<String, Double> usages; + public final Set<UsageDetails> usages; - public NodeAndUsages(int index, Map<String, Double> usages) { + public NodeAndUsages(int index, Set<UsageDetails> usages) { this.index = index; this.usages = usages; } } - static class NameAndUsage { + static class UsageDetails { + public final String type; public final String name; public final double usage; - public NameAndUsage(String name, double usage) { + public UsageDetails(String type, String name, double usage) { + this.type = type; this.name = name; this.usage = usage; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + UsageDetails that = (UsageDetails) o; + return Double.compare(that.usage, usage) == 0 && + Objects.equals(type, that.type) && + Objects.equals(name, that.name); + } + + @Override + public int hashCode() { + return Objects.hash(type, name, usage); + } + } + + static UsageDetails usage(String type, double usage) { + return new UsageDetails(type, null, usage); + } + + static UsageDetails usage(String type, String name, double usage) { + return new UsageDetails(type, name, usage); } - static NameAndUsage usage(String name, double usage) { - return new NameAndUsage(name, usage); + static Map<String, Double> mapOf(UsageDetails... usages) { + return Arrays.stream(usages).collect(Collectors.toMap(u -> u.type, u -> u.usage)); } - static Map<String, Double> mapOf(NameAndUsage... usages) { - return Arrays.stream(usages).collect(Collectors.toMap(u -> u.name, u -> u.usage)); + static Set<UsageDetails> setOf(UsageDetails... usages) { + // Preserve input order to make stringification tests deterministic + return Arrays.stream(usages).collect(Collectors.toCollection(LinkedHashSet::new)); } - static NodeAndUsages forNode(int index, NameAndUsage... usages) { - return new NodeAndUsages(index, mapOf(usages)); + static NodeAndUsages forNode(int index, UsageDetails... usages) { + return new NodeAndUsages(index, setOf(usages)); } - static String createResourceUsageJson(Map<String, Double> usages) { - String usageInnerJson = usages.entrySet().stream() - .map(kv -> String.format("\"%s\":{\"usage\": %.3g}", kv.getKey(), kv.getValue())) + static String createResourceUsageJson(Set<UsageDetails> usages) { + // We deal only in the finest of manual JSON string building technologies(tm). + String usageInnerJson = usages.stream() + .map(u -> String.format("\"%s\":{\"usage\": %.3g%s}", + u.type, u.usage, + (u.name != null ? String.format(",\"name\":\"%s\"", u.name) : ""))) .collect(Collectors.joining(",")); return String.format("{\"content-node\":{\"resource-usage\":{%s}}}", usageInnerJson); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculatorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculatorTest.java index 5a5cda1f4ed..54686919a7b 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculatorTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculatorTest.java @@ -10,6 +10,7 @@ import static com.yahoo.vespa.clustercontroller.core.ClusterFixture.storageNode; import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.NodeAndUsages; import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.forNode; import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.mapOf; +import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.setOf; import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.usage; import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.createResourceUsageJson; @@ -25,7 +26,10 @@ public class ResourceExhaustionCalculatorTest { if (highestIndex.isEmpty()) { throw new IllegalArgumentException("Can't have an empty cluster"); } - var cf = ClusterFixture.forFlatCluster(highestIndex.getAsInt() + 1).bringEntireClusterUp(); + var cf = ClusterFixture + .forFlatCluster(highestIndex.getAsInt() + 1) + .assignDummyRpcAddresses() + .bringEntireClusterUp(); for (var nu : nodeAndUsages) { cf.cluster().getNodeInfo(storageNode(nu.index)) .setHostInfo(HostInfo.createHostInfo(createResourceUsageJson(nu.usages))); @@ -50,7 +54,32 @@ public class ResourceExhaustionCalculatorTest { var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); assertNotNull(feedBlock); assertTrue(feedBlock.blockFeedInCluster()); - assertEquals("disk on node 1 (0.510 > 0.500)", feedBlock.getDescription()); + assertEquals("disk on node 1 [storage.1.local] (0.510 > 0.500)", feedBlock.getDescription()); + } + + @Test + public void feed_block_description_can_contain_optional_name_component() { + var calc = new ResourceExhaustionCalculator(true, mapOf(usage("disk", 0.5), usage("memory", 0.8))); + var cf = createFixtureWithReportedUsages(forNode(1, usage("disk", "a-fancy-disk", 0.51), usage("memory", 0.79)), + forNode(2, usage("disk", 0.4), usage("memory", 0.6))); + var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); + assertNotNull(feedBlock); + assertTrue(feedBlock.blockFeedInCluster()); + assertEquals("disk:a-fancy-disk on node 1 [storage.1.local] (0.510 > 0.500)", feedBlock.getDescription()); + } + + @Test + public void missing_or_malformed_rpc_addresses_are_emitted_as_unknown_hostnames() { + var calc = new ResourceExhaustionCalculator(true, mapOf(usage("disk", 0.5), usage("memory", 0.8))); + var cf = createFixtureWithReportedUsages(forNode(1, usage("disk", 0.51), usage("memory", 0.79)), + forNode(2, usage("disk", 0.4), usage("memory", 0.85))); + cf.cluster().getNodeInfo(storageNode(1)).setRpcAddress(null); + cf.cluster().getNodeInfo(storageNode(2)).setRpcAddress("max mekker"); + var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); + assertNotNull(feedBlock); + assertTrue(feedBlock.blockFeedInCluster()); + assertEquals("disk on node 1 [unknown hostname] (0.510 > 0.500), " + + "memory on node 2 [unknown hostname] (0.850 > 0.800)", feedBlock.getDescription()); } @Test @@ -61,9 +90,9 @@ public class ResourceExhaustionCalculatorTest { var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); assertNotNull(feedBlock); assertTrue(feedBlock.blockFeedInCluster()); - assertEquals("disk on node 1 (0.510 > 0.400), " + - "memory on node 1 (0.850 > 0.800), " + - "disk on node 2 (0.450 > 0.400)", + assertEquals("disk on node 1 [storage.1.local] (0.510 > 0.400), " + + "memory on node 1 [storage.1.local] (0.850 > 0.800), " + + "disk on node 2 [storage.2.local] (0.450 > 0.400)", feedBlock.getDescription()); } @@ -76,9 +105,9 @@ public class ResourceExhaustionCalculatorTest { var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); assertNotNull(feedBlock); assertTrue(feedBlock.blockFeedInCluster()); - assertEquals("disk on node 1 (0.510 > 0.400), " + - "memory on node 1 (0.850 > 0.800), " + - "disk on node 2 (0.450 > 0.400) (... and 2 more)", + assertEquals("disk on node 1 [storage.1.local] (0.510 > 0.400), " + + "memory on node 1 [storage.1.local] (0.850 > 0.800), " + + "disk on node 2 [storage.2.local] (0.450 > 0.400) (... and 2 more)", feedBlock.getDescription()); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfoTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfoTest.java index f9b0a4ca36f..b2d3eb54f78 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfoTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfoTest.java @@ -75,6 +75,7 @@ public class HostInfoTest { assertEquals(resourceUsage.size(), 2); assertEquals(Optional.ofNullable(resourceUsage.get("memory")).map(ResourceUsage::getUsage).orElse(0.0), 0.85, 0.00001); assertEquals(Optional.ofNullable(resourceUsage.get("disk")).map(ResourceUsage::getUsage).orElse(0.0), 0.6, 0.00001); + assertEquals(Optional.ofNullable(resourceUsage.get("disk")).map(ResourceUsage::getName).orElse("missing"), "a cool disk"); assertNull(resourceUsage.get("flux-capacitor")); } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index a9a83386573..6517c3e57fc 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -64,8 +64,8 @@ public interface ModelContext { * - Remove below method once all config-model versions in hosted production include changes from 1) */ interface FeatureFlags { - @ModelFeatureFlag(owners = {"bjorncs", "jonmv"}) default boolean enableAutomaticReindexing() { return false; } - @ModelFeatureFlag(owners = {"bjorncs", "jonmv"}) default double reindexerWindowSizeIncrement() { return 0.2; } + @ModelFeatureFlag(owners = {"bjorncs", "jonmv"}, removeAfter = "7.352") default boolean enableAutomaticReindexing() { return true; } + @ModelFeatureFlag(owners = {"jonmv"}) default double reindexerWindowSizeIncrement() { return 0.2; } @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Revisit in May or June 2021") default double defaultTermwiseLimit() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"vekterli"}) default boolean useThreePhaseUpdates() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Select sequencer type use while feeding") default String feedSequencerType() { throw new UnsupportedOperationException("TODO specify default value"); } @@ -77,7 +77,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"tokle"}) default boolean useAccessControlTlsHandshakeClientAuth() { return false; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useAsyncMessageHandlingOnSchedule() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.348") default int contentNodeBucketDBStripeBits() { return 4; } - @ModelFeatureFlag(owners = {"baldersheim"}) default int mergeChunkSize() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.350") default int mergeChunkSize() { return 0x2000000; } @ModelFeatureFlag(owners = {"baldersheim"}) default double feedConcurrency() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useBucketExecutorForLidSpaceCompact() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"musum", "mpolden"}, comment = "Revisit in February 2021") default boolean reconfigurableZookeeperServer() { return false; } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 686cb7ce7c6..6ca1c8f2b79 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -47,7 +47,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private Quota quota = Quota.unlimited(); private boolean useAccessControlTlsHandshakeClientAuth; private boolean useAsyncMessageHandlingOnSchedule = false; - private int mergeChunkSize = 0x400000 - 0x1000; // 4M -4k private double feedConcurrency = 0.5; private boolean enableAutomaticReindexing = false; private boolean reconfigurableZookeeperServer = false; @@ -82,7 +81,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public Quota quota() { return quota; } @Override public boolean useAccessControlTlsHandshakeClientAuth() { return useAccessControlTlsHandshakeClientAuth; } @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; } - @Override public int mergeChunkSize() { return mergeChunkSize; } @Override public double feedConcurrency() { return feedConcurrency; } @Override public boolean enableAutomaticReindexing() { return enableAutomaticReindexing; } @Override public boolean reconfigurableZookeeperServer() { return reconfigurableZookeeperServer; } @@ -94,11 +92,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setMergeChunkSize(int size) { - mergeChunkSize = size; - return this; - } - public TestProperties setAsyncMessageHandlingOnSchedule(boolean value) { useAsyncMessageHandlingOnSchedule = value; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java index 3fe6ce3ff27..8423f7d723a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java @@ -32,9 +32,7 @@ public class ClusterControllerContainerCluster extends ContainerCluster<ClusterC public ReindexingContext reindexingContext() { return reindexingContext; } private static ReindexingContext createReindexingContext(DeployState deployState) { - Reindexing reindexing = deployState.featureFlags().enableAutomaticReindexing() - ? deployState.reindexing().orElse(Reindexing.DISABLED_INSTANCE) - : Reindexing.DISABLED_INSTANCE; + Reindexing reindexing = deployState.reindexing().orElse(Reindexing.DISABLED_INSTANCE); return new ReindexingContext(reindexing, deployState.featureFlags().reindexerWindowSizeIncrement()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidator.java index 0add9f243fe..a462cb4fdb3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidator.java @@ -39,7 +39,7 @@ public class NodeResourceChangeValidator implements ChangeValidator { } private boolean changeRequiresRestart(NodeResources currentResources, NodeResources nextResources) { - return currentResources.memoryGb() != nextResources.memoryGb(); + return !currentResources.equals(nextResources); } private Optional<NodeResources> resourcesOf(ClusterSpec.Id clusterId, VespaModel model) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java index 5074f1ecbbe..911e5e2e9bd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerThreadpool.java @@ -22,8 +22,6 @@ public class ContainerThreadpool extends SimpleComponent implements ContainerThr private final String name; private final UserOptions userOptions; - public ContainerThreadpool(String name) { this(name, null); } - public ContainerThreadpool(String name, UserOptions userOptions) { super(new ComponentModel( BundleInstantiationSpecification.getFromStrings( diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java index 2ca97b297b0..3dae3160c51 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/DefaultThreadpoolProvider.java @@ -5,6 +5,7 @@ import com.yahoo.container.bundle.BundleInstantiationSpecification; import com.yahoo.container.handler.ThreadPoolProvider; import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster; import com.yahoo.vespa.model.container.component.SimpleComponent; /** @@ -25,11 +26,18 @@ class DefaultThreadpoolProvider extends SimpleComponent implements ThreadpoolCon this.cluster = cluster; } + private int defaultThreadsByClusterType() { + if (cluster instanceof MetricsProxyContainerCluster) { + return 4; + } + return 10; + } + @Override public void getConfig(ThreadpoolConfig.Builder builder) { if (!(cluster instanceof ApplicationContainerCluster)) { // Container clusters such as logserver, metricsproxy and clustercontroller - int defaultWorkerThreads = 10; + int defaultWorkerThreads = defaultThreadsByClusterType(); builder.maxthreads(defaultWorkerThreads); builder.corePoolSize(defaultWorkerThreads); builder.queueSize(50); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java index 6a858bd2e02..a66d98ad52c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/AccessLogBuilder.java @@ -82,6 +82,12 @@ public class AccessLogBuilder { } private static CompressionType compressionType(Element spec, DeployState deployState, boolean isHostedVespa) { + CompressionType fallback; + if (isHostedVespa && deployState.featureFlags().enableZstdCompressionAccessLog()) { + fallback = CompressionType.ZSTD; + } else { + fallback = CompressionType.GZIP; + } return Optional.ofNullable(spec.getAttribute("compressionType")) .filter(value -> !value.isBlank()) .map(value -> { @@ -94,7 +100,7 @@ public class AccessLogBuilder { throw new IllegalArgumentException("Unknown compression type: " + value); } }) - .orElse(isHostedVespa && deployState.featureFlags().enableZstdCompressionAccessLog() ? CompressionType.ZSTD : CompressionType.GZIP); + .orElse(fallback); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java index ab734c506c1..8be57ff3bde 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java @@ -2,8 +2,12 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.config.model.ConfigModelContext; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.container.logging.FileConnectionLog; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; +import com.yahoo.vespa.model.container.component.AccessLogComponent; +import com.yahoo.vespa.model.container.component.ConnectionLogComponent; import com.yahoo.vespa.model.container.configserver.ConfigserverCluster; import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; import org.w3c.dom.Element; @@ -34,7 +38,22 @@ public class ConfigServerContainerModelBuilder extends ContainerModelBuilder { // in ConfigModelContext.DeployState.properties are not set) @Override protected void addStatusHandlers(ApplicationContainerCluster cluster, boolean isHostedVespa) { - super.addStatusHandlers(cluster, options.hostedVespa().orElse(Boolean.FALSE)); + super.addStatusHandlers(cluster, isHosted()); } + // Override access log configuration for hosted configserver/controller + @Override + protected void addAccessLogs(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { + if (isHosted()){ + cluster.addComponent( + new AccessLogComponent( + AccessLogComponent.AccessLogType.jsonAccessLog, AccessLogComponent.CompressionType.ZSTD, + "logs/vespa/configserver/access-json.log.%Y%m%d%H%M%S", null, true, true, "access-json.log")); + cluster.addComponent(new ConnectionLogComponent(FileConnectionLog.class, cluster.getName())); + } else { + super.addAccessLogs(deployState, cluster, spec); + } + } + + private boolean isHosted() { return options.hostedVespa().orElse(Boolean.FALSE); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index d650b10a910..7477ccdd970 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -338,7 +338,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addConfiguredComponents(deployState, cluster, spec, "server"); } - private void addAccessLogs(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { + protected void addAccessLogs(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { List<Element> accessLogElements = getAccessLogElements(spec); for (Element accessLog : accessLogElements) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java index 8d6127970c8..66ec0d81947 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java @@ -19,12 +19,14 @@ import org.w3c.dom.Element; public class ClusterControllerConfig extends AbstractConfigProducer<ClusterControllerConfig> implements FleetcontrollerConfig.Producer { public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<ClusterControllerConfig> { - String clusterName; - ModelElement clusterElement; + private final String clusterName; + private final ModelElement clusterElement; + private final ResourceLimits resourceLimits; - public Builder(String clusterName, ModelElement clusterElement) { + public Builder(String clusterName, ModelElement clusterElement, ResourceLimits resourceLimits) { this.clusterName = clusterName; this.clusterElement = clusterElement; + this.resourceLimits = resourceLimits; } @Override @@ -51,27 +53,29 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr tuning.childAsDouble("min-storage-up-ratio"), bucketSplittingMinimumBits, minNodeRatioPerGroup, - enableClusterFeedBlock); + enableClusterFeedBlock, + resourceLimits); } else { return new ClusterControllerConfig(ancestor, clusterName, null, null, null, null, null, null, bucketSplittingMinimumBits, minNodeRatioPerGroup, - enableClusterFeedBlock); + enableClusterFeedBlock, resourceLimits); } } } - String clusterName; - Duration initProgressTime; - Duration transitionTime; - Long maxPrematureCrashes; - Duration stableStateTimePeriod; - Double minDistributorUpRatio; - Double minStorageUpRatio; - Integer minSplitBits; - private Double minNodeRatioPerGroup; - private boolean enableClusterFeedBlock = false; + private final String clusterName; + private final Duration initProgressTime; + private final Duration transitionTime; + private final Long maxPrematureCrashes; + private final Duration stableStateTimePeriod; + private final Double minDistributorUpRatio; + private final Double minStorageUpRatio; + private final Integer minSplitBits; + private final Double minNodeRatioPerGroup; + private final boolean enableClusterFeedBlock; + private final ResourceLimits resourceLimits; // TODO refactor; too many args private ClusterControllerConfig(AbstractConfigProducer parent, @@ -84,7 +88,8 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr Double minStorageUpRatio, Integer minSplitBits, Double minNodeRatioPerGroup, - boolean enableClusterFeedBlock) { + boolean enableClusterFeedBlock, + ResourceLimits resourceLimits) { super(parent, "fleetcontroller"); this.clusterName = clusterName; @@ -97,6 +102,7 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr this.minSplitBits = minSplitBits; this.minNodeRatioPerGroup = minNodeRatioPerGroup; this.enableClusterFeedBlock = enableClusterFeedBlock; + this.resourceLimits = resourceLimits; } @Override @@ -139,18 +145,7 @@ public class ClusterControllerConfig extends AbstractConfigProducer<ClusterContr builder.min_node_ratio_per_group(minNodeRatioPerGroup); } builder.enable_cluster_feed_block(enableClusterFeedBlock); - setDefaultClusterFeedBlockLimits(builder); + resourceLimits.getConfig(builder); } - private static void setDefaultClusterFeedBlockLimits(FleetcontrollerConfig.Builder builder) { - // TODO: Override these based on resource-limits in services.xml (if they are specified). - // TODO: Choose other defaults when this is default enabled. - // Note: The resource categories must match the ones used in host info reporting - // between content nodes and cluster controller: - // storage/src/vespa/storage/persistence/filestorage/service_layer_host_info_reporter.cpp - builder.cluster_feed_block_limit.put("memory", 0.79); - builder.cluster_feed_block_limit.put("disk", 0.79); - builder.cluster_feed_block_limit.put("attribute-enum-store", 0.89); - builder.cluster_feed_block_limit.put("attribute-multi-value", 0.89); - } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java new file mode 100644 index 00000000000..5324ee171ec --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterResourceLimits.java @@ -0,0 +1,103 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import com.yahoo.vespa.model.builder.xml.dom.ModelElement; +import com.yahoo.vespa.model.content.cluster.DomResourceLimitsBuilder; + +import java.util.Optional; +import java.util.function.Consumer; + +/** + * Class tracking the feed block resource limits for a content cluster. + * + * This includes the limits used by the cluster controller and the content nodes (proton). + * + * @author geirst + */ +public class ClusterResourceLimits { + + private final ResourceLimits clusterControllerLimits; + private final ResourceLimits contentNodeLimits; + + private ClusterResourceLimits(Builder builder) { + clusterControllerLimits = builder.ctrlBuilder.build(); + contentNodeLimits = builder.nodeBuilder.build(); + } + + public ResourceLimits getClusterControllerLimits() { + return clusterControllerLimits; + } + + public ResourceLimits getContentNodeLimits() { + return contentNodeLimits; + } + + public static class Builder { + + private ResourceLimits.Builder ctrlBuilder = new ResourceLimits.Builder(); + private ResourceLimits.Builder nodeBuilder = new ResourceLimits.Builder(); + + public ClusterResourceLimits build(ModelElement clusterElem) { + + ModelElement tuningElem = clusterElem.childByPath("tuning"); + if (tuningElem != null) { + ctrlBuilder = DomResourceLimitsBuilder.createBuilder(tuningElem); + } + + ModelElement protonElem = clusterElem.childByPath("engine.proton"); + if (protonElem != null) { + nodeBuilder = DomResourceLimitsBuilder.createBuilder(protonElem); + } + + deriveLimits(); + return new ClusterResourceLimits(this); + } + + public void setClusterControllerBuilder(ResourceLimits.Builder builder) { + ctrlBuilder = builder; + } + + public void setContentNodeBuilder(ResourceLimits.Builder builder) { + nodeBuilder = builder; + } + + public ClusterResourceLimits build() { + deriveLimits(); + return new ClusterResourceLimits(this); + } + + private void deriveLimits() { + deriveClusterControllerLimit(ctrlBuilder.getDiskLimit(), nodeBuilder.getDiskLimit(), ctrlBuilder::setDiskLimit); + deriveClusterControllerLimit(ctrlBuilder.getMemoryLimit(), nodeBuilder.getMemoryLimit(), ctrlBuilder::setMemoryLimit); + + deriveContentNodeLimit(nodeBuilder.getDiskLimit(), ctrlBuilder.getDiskLimit(), nodeBuilder::setDiskLimit); + deriveContentNodeLimit(nodeBuilder.getMemoryLimit(), ctrlBuilder.getMemoryLimit(), nodeBuilder::setMemoryLimit); + } + + private void deriveClusterControllerLimit(Optional<Double> clusterControllerLimit, + Optional<Double> contentNodeLimit, + Consumer<Double> setter) { + if (!clusterControllerLimit.isPresent()) { + contentNodeLimit.ifPresent(limit -> + // TODO: emit warning when using cluster controller resource limits are default enabled. + setter.accept(limit)); + } + } + + private void deriveContentNodeLimit(Optional<Double> contentNodeLimit, + Optional<Double> clusterControllerLimit, + Consumer<Double> setter) { + if (!contentNodeLimit.isPresent()) { + clusterControllerLimit.ifPresent(limit -> + setter.accept(calcContentNodeLimit(limit))); + } + } + + private double calcContentNodeLimit(double clusterControllerLimit) { + // Note that validation in the range [0.0-1.0] is handled by the rnc schema. + return clusterControllerLimit + ((1.0 - clusterControllerLimit) / 2); + } + + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index dd29df61f35..d7df62d56cf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -79,13 +79,15 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private final Map<String, NewDocumentType> documentDefinitions; private final Set<NewDocumentType> globallyDistributedDocuments; private final boolean combined; + private final ResourceLimits resourceLimits; public Builder(Map<String, NewDocumentType> documentDefinitions, Set<NewDocumentType> globallyDistributedDocuments, - boolean combined) { + boolean combined, ResourceLimits resourceLimits) { this.documentDefinitions = documentDefinitions; this.globallyDistributedDocuments = globallyDistributedDocuments; this.combined = combined; + this.resourceLimits = resourceLimits; } @Override @@ -106,10 +108,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> if (tuning != null) { search.setTuning(new DomSearchTuningBuilder().build(deployState, search, tuning.getXml())); } - ModelElement protonElem = clusterElem.childByPath("engine.proton"); - if (protonElem != null) { - search.setResourceLimits(DomResourceLimitsBuilder.build(protonElem)); - } + search.setResourceLimits(resourceLimits); buildAllStreamingSearchClusters(deployState, clusterElem, clusterName, search); buildIndexedSearchCluster(deployState, clusterElem, clusterName, search); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java index 28e8c36d202..e96ba47c6b3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ResourceLimits.java @@ -1,16 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; +import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import java.util.Optional; /** - * Class tracking resource limits for a content cluster with engine proton. + * Class tracking feed block resource limits used by a component in a content cluster (e.g. cluster controller or content node). * * @author geirst */ -public class ResourceLimits implements ProtonConfig.Producer { +public class ResourceLimits implements FleetcontrollerConfig.Producer, ProtonConfig.Producer { private final Optional<Double> diskLimit; private final Optional<Double> memoryLimit; @@ -20,6 +21,26 @@ public class ResourceLimits implements ProtonConfig.Producer { this.memoryLimit = builder.memoryLimit; } + public Optional<Double> getDiskLimit() { + return diskLimit; + } + + public Optional<Double> getMemoryLimit() { + return memoryLimit; + } + + @Override + public void getConfig(FleetcontrollerConfig.Builder builder) { + // TODO: Choose other defaults when this is default enabled. + // Note: The resource categories must match the ones used in host info reporting + // between content nodes and cluster controller: + // storage/src/vespa/storage/persistence/filestorage/service_layer_host_info_reporter.cpp + builder.cluster_feed_block_limit.put("memory", memoryLimit.orElse(0.79)); + builder.cluster_feed_block_limit.put("disk", diskLimit.orElse(0.79)); + builder.cluster_feed_block_limit.put("attribute-enum-store", 0.89); + builder.cluster_feed_block_limit.put("attribute-multi-value", 0.89); + } + @Override public void getConfig(ProtonConfig.Builder builder) { if (diskLimit.isPresent()) { @@ -39,11 +60,19 @@ public class ResourceLimits implements ProtonConfig.Producer { return new ResourceLimits(this); } + public Optional<Double> getDiskLimit() { + return diskLimit; + } + public Builder setDiskLimit(double diskLimit) { this.diskLimit = Optional.of(diskLimit); return this; } + public Optional<Double> getMemoryLimit() { + return memoryLimit; + } + public Builder setMemoryLimit(double memoryLimit) { this.memoryLimit = Optional.of(memoryLimit); return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index a627e030156..44de4a1abec 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -38,6 +38,7 @@ import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; import com.yahoo.vespa.model.content.ClusterControllerConfig; +import com.yahoo.vespa.model.content.ClusterResourceLimits; import com.yahoo.vespa.model.content.ContentSearch; import com.yahoo.vespa.model.content.ContentSearchCluster; import com.yahoo.vespa.model.content.DistributionBitCalculator; @@ -134,11 +135,14 @@ public class ContentCluster extends AbstractConfigProducer implements ContentCluster c = new ContentCluster(context.getParentProducer(), getClusterId(contentElement), documentDefinitions, globallyDistributedDocuments, routingSelection, deployState.zone(), deployState.isHosted()); - c.clusterControllerConfig = new ClusterControllerConfig.Builder(getClusterId(contentElement), contentElement).build(deployState, c, contentElement.getXml()); + var resourceLimits = new ClusterResourceLimits.Builder().build(contentElement); + c.clusterControllerConfig = new ClusterControllerConfig.Builder(getClusterId(contentElement), + contentElement, + resourceLimits.getClusterControllerLimits()).build(deployState, c, contentElement.getXml()); c.search = new ContentSearchCluster.Builder(documentDefinitions, - globallyDistributedDocuments, - isCombined(getClusterId(contentElement), containers)) - .build(deployState, c, contentElement.getXml()); + globallyDistributedDocuments, + isCombined(getClusterId(contentElement), containers), + resourceLimits.getContentNodeLimits()).build(deployState, c, contentElement.getXml()); c.persistenceFactory = new EngineFactoryBuilder().build(contentElement, c); c.storageNodes = new StorageCluster.Builder().build(deployState, c, w3cContentElement); c.distributorNodes = new DistributorCluster.Builder(c).build(deployState, c, w3cContentElement); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java index 8e91f14238e..210f062f9b2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/DomResourceLimitsBuilder.java @@ -5,17 +5,17 @@ import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.content.ResourceLimits; /** - * Builder for resource limits for a content cluster with engine proton. + * Builder for feed block resource limits. * * @author geirst */ public class DomResourceLimitsBuilder { - public static ResourceLimits build(ModelElement contentXml) { + public static ResourceLimits.Builder createBuilder(ModelElement contentXml) { ResourceLimits.Builder builder = new ResourceLimits.Builder(); ModelElement resourceLimits = contentXml.child("resource-limits"); if (resourceLimits == null) { - return builder.build(); + return builder; } if (resourceLimits.child("disk") != null) { builder.setDiskLimit(resourceLimits.childAsDouble("disk")); @@ -23,7 +23,7 @@ public class DomResourceLimitsBuilder { if (resourceLimits.child("memory") != null) { builder.setMemoryLimit(resourceLimits.childAsDouble("memory")); } - return builder.build(); + return builder; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java index ab5dab4fbb9..57292b32a35 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java @@ -47,7 +47,6 @@ public class FileStorProducer implements StorFilestorConfig.Producer { private final int reponseNumThreads; private final StorFilestorConfig.Response_sequencer_type.Enum responseSequencerType; private final boolean useAsyncMessageHandlingOnSchedule; - private final int mergeChunkSize; private static StorFilestorConfig.Response_sequencer_type.Enum convertResponseSequencerType(String sequencerType) { try { @@ -56,17 +55,13 @@ public class FileStorProducer implements StorFilestorConfig.Producer { return StorFilestorConfig.Response_sequencer_type.Enum.ADAPTIVE; } } - private static int alignUp2MiB(int value) { - final int twoMB = 0x200000; - return ((value + twoMB - 1)/twoMB) * twoMB; - } + public FileStorProducer(ModelContext.FeatureFlags featureFlags, ContentCluster parent, Integer numThreads) { this.numThreads = numThreads; this.cluster = parent; this.reponseNumThreads = featureFlags.defaultNumResponseThreads(); this.responseSequencerType = convertResponseSequencerType(featureFlags.responseSequencerType()); useAsyncMessageHandlingOnSchedule = featureFlags.useAsyncMessageHandlingOnSchedule(); - mergeChunkSize = alignUp2MiB(featureFlags.mergeChunkSize()); // Align up to default huge page size. } @Override @@ -78,7 +73,6 @@ public class FileStorProducer implements StorFilestorConfig.Producer { builder.num_response_threads(reponseNumThreads); builder.response_sequencer_type(responseSequencerType); builder.use_async_message_handling_on_schedule(useAsyncMessageHandlingOnSchedule); - builder.bucket_merge_chunk_size(mergeChunkSize); } } diff --git a/config-model/src/main/python/ES_Vespa_parser.py b/config-model/src/main/python/ES_Vespa_parser.py index b3398fd0403..86df16981f0 100644 --- a/config-model/src/main/python/ES_Vespa_parser.py +++ b/config-model/src/main/python/ES_Vespa_parser.py @@ -32,7 +32,7 @@ class ElasticSearchParser: self.application_name = args.application_name def main(self): - self.path = os.getcwd() + "/application/" + self.path = os.getcwd() + "/" + self.application_name + "/" try: os.mkdir(self.path, 0o777) print(" > Created folder '" + self.path + "'") diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc index 5646bc72056..a48d38b9f2c 100644 --- a/config-model/src/main/resources/schema/content.rnc +++ b/config-model/src/main/resources/schema/content.rnc @@ -98,7 +98,8 @@ ClusterTuning = element tuning { ClusterControllerTuning? & Maintenance? & PersistenceThreads? & - MinNodeRatioPerGroup? + MinNodeRatioPerGroup? & + ResourceLimits? } Content = element content { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidatorTest.java index ecf026e7d88..180e4913d5c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidatorTest.java @@ -61,7 +61,7 @@ public class NodeResourceChangeValidatorTest { Clock.systemUTC().instant()); } - private static VespaModel model(int mem1, int mem2, int mem3, int mem4) { + private static VespaModel model(int mem1, int mem2, int cpu1, int cpu2) { var properties = new TestProperties(); properties.setHostedVespa(true); var deployState = new DeployState.Builder().properties(properties) @@ -82,7 +82,7 @@ public class NodeResourceChangeValidatorTest { " </container>\n" + " <content id='content1' version='1.0'>\n" + " <nodes count='3'>\n" + - " <resources vcpu='1' memory='" + mem3 + "Gb' disk='100Gb'/>" + + " <resources vcpu='" + cpu1 + "' memory='8Gb' disk='100Gb'/>" + " </nodes>\n" + " <documents>\n" + " <document mode='index' type='test'/>\n" + @@ -91,7 +91,7 @@ public class NodeResourceChangeValidatorTest { " </content>\n" + " <content id='content2' version='1.0'>\n" + " <nodes count='4'>\n" + - " <resources vcpu='1' memory='" + mem4 + "Gb' disk='100Gb'/>" + + " <resources vcpu='" + cpu2 + "' memory='8Gb' disk='100Gb'/>" + " </nodes>\n" + " <documents>\n" + " <document mode='streaming' type='test'/>\n" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java index 294df42bd77..60af25a3087 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java @@ -78,11 +78,11 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { } private static void verifyIgnoreJvmGCOptions(boolean isHosted) throws IOException, SAXException { - verifyIgnoreJvmGCOptionsIfJvmArgs("jvmargs", ContainerCluster.G1GC); - verifyIgnoreJvmGCOptionsIfJvmArgs( "jvm-options", "-XX:+UseG1GC"); + verifyIgnoreJvmGCOptionsIfJvmArgs("jvmargs", ContainerCluster.G1GC, isHosted); + verifyIgnoreJvmGCOptionsIfJvmArgs( "jvm-options", "-XX:+UseG1GC", isHosted); } - private static void verifyIgnoreJvmGCOptionsIfJvmArgs(String jvmOptionsName, String expectedGC) throws IOException, SAXException { + private static void verifyIgnoreJvmGCOptionsIfJvmArgs(String jvmOptionsName, String expectedGC, boolean isHosted) throws IOException, SAXException { String servicesXml = "<container version='1.0'>" + " <nodes jvm-gc-options='-XX:+UseG1GC' " + jvmOptionsName + "='-XX:+UseParNewGC'>" + @@ -95,6 +95,7 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() .applicationPackage(applicationPackage) .deployLogger(logger) + .properties(new TestProperties().setHostedVespa(isHosted)) .build()); QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); model.getConfig(qrStartBuilder, "container/container.0"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java new file mode 100644 index 00000000000..bc830c079d0 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ClusterResourceLimitsTest.java @@ -0,0 +1,101 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.content; + +import org.junit.Test; + +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +/** + * @author geirst + */ +public class ClusterResourceLimitsTest { + + private static class Fixture { + ResourceLimits.Builder ctrlBuilder = new ResourceLimits.Builder(); + ResourceLimits.Builder nodeBuilder = new ResourceLimits.Builder(); + + public Fixture ctrlDisk(double limit) { + ctrlBuilder.setDiskLimit(limit); + return this; + } + public Fixture ctrlMemory(double limit) { + ctrlBuilder.setMemoryLimit(limit); + return this; + } + public Fixture nodeDisk(double limit) { + nodeBuilder.setDiskLimit(limit); + return this; + } + public Fixture nodeMemory(double limit) { + nodeBuilder.setMemoryLimit(limit); + return this; + } + public ClusterResourceLimits build() { + var builder = new ClusterResourceLimits.Builder(); + builder.setClusterControllerBuilder(ctrlBuilder); + builder.setContentNodeBuilder(nodeBuilder); + return builder.build(); + } + } + + @Test + public void content_node_limits_are_derived_from_cluster_controller_limits_if_not_set() { + assertLimits(0.6, 0.7, 0.8, 0.85, + new Fixture().ctrlDisk(0.6).ctrlMemory(0.7)); + assertLimits(0.6, null, 0.8, null, + new Fixture().ctrlDisk(0.6)); + assertLimits(null, 0.7, null, 0.85, + new Fixture().ctrlMemory(0.7)); + } + + @Test + public void content_node_limits_can_be_set_explicit() { + assertLimits(0.6, 0.7, 0.9, 0.95, + new Fixture().ctrlDisk(0.6).ctrlMemory(0.7).nodeDisk(0.9).nodeMemory(0.95)); + assertLimits(0.6, null, 0.9, null, + new Fixture().ctrlDisk(0.6).nodeDisk(0.9)); + assertLimits(null, 0.7, null, 0.95, + new Fixture().ctrlMemory(0.7).nodeMemory(0.95)); + } + + @Test + public void cluster_controller_limits_are_equal_to_content_node_limits_if_not_set() { + assertLimits(0.9, 0.95, 0.9, 0.95, + new Fixture().nodeDisk(0.9).nodeMemory(0.95)); + assertLimits(0.9, null, 0.9, null, + new Fixture().nodeDisk(0.9)); + assertLimits(null, 0.95, null, 0.95, + new Fixture().nodeMemory(0.95)); + } + + @Test + public void limits_are_derived_from_the_other_if_not_set() { + assertLimits(0.6, 0.95, 0.8, 0.95, + new Fixture().ctrlDisk(0.6).nodeMemory(0.95)); + assertLimits(0.9, 0.7, 0.9, 0.85, + new Fixture().ctrlMemory(0.7).nodeDisk(0.9)); + } + + private void assertLimits(Double expCtrlDisk, Double expCtrlMemory, Double expNodeDisk, Double expNodeMemory, Fixture f) { + var limits = f.build(); + assertLimits(expCtrlDisk, expCtrlMemory, limits.getClusterControllerLimits()); + assertLimits(expNodeDisk, expNodeMemory, limits.getContentNodeLimits()); + } + + private void assertLimits(Double expDisk, Double expMemory, ResourceLimits limits) { + assertLimit(expDisk, limits.getDiskLimit()); + assertLimit(expMemory, limits.getMemoryLimit()); + } + + private void assertLimit(Double expLimit, Optional<Double> actLimit) { + if (expLimit == null) { + assertFalse(actLimit.isPresent()); + } else { + assertEquals(expLimit, actLimit.get(), 0.00001); + } + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java index 3415044b088..bc60908e268 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java @@ -69,15 +69,30 @@ public class ContentSearchClusterTest { } private static ProtonConfig getProtonConfig(ContentCluster cluster) { - ProtonConfig.Builder protonCfgBuilder = new ProtonConfig.Builder(); - cluster.getSearch().getConfig(protonCfgBuilder); - return new ProtonConfig(protonCfgBuilder); + var builder = new ProtonConfig.Builder(); + cluster.getSearch().getConfig(builder); + return new ProtonConfig(builder); } - private static void assertProtonResourceLimits(double expDiskLimit, double expMemoryLimits, String clusterXml) throws Exception { - ProtonConfig cfg = getProtonConfig(createCluster(clusterXml)); + private static void assertProtonResourceLimits(double expDiskLimit, double expMemoryLimit, String clusterXml) throws Exception { + assertProtonResourceLimits(expDiskLimit, expMemoryLimit, createCluster(clusterXml)); + } + + private static void assertProtonResourceLimits(double expDiskLimit, double expMemoryLimit, ContentCluster cluster) { + var cfg = getProtonConfig(cluster); assertEquals(expDiskLimit, cfg.writefilter().disklimit(), EPSILON); - assertEquals(expMemoryLimits, cfg.writefilter().memorylimit(), EPSILON); + assertEquals(expMemoryLimit, cfg.writefilter().memorylimit(), EPSILON); + } + + private static void assertClusterControllerResourceLimits(double expDiskLimit, double expMemoryLimit, String clusterXml) throws Exception { + assertClusterControllerResourceLimits(expDiskLimit, expMemoryLimit, createCluster(clusterXml)); + } + + private static void assertClusterControllerResourceLimits(double expDiskLimit, double expMemoryLimit, ContentCluster cluster) { + var limits = getFleetcontrollerConfig(cluster).cluster_feed_block_limit(); + assertEquals(4, limits.size()); + assertEquals(expDiskLimit, limits.get("disk"), EPSILON); + assertEquals(expMemoryLimit, limits.get("memory"), EPSILON); } @Test @@ -105,6 +120,19 @@ public class ContentSearchClusterTest { } @Test + public void cluster_controller_resource_limits_can_be_set() throws Exception { + assertClusterControllerResourceLimits(0.92, 0.93, + new ContentClusterBuilder().clusterControllerDiskLimit(0.92).clusterControllerMemoryLimit(0.93).getXml()); + } + + @Test + public void resource_limits_are_derived_from_the_other_if_not_specified() throws Exception { + var cluster = createCluster(new ContentClusterBuilder().clusterControllerDiskLimit(0.5).protonMemoryLimit(0.95).getXml()); + assertProtonResourceLimits(0.75, 0.95, cluster); + assertClusterControllerResourceLimits(0.5, 0.95, cluster); + } + + @Test public void requireThatGloballyDistributedDocumentTypeIsTaggedAsSuch() throws Exception { ProtonConfig cfg = getProtonConfig(createClusterWithGlobalType()); assertEquals(2, cfg.documentdb().size()); @@ -149,8 +177,9 @@ public class ContentSearchClusterTest { } private static FleetcontrollerConfig getFleetcontrollerConfig(ContentCluster cluster) { - FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder(); + var builder = new FleetcontrollerConfig.Builder(); cluster.getConfig(builder); + cluster.getClusterControllerConfig().getConfig(builder); builder.cluster_name("unknown"); builder.index(0); builder.zookeeper_server("unknown"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java index 01bbffce360..3a59f35ce2e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import org.junit.Test; import org.w3c.dom.Document; +import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.Assert.assertEquals; public class FleetControllerClusterTest { @@ -19,8 +20,11 @@ public class FleetControllerClusterTest { var deployState = new DeployState.Builder().properties( new TestProperties().enableFeedBlockInDistributor(enableFeedBlockInDistributor)).build(); MockRoot root = new MockRoot("", deployState); - return new ClusterControllerConfig.Builder("storage", new ModelElement(doc.getDocumentElement())).build(root.getDeployState(), root, - new ModelElement(doc.getDocumentElement()).getXml()); + var clusterElement = new ModelElement(doc.getDocumentElement()); + return new ClusterControllerConfig.Builder("storage", + clusterElement, + new ClusterResourceLimits.Builder().build(clusterElement).getClusterControllerLimits()). + build(root.getDeployState(), root, clusterElement.getXml()); } private ClusterControllerConfig parse(String xml) { @@ -94,15 +98,43 @@ public class FleetControllerClusterTest { assertEquals(0.0, config.min_node_ratio_per_group(), 0.01); } + @Test public void default_cluster_feed_block_limits_are_set() { - var config = getConfigForBasicCluster(); + assertLimits(0.79, 0.79, getConfigForBasicCluster()); + } + + @Test + public void resource_limits_can_be_set_in_tuning() { + assertLimits(0.6, 0.7, getConfigForResourceLimitsTuning(0.6, 0.7)); + assertLimits(0.6, 0.79, getConfigForResourceLimitsTuning(0.6, null)); + assertLimits(0.79, 0.7, getConfigForResourceLimitsTuning(null, 0.7)); + } + + private static double DELTA = 0.00001; + + private void assertLimits(double expDisk, double expMemory, FleetcontrollerConfig config) { var limits = config.cluster_feed_block_limit(); assertEquals(4, limits.size()); - assertEquals(0.79, limits.get("memory"), 0.0001); - assertEquals(0.79, limits.get("disk"), 0.0001); - assertEquals(0.89, limits.get("attribute-enum-store"), 0.0001); - assertEquals(0.89, limits.get("attribute-multi-value"), 0.0001); + assertEquals(expDisk, limits.get("disk"), DELTA); + assertEquals(expMemory, limits.get("memory"), DELTA); + assertEquals(0.89, limits.get("attribute-enum-store"), DELTA); + assertEquals(0.89, limits.get("attribute-multi-value"), DELTA); + } + + private FleetcontrollerConfig getConfigForResourceLimitsTuning(Double diskLimit, Double memoryLimit) { + FleetcontrollerConfig.Builder builder = new FleetcontrollerConfig.Builder(); + parse(joinLines("<cluster id=\"test\">", + "<documents/>", + "<tuning>", + " <resource-limits>", + (diskLimit != null ? (" <disk>" + diskLimit + "</disk>") : ""), + (memoryLimit != null ? (" <memory>" + memoryLimit + "</memory>") : ""), + " </resource-limits>", + "</tuning>" + + "</cluster>")). + getConfig(builder); + return new FleetcontrollerConfig(builder); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index 9c857414717..5cf57430f91 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -287,19 +287,6 @@ public class StorageClusterTest { assertEquals(StorFilestorConfig.Response_sequencer_type.THROUGHPUT, config.response_sequencer_type()); } - private void verifyMergeChunkSize(int expected, int value) { - StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); - simpleCluster(new TestProperties().setMergeChunkSize(value)).getConfig(builder); - StorFilestorConfig config = new StorFilestorConfig(builder); - assertEquals(expected, config.bucket_merge_chunk_size()); - } - - @Test - public void testFeatureFlagControlOfMergeChunkSize() { - verifyMergeChunkSize(0x200000, 13); - verifyMergeChunkSize(0x1600000, 0x1500000); - } - private void verifyAsyncMessageHandlingOnSchedule(boolean expected, boolean value) { StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder(); simpleCluster(new TestProperties().setAsyncMessageHandlingOnSchedule(value)).getConfig(builder); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java index 866c03d82f0..491326fdc9c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterBuilder.java @@ -26,6 +26,8 @@ public class ContentClusterBuilder { private Optional<String> dispatchXml = Optional.empty(); private Optional<Double> protonDiskLimit = Optional.empty(); private Optional<Double> protonMemoryLimit = Optional.empty(); + private Optional<Double> clusterControllerDiskLimit = Optional.empty(); + private Optional<Double> clusterControllerMemoryLimit = Optional.empty(); public ContentClusterBuilder() { } @@ -67,13 +69,23 @@ public class ContentClusterBuilder { return this; } - public ContentClusterBuilder protonDiskLimit(double diskLimit) { - protonDiskLimit = Optional.of(diskLimit); + public ContentClusterBuilder protonDiskLimit(double limit) { + protonDiskLimit = Optional.of(limit); return this; } - public ContentClusterBuilder protonMemoryLimit(double memoryLimit) { - protonMemoryLimit = Optional.of(memoryLimit); + public ContentClusterBuilder protonMemoryLimit(double limit) { + protonMemoryLimit = Optional.of(limit); + return this; + } + + public ContentClusterBuilder clusterControllerDiskLimit(double limit) { + clusterControllerDiskLimit = Optional.of(limit); + return this; + } + + public ContentClusterBuilder clusterControllerMemoryLimit(double limit) { + clusterControllerMemoryLimit = Optional.of(limit); return this; } @@ -88,14 +100,17 @@ public class ContentClusterBuilder { " <engine>", " <proton>", " <searchable-copies>" + searchableCopies + "</searchable-copies>", - getResourceLimitsXml(" "), + getProtonResourceLimitsXml(" "), " </proton>", " </engine>"); if (dispatchXml.isPresent()) { xml += dispatchXml.get(); } - return xml + groupXml + - "</content>"; + xml += groupXml; + xml += joinLines(" <tuning>", + getTuningResourceLimitsXml(" "), + " </tuning>"); + return xml + "</content>"; } private static String getSimpleGroupXml() { @@ -104,11 +119,19 @@ public class ContentClusterBuilder { " </group>"); } - private String getResourceLimitsXml(String indent) { - if (protonDiskLimit.isPresent() || protonMemoryLimit.isPresent()) { + private String getProtonResourceLimitsXml(String indent) { + return getResourceLimitsXml(indent, protonDiskLimit, protonMemoryLimit); + } + + private String getTuningResourceLimitsXml(String indent) { + return getResourceLimitsXml(indent, clusterControllerDiskLimit, clusterControllerMemoryLimit); + } + + private String getResourceLimitsXml(String indent, Optional<Double> diskLimit, Optional<Double> memoryLimit) { + if (diskLimit.isPresent() || memoryLimit.isPresent()) { String xml = joinLines(indent + "<resource-limits>", - getXmlLine("disk", protonDiskLimit, indent + " "), - getXmlLine("memory", protonMemoryLimit, indent + " "), + getXmlLine("disk", diskLimit, indent + " "), + getXmlLine("memory", memoryLimit, indent + " "), indent + "</resource-limits>"); return xml; } diff --git a/config/src/tests/configfetcher/configfetcher.cpp b/config/src/tests/configfetcher/configfetcher.cpp index 508322bb74e..0cff6208873 100644 --- a/config/src/tests/configfetcher/configfetcher.cpp +++ b/config/src/tests/configfetcher/configfetcher.cpp @@ -5,6 +5,7 @@ #include <vespa/vespalib/util/exception.h> #include "config-my.h" #include <atomic> +#include <thread> using namespace config; diff --git a/config/src/tests/configholder/configholder.cpp b/config/src/tests/configholder/configholder.cpp index d69ad8022f9..1f2c1f27df6 100644 --- a/config/src/tests/configholder/configholder.cpp +++ b/config/src/tests/configholder/configholder.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/common/configholder.h> +#include <thread> using namespace config; diff --git a/config/src/tests/configretriever/configretriever.cpp b/config/src/tests/configretriever/configretriever.cpp index d4ebe2da994..c8157ff7c16 100644 --- a/config/src/tests/configretriever/configretriever.cpp +++ b/config/src/tests/configretriever/configretriever.cpp @@ -1,5 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "config-bootstrap.h" +#include "config-foo.h" +#include "config-bar.h" #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/config/print.h> @@ -11,9 +14,7 @@ #include <vespa/config/subscription/configsubscription.h> #include <vespa/config/subscription/sourcespec.h> #include <vespa/config/common/exceptions.h> -#include "config-bootstrap.h" -#include "config-foo.h" -#include "config-bar.h" +#include <thread> #include <atomic> using namespace config; diff --git a/config/src/tests/frt/frt.cpp b/config/src/tests/frt/frt.cpp index 6cffe079dae..cb09b8f7254 100644 --- a/config/src/tests/frt/frt.cpp +++ b/config/src/tests/frt/frt.cpp @@ -1,5 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "config-my.h" +#include "config-bar.h" #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/common/iconfigholder.h> #include <vespa/config/common/trace.h> @@ -15,9 +17,7 @@ #include <vespa/fnet/frt/supervisor.h> #include <vespa/config/frt/protocol.h> #include <lz4.h> -#include "config-my.h" -#include "config-bar.h" - +#include <thread> using namespace config; using namespace vespalib; diff --git a/config/src/tests/subscriber/subscriber.cpp b/config/src/tests/subscriber/subscriber.cpp index 9d5aeddaf37..d58699f26e0 100644 --- a/config/src/tests/subscriber/subscriber.cpp +++ b/config/src/tests/subscriber/subscriber.cpp @@ -1,13 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "config-foo.h" +#include "config-bar.h" +#include "config-baz.h" #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/config/config.h> #include <vespa/config/common/misc.h> #include <vespa/config/common/configholder.h> #include <vespa/config/subscription/configsubscription.h> #include <vespa/config/common/exceptions.h> -#include "config-foo.h" -#include "config-bar.h" -#include "config-baz.h" +#include <thread> using namespace config; using namespace vespalib; diff --git a/config/src/tests/subscription/subscription.cpp b/config/src/tests/subscription/subscription.cpp index 9583111a2f7..7a5fccec019 100644 --- a/config/src/tests/subscription/subscription.cpp +++ b/config/src/tests/subscription/subscription.cpp @@ -4,6 +4,7 @@ #include <vespa/config/common/configholder.h> #include <vespa/config/subscription/configsubscription.h> #include <config-my.h> +#include <thread> using namespace config; using namespace std::chrono_literals; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index f2f5eb3afa1..a72381d4e84 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -140,7 +140,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye private final TesterClient testerClient; private final Metric metric; private final ClusterReindexingStatusClient clusterReindexingStatusClient; - private final BooleanFlag waitForResourcesInPrepareFlag; @Inject public ApplicationRepository(TenantRepository tenantRepository, @@ -193,7 +192,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye this.testerClient = Objects.requireNonNull(testerClient); this.metric = Objects.requireNonNull(metric); this.clusterReindexingStatusClient = clusterReindexingStatusClient; - this.waitForResourcesInPrepareFlag = Flags.WAIT_FOR_RESOURCES_IN_PREPARE.bindTo(flagSource); } public static class Builder { @@ -401,10 +399,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye SessionRepository sessionRepository = tenant.getSessionRepository(); DeployLogger logger = new SilentDeployLogger(); Session newSession = sessionRepository.createSessionFromExisting(activeSession, true, timeoutBudget); - boolean waitForResourcesInPrepare = waitForResourcesInPrepareFlag.value(); return Optional.of(Deployment.unprepared(newSession, this, hostProvisioner, tenant, logger, timeout, clock, - false /* don't validate as this is already deployed */, bootstrap, waitForResourcesInPrepare)); + false /* don't validate as this is already deployed */, bootstrap)); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index ff2bbc12e29..980134f8884 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -81,8 +81,8 @@ public class Deployment implements com.yahoo.config.provision.Deployment { public static Deployment unprepared(Session session, ApplicationRepository applicationRepository, Optional<Provisioner> provisioner, Tenant tenant, DeployLogger logger, - Duration timeout, Clock clock, boolean validate, boolean isBootstrap, boolean waitForResourcesInPrepare) { - Supplier<PrepareParams> params = createPrepareParams(clock, timeout, session, isBootstrap, !validate, false, waitForResourcesInPrepare); + Duration timeout, Clock clock, boolean validate, boolean isBootstrap) { + Supplier<PrepareParams> params = createPrepareParams(clock, timeout, session, isBootstrap, !validate, false, true); return new Deployment(session, applicationRepository, params, provisioner, tenant, logger, clock, true, false); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index fa30cb895c0..aa9ae65394f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -147,7 +147,6 @@ public class ModelContextImpl implements ModelContext { public static class FeatureFlags implements ModelContext.FeatureFlags { - private final boolean enableAutomaticReindexing; private final double reindexerWindowSizeIncrement; private final double defaultTermwiseLimit; private final boolean useThreePhaseUpdates; @@ -159,7 +158,6 @@ public class ModelContextImpl implements ModelContext { private final boolean skipMbusReplyThread; private final boolean useAccessControlTlsHandshakeClientAuth; private final boolean useAsyncMessageHandlingOnSchedule; - private final int mergeChunkSize; private final double feedConcurrency; private final boolean reconfigurableZookeeperServer; private final boolean enableJdiscConnectionLog; @@ -168,7 +166,6 @@ public class ModelContextImpl implements ModelContext { private final boolean enableFeedBlockInDistributor; public FeatureFlags(FlagSource source, ApplicationId appId) { - this.enableAutomaticReindexing = flagValue(source, appId, Flags.ENABLE_AUTOMATIC_REINDEXING); this.reindexerWindowSizeIncrement = flagValue(source, appId, Flags.REINDEXER_WINDOW_SIZE_INCREMENT); this.defaultTermwiseLimit = flagValue(source, appId, Flags.DEFAULT_TERM_WISE_LIMIT); this.useThreePhaseUpdates = flagValue(source, appId, Flags.USE_THREE_PHASE_UPDATES); @@ -180,7 +177,6 @@ public class ModelContextImpl implements ModelContext { this.skipMbusReplyThread = flagValue(source, appId, Flags.SKIP_MBUS_REPLY_THREAD); this.useAccessControlTlsHandshakeClientAuth = flagValue(source, appId, Flags.USE_ACCESS_CONTROL_CLIENT_AUTHENTICATION); this.useAsyncMessageHandlingOnSchedule = flagValue(source, appId, Flags.USE_ASYNC_MESSAGE_HANDLING_ON_SCHEDULE); - this.mergeChunkSize = flagValue(source, appId, Flags.MERGE_CHUNK_SIZE); this.feedConcurrency = flagValue(source, appId, Flags.FEED_CONCURRENCY); this.reconfigurableZookeeperServer = flagValue(source, appId, Flags.RECONFIGURABLE_ZOOKEEPER_SERVER_FOR_CLUSTER_CONTROLLER); this.enableJdiscConnectionLog = flagValue(source, appId, Flags.ENABLE_JDISC_CONNECTION_LOG); @@ -189,7 +185,6 @@ public class ModelContextImpl implements ModelContext { this.enableFeedBlockInDistributor = flagValue(source, appId, Flags.ENABLE_FEED_BLOCK_IN_DISTRIBUTOR); } - @Override public boolean enableAutomaticReindexing() { return enableAutomaticReindexing; } @Override public double reindexerWindowSizeIncrement() { return reindexerWindowSizeIncrement; } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @Override public boolean useThreePhaseUpdates() { return useThreePhaseUpdates; } @@ -201,7 +196,6 @@ public class ModelContextImpl implements ModelContext { @Override public boolean skipMbusReplyThread() { return skipMbusReplyThread; } @Override public boolean useAccessControlTlsHandshakeClientAuth() { return useAccessControlTlsHandshakeClientAuth; } @Override public boolean useAsyncMessageHandlingOnSchedule() { return useAsyncMessageHandlingOnSchedule; } - @Override public int mergeChunkSize() { return mergeChunkSize; } @Override public double feedConcurrency() { return feedConcurrency; } @Override public boolean reconfigurableZookeeperServer() { return reconfigurableZookeeperServer; } @Override public boolean enableJdiscConnectionLog() { return enableJdiscConnectionLog; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java index 582ecd13ce5..b1edc031e0b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java @@ -95,7 +95,6 @@ public class ModelContextImplTest { assertEquals(new Version(8), context.wantedNodeVespaVersion()); assertEquals(1.0, context.properties().featureFlags().defaultTermwiseLimit(), 0.0); assertFalse(context.properties().featureFlags().useAsyncMessageHandlingOnSchedule()); - assertEquals(0x2000000, context.properties().featureFlags().mergeChunkSize()); assertEquals(0.5, context.properties().featureFlags().feedConcurrency(), 0.0); } diff --git a/container-core/pom.xml b/container-core/pom.xml index 7c98b524c73..051b572b28f 100644 --- a/container-core/pom.xml +++ b/container-core/pom.xml @@ -42,6 +42,7 @@ <scope>test</scope> </dependency> <dependency> + <!-- TODO Vespa 8: stop providing org.json:json --> <groupId>org.json</groupId> <artifactId>json</artifactId> </dependency> diff --git a/container-core/src/main/java/com/yahoo/container/handler/metrics/HttpHandlerBase.java b/container-core/src/main/java/com/yahoo/container/handler/metrics/HttpHandlerBase.java index 92840cee48f..8c902f88e38 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/metrics/HttpHandlerBase.java +++ b/container-core/src/main/java/com/yahoo/container/handler/metrics/HttpHandlerBase.java @@ -1,13 +1,14 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler.metrics; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.restapi.Path; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; import java.net.URI; import java.util.List; @@ -26,6 +27,8 @@ import static java.util.logging.Level.WARNING; */ public abstract class HttpHandlerBase extends ThreadedHttpRequestHandler { + private static final ObjectMapper jsonMapper = new ObjectMapper(); + protected HttpHandlerBase(Executor executor) { super(executor); } @@ -49,15 +52,14 @@ public abstract class HttpHandlerBase extends ThreadedHttpRequestHandler { protected JsonResponse resourceListResponse(URI requestUri, List<String> resources) { try { return new JsonResponse(OK, resourceList(requestUri, resources)); - } catch (JSONException e) { + } catch (JsonProcessingException e) { log.log(WARNING, "Bad JSON construction in generated resource list for " + requestUri.getPath(), e); return new ErrorResponse(INTERNAL_SERVER_ERROR, "An error occurred when generating the list of api resources."); } } - // TODO: Use jackson with a "Resources" class instead of JSONObject - private static String resourceList(URI requestUri, List<String> resources) throws JSONException { + private static String resourceList(URI requestUri, List<String> resources) throws JsonProcessingException { int port = requestUri.getPort(); String host = requestUri.getHost(); StringBuilder base = new StringBuilder("http://"); @@ -66,13 +68,14 @@ public abstract class HttpHandlerBase extends ThreadedHttpRequestHandler { base.append(":").append(port); } String uriBase = base.toString(); - JSONArray linkList = new JSONArray(); + ArrayNode linkList = jsonMapper.createArrayNode(); for (String api : resources) { - JSONObject resource = new JSONObject(); + ObjectNode resource = jsonMapper.createObjectNode(); resource.put("url", uriBase + api); - linkList.put(resource); + linkList.add(resource); } - return new JSONObject().put("resources", linkList).toString(4); + return jsonMapper.writerWithDefaultPrettyPrinter() + .writeValueAsString(jsonMapper.createObjectNode().set("resources", linkList)); } } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/CoredumpGatherer.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/CoredumpGatherer.java index d105eaa9d98..f1ef7894511 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/CoredumpGatherer.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/CoredumpGatherer.java @@ -1,17 +1,16 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.state; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.yahoo.vespa.defaults.Defaults; -import org.json.JSONException; -import org.json.JSONObject; import java.io.IOException; import java.io.UncheckedIOException; -import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.time.Instant; -import java.util.Set; import java.util.stream.Stream; /** @@ -19,19 +18,17 @@ import java.util.stream.Stream; */ public class CoredumpGatherer { + private static final ObjectMapper jsonMapper = new ObjectMapper(); + private static final Path COREDUMP_PATH = Path.of(Defaults.getDefaults().underVespaHome("var/crash/processing")); - public static JSONObject gatherCoredumpMetrics(FileWrapper fileWrapper) { + public static JsonNode gatherCoredumpMetrics(FileWrapper fileWrapper) { int coredumps = getNumberOfCoredumps(fileWrapper); - JSONObject packet = new JSONObject(); - - try { - packet.put("status_code", coredumps == 0 ? 0 : 1); - packet.put("status_msg", coredumps == 0 ? "OK" : String.format("Found %d coredump(s)", coredumps)); - packet.put("timestamp", Instant.now().getEpochSecond()); - packet.put("application", "system-coredumps-processing"); - - } catch (JSONException e) {} + ObjectNode packet = jsonMapper.createObjectNode(); + packet.put("status_code", coredumps == 0 ? 0 : 1); + packet.put("status_msg", coredumps == 0 ? "OK" : String.format("Found %d coredump(s)", coredumps)); + packet.put("timestamp", Instant.now().getEpochSecond()); + packet.put("application", "system-coredumps-processing"); return packet; } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/HostLifeGatherer.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/HostLifeGatherer.java index 730f7bc13cd..28f99096d84 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/HostLifeGatherer.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/HostLifeGatherer.java @@ -1,8 +1,9 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.state; -import org.json.JSONException; -import org.json.JSONObject; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; import java.nio.file.Path; @@ -13,9 +14,11 @@ import java.time.Instant; */ public class HostLifeGatherer { + private static final ObjectMapper jsonMapper = new ObjectMapper(); + private static final Path UPTIME_PATH = Path.of("/proc"); - public static JSONObject getHostLifePacket(FileWrapper fileWrapper) { + public static JsonNode getHostLifePacket(FileWrapper fileWrapper) { long upTime; int statusCode = 0; String statusMessage = "OK"; @@ -29,19 +32,15 @@ public class HostLifeGatherer { } - JSONObject jsonObject = new JSONObject(); - try { - jsonObject.put("status_code", statusCode); - jsonObject.put("status_msg", statusMessage); - jsonObject.put("timestamp", Instant.now().getEpochSecond()); - jsonObject.put("application", "host_life"); - JSONObject metrics = new JSONObject(); - metrics.put("uptime", upTime); - metrics.put("alive", 1); - jsonObject.put("metrics", metrics); - - } catch (JSONException e) {} - + ObjectNode jsonObject = jsonMapper.createObjectNode(); + jsonObject.put("status_code", statusCode); + jsonObject.put("status_msg", statusMessage); + jsonObject.put("timestamp", Instant.now().getEpochSecond()); + jsonObject.put("application", "host_life"); + ObjectNode metrics = jsonMapper.createObjectNode(); + metrics.put("uptime", upTime); + metrics.put("alive", 1); + jsonObject.set("metrics", metrics); return jsonObject; } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/JSONObjectWithLegibleException.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/JSONObjectWithLegibleException.java deleted file mode 100644 index d22dd9d6f4b..00000000000 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/JSONObjectWithLegibleException.java +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.jdisc.state; - -import org.json.JSONException; -import org.json.JSONObject; - -import java.util.Collection; -import java.util.Map; - -/** - * A JSONObject that wraps the checked JSONException in a RuntimeException with a legible error message. - * - * @author gjoranv - */ -class JSONObjectWithLegibleException extends JSONObject { - - @Override - public JSONObject put(String s, boolean b) { - try { - return super.put(s, b); - } catch (JSONException e) { - throw new RuntimeException(getErrorMessage(s, b, e), e); - } - } - - @Override - public JSONObject put(String s, double v) { - try { - Double guardedVal = (((Double) v).isNaN() || ((Double) v).isInfinite()) ? - 0.0 : v; - return super.put(s, guardedVal); - } catch (JSONException e) { - throw new RuntimeException(getErrorMessage(s, v, e), e); - } - } - - @Override - public JSONObject put(String s, int i) { - try { - return super.put(s, i); - } catch (JSONException e) { - throw new RuntimeException(getErrorMessage(s, i, e), e); - } - } - - @Override - public JSONObject put(String s, long l) { - try { - return super.put(s, l); - } catch (JSONException e) { - throw new RuntimeException(getErrorMessage(s, l, e), e); - } - } - - @Override - public JSONObject put(String s, Collection collection) { - try { - return super.put(s, collection); - } catch (JSONException e) { - throw new RuntimeException(getErrorMessage(s, collection, e), e); - } - } - - @Override - public JSONObject put(String s, Map map) { - try { - return super.put(s, map); - } catch (JSONException e) { - throw new RuntimeException(getErrorMessage(s, map, e), e); - } - } - - @Override - public JSONObject put(String s, Object o) { - try { - return super.put(s, o); - } catch (JSONException e) { - throw new RuntimeException(getErrorMessage(s, o, e), e); - } - } - - private String getErrorMessage(String key, Object value, JSONException e) { - return "Trying to add invalid JSON object with key '" + key + - "' and value '" + value + "' - " + e.getMessage(); - } - -} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/JsonUtil.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/JsonUtil.java new file mode 100644 index 00000000000..4c697fb5ada --- /dev/null +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/JsonUtil.java @@ -0,0 +1,15 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc.state; + +/** + * @author bjorncs + */ +class JsonUtil { + + private JsonUtil() {} + + static double sanitizeDouble(double value) { + return (((Double) value).isNaN() || ((Double) value).isInfinite()) ? 0.0 : value; + } + +} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricGatherer.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricGatherer.java index 6a06a6362f5..add69403455 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricGatherer.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricGatherer.java @@ -1,7 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.state; -import org.json.JSONObject; +import com.fasterxml.jackson.databind.JsonNode; import java.util.ArrayList; import java.util.List; @@ -13,9 +13,9 @@ import java.util.List; */ public class MetricGatherer { - static List<JSONObject> getAdditionalMetrics() { + static List<JsonNode> getAdditionalMetrics() { FileWrapper fileWrapper = new FileWrapper(); - List<JSONObject> packetList = new ArrayList<>(); + List<JsonNode> packetList = new ArrayList<>(); packetList.add(CoredumpGatherer.gatherCoredumpMetrics(fileWrapper)); if (System.getProperty("os.name").contains("nux")) packetList.add(HostLifeGatherer.getHostLifePacket(fileWrapper)); diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java index c1a6f650a9c..824323af3d6 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java @@ -1,6 +1,11 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.state; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.inject.Inject; import com.yahoo.collections.Tuple2; import com.yahoo.component.provider.ComponentRegistry; @@ -13,9 +18,6 @@ import com.yahoo.jdisc.handler.ResponseDispatch; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.http.HttpHeaders; import com.yahoo.metrics.MetricsPresentationConfig; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -26,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import static com.yahoo.container.jdisc.state.JsonUtil.sanitizeDouble; import static com.yahoo.container.jdisc.state.StateHandler.getSnapshotPreprocessor; /** @@ -44,6 +47,8 @@ import static com.yahoo.container.jdisc.state.StateHandler.getSnapshotPreprocess */ public class MetricsPacketsHandler extends AbstractRequestHandler { + private static final ObjectMapper jsonMapper = new ObjectMapper(); + static final String APPLICATION_KEY = "application"; static final String TIMESTAMP_KEY = "timestamp"; static final String STATUS_CODE_KEY = "status_code"; @@ -97,19 +102,19 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { } String output = jsonToString(getStatusPacket()) + getAllMetricsPackets() + "\n"; return output.getBytes(StandardCharsets.UTF_8); - } catch (JSONException e) { + } catch (JsonProcessingException e) { throw new RuntimeException("Bad JSON construction.", e); } } - private byte[] getMetricsArray() throws JSONException { - JSONObject root = new JSONObject(); - JSONArray jsonArray = new JSONArray(); - jsonArray.put(getStatusPacket()); + private byte[] getMetricsArray() throws JsonProcessingException { + ObjectNode root = jsonMapper.createObjectNode(); + ArrayNode jsonArray = jsonMapper.createArrayNode(); + jsonArray.add(getStatusPacket()); getPacketsForSnapshot(getSnapshot(), applicationName, timer.currentTimeMillis()) - .forEach(jsonArray::put); - MetricGatherer.getAdditionalMetrics().forEach(jsonArray::put); - root.put("metrics", jsonArray); + .forEach(jsonArray::add); + MetricGatherer.getAdditionalMetrics().forEach(jsonArray::add); + root.set("metrics", jsonArray); return jsonToString(root) .getBytes(StandardCharsets.UTF_8); } @@ -117,8 +122,8 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { /** * Exactly one status packet is added to the response. */ - private JSONObject getStatusPacket() throws JSONException { - JSONObject packet = new JSONObjectWithLegibleException(); + private JsonNode getStatusPacket() { + ObjectNode packet = jsonMapper.createObjectNode(); packet.put(APPLICATION_KEY, applicationName); StateMonitor.Status status = monitor.status(); @@ -127,14 +132,15 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { return packet; } - private String jsonToString(JSONObject jsonObject) throws JSONException { - return jsonObject.toString(4); + private static String jsonToString(JsonNode jsonObject) throws JsonProcessingException { + return jsonMapper.writerWithDefaultPrettyPrinter() + .writeValueAsString(jsonObject); } - private String getAllMetricsPackets() throws JSONException { + private String getAllMetricsPackets() throws JsonProcessingException { StringBuilder ret = new StringBuilder(); - List<JSONObject> metricsPackets = getPacketsForSnapshot(getSnapshot(), applicationName, timer.currentTimeMillis()); - for (JSONObject packet : metricsPackets) { + List<JsonNode> metricsPackets = getPacketsForSnapshot(getSnapshot(), applicationName, timer.currentTimeMillis()); + for (JsonNode packet : metricsPackets) { ret.append(PACKET_SEPARATOR); // For legibility and parsing in unit tests ret.append(jsonToString(packet)); } @@ -143,22 +149,23 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { private MetricSnapshot getSnapshot() { if (snapshotPreprocessor == null) { - return monitor.snapshot(); + // TODO: throw exception in ctor instead + return new MetricSnapshot(0L, 0L, TimeUnit.MILLISECONDS); } else { return snapshotPreprocessor.latestSnapshot(); } } - private List<JSONObject> getPacketsForSnapshot(MetricSnapshot metricSnapshot, String application, long timestamp) throws JSONException { + private List<JsonNode> getPacketsForSnapshot(MetricSnapshot metricSnapshot, String application, long timestamp) { if (metricSnapshot == null) return Collections.emptyList(); - List<JSONObject> packets = new ArrayList<>(); + List<JsonNode> packets = new ArrayList<>(); for (Map.Entry<MetricDimensions, MetricSet> snapshotEntry : metricSnapshot) { MetricDimensions metricDimensions = snapshotEntry.getKey(); MetricSet metricSet = snapshotEntry.getValue(); - JSONObjectWithLegibleException packet = new JSONObjectWithLegibleException(); + ObjectNode packet = jsonMapper.createObjectNode(); addMetaData(timestamp, application, packet); addDimensions(metricDimensions, packet); addMetrics(metricSet, packet); @@ -167,25 +174,27 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { return packets; } - private void addMetaData(long timestamp, String application, JSONObjectWithLegibleException packet) { + private void addMetaData(long timestamp, String application, ObjectNode packet) { packet.put(APPLICATION_KEY, application); packet.put(TIMESTAMP_KEY, TimeUnit.MILLISECONDS.toSeconds(timestamp)); } - private void addDimensions(MetricDimensions metricDimensions, JSONObjectWithLegibleException packet) throws JSONException { + private void addDimensions(MetricDimensions metricDimensions, ObjectNode packet) { + if (metricDimensions == null) return; + Iterator<Map.Entry<String, String>> dimensionsIterator = metricDimensions.iterator(); if (dimensionsIterator.hasNext()) { - JSONObject jsonDim = new JSONObjectWithLegibleException(); - packet.put(DIMENSIONS_KEY, jsonDim); + ObjectNode jsonDim = jsonMapper.createObjectNode(); + packet.set(DIMENSIONS_KEY, jsonDim); for (Map.Entry<String, String> dimensionEntry : metricDimensions) { jsonDim.put(dimensionEntry.getKey(), dimensionEntry.getValue()); } } } - private void addMetrics(MetricSet metricSet, JSONObjectWithLegibleException packet) throws JSONException { - JSONObjectWithLegibleException metrics = new JSONObjectWithLegibleException(); - packet.put(METRICS_KEY, metrics); + private void addMetrics(MetricSet metricSet, ObjectNode packet) { + ObjectNode metrics = jsonMapper.createObjectNode(); + packet.set(METRICS_KEY, metrics); for (Map.Entry<String, MetricValue> metric : metricSet) { String name = metric.getKey(); MetricValue value = metric.getValue(); @@ -193,9 +202,9 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { metrics.put(name + ".count", ((CountMetric) value).getCount()); } else if (value instanceof GaugeMetric) { GaugeMetric gauge = (GaugeMetric) value; - metrics.put(name + ".average", gauge.getAverage()) - .put(name + ".last", gauge.getLast()) - .put(name + ".max", gauge.getMax()); + metrics.put(name + ".average", sanitizeDouble(gauge.getAverage())) + .put(name + ".last", sanitizeDouble(gauge.getLast())) + .put(name + ".max", sanitizeDouble(gauge.getMax())); if (gauge.getPercentiles().isPresent()) { for (Tuple2<String, Double> prefixAndValue : gauge.getPercentiles().get()) { metrics.put(name + "." + prefixAndValue.first + "percentile", prefixAndValue.second.doubleValue()); diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateHandler.java index 8858465e846..5ac2871d9dd 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateHandler.java @@ -1,6 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.state; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.inject.Inject; import com.yahoo.collections.Tuple2; import com.yahoo.component.Vtag; @@ -16,21 +21,18 @@ import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.http.HttpHeaders; import com.yahoo.metrics.MetricsPresentationConfig; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; import java.net.URI; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; -import java.io.PrintStream; -import java.io.ByteArrayOutputStream; + +import static com.yahoo.container.jdisc.state.JsonUtil.sanitizeDouble; /** * A handler which returns state (health) information from this container instance: Status, metrics and vespa version. @@ -39,6 +41,8 @@ import java.io.ByteArrayOutputStream; */ public class StateHandler extends AbstractRequestHandler { + private static final ObjectMapper jsonMapper = new ObjectMapper(); + public static final String STATE_API_ROOT = "/state/v1"; private static final String METRICS_PATH = "metrics"; private static final String HISTOGRAMS_PATH = "metrics/histograms"; @@ -63,11 +67,7 @@ public class StateHandler extends AbstractRequestHandler { static SnapshotProvider getSnapshotPreprocessor(ComponentRegistry<SnapshotProvider> preprocessors, MetricsPresentationConfig presentation) { List<SnapshotProvider> allPreprocessors = preprocessors.allComponents(); - if (presentation.slidingwindow() && allPreprocessors.size() > 0) { - return allPreprocessors.get(0); - } else { - return null; - } + return (allPreprocessors.size() > 0) ? allPreprocessors.get(0) : null; } @Override @@ -128,17 +128,16 @@ public class StateHandler extends AbstractRequestHandler { } base.append(STATE_API_ROOT); String uriBase = base.toString(); - JSONArray linkList = new JSONArray(); + ArrayNode linkList = jsonMapper.createArrayNode(); for (String api : new String[] {METRICS_PATH, CONFIG_GENERATION_PATH, HEALTH_PATH, VERSION_PATH}) { - JSONObject resource = new JSONObject(); + ObjectNode resource = jsonMapper.createObjectNode(); resource.put("url", uriBase + "/" + api); - linkList.put(resource); + linkList.add(resource); } - return new JSONObjectWithLegibleException() - .put("resources", linkList) - .toString(4).getBytes(StandardCharsets.UTF_8); - } catch (JSONException e) { - throw new RuntimeException("Bad JSON construction.", e); + JsonNode resources = jsonMapper.createObjectNode().set("resources", linkList); + return toPrettyString(resources); + } catch (JsonProcessingException e) { + throw new RuntimeException("Bad JSON construction", e); } } @@ -158,31 +157,31 @@ public class StateHandler extends AbstractRequestHandler { private static byte[] buildConfigOutput(ApplicationMetadataConfig config) { try { - return new JSONObjectWithLegibleException() - .put(CONFIG_GENERATION_PATH, new JSONObjectWithLegibleException() - .put("generation", config.generation()) - .put("container", new JSONObjectWithLegibleException() - .put("generation", config.generation()))) - .toString(4).getBytes(StandardCharsets.UTF_8); - } catch (JSONException e) { + return toPrettyString( + jsonMapper.createObjectNode() + .set(CONFIG_GENERATION_PATH, jsonMapper.createObjectNode() + .put("generation", config.generation()) + .set("container", jsonMapper.createObjectNode() + .put("generation", config.generation())))); + } catch (JsonProcessingException e) { throw new RuntimeException("Bad JSON construction.", e); } } private static byte[] buildVersionOutput() { try { - return new JSONObjectWithLegibleException() - .put("version", Vtag.currentVersion) - .toString(4).getBytes(StandardCharsets.UTF_8); - } catch (JSONException e) { + return toPrettyString( + jsonMapper.createObjectNode() + .put("version", Vtag.currentVersion.toString())); + } catch (JsonProcessingException e) { throw new RuntimeException("Bad JSON construction.", e); } } private byte[] buildMetricOutput(String consumer) { try { - return buildJsonForConsumer(consumer).toString(4).getBytes(StandardCharsets.UTF_8); - } catch (JSONException e) { + return toPrettyString(buildJsonForConsumer(consumer)); + } catch (JsonProcessingException e) { throw new RuntimeException("Bad JSON construction.", e); } } @@ -195,17 +194,18 @@ public class StateHandler extends AbstractRequestHandler { return baos.toByteArray(); } - private JSONObjectWithLegibleException buildJsonForConsumer(String consumer) throws JSONException { - JSONObjectWithLegibleException ret = new JSONObjectWithLegibleException(); + private ObjectNode buildJsonForConsumer(String consumer) { + ObjectNode ret = jsonMapper.createObjectNode(); ret.put("time", timer.currentTimeMillis()); - ret.put("status", new JSONObjectWithLegibleException().put("code", getStatus().name())); - ret.put(METRICS_PATH, buildJsonForSnapshot(consumer, getSnapshot())); + ret.set("status", jsonMapper.createObjectNode().put("code", getStatus().name())); + ret.set(METRICS_PATH, buildJsonForSnapshot(consumer, getSnapshot())); return ret; } private MetricSnapshot getSnapshot() { if (snapshotPreprocessor == null) { - return monitor.snapshot(); + // TODO: throw exception in ctor instead + return new MetricSnapshot(0L, 0L, TimeUnit.MILLISECONDS); } else { return snapshotPreprocessor.latestSnapshot(); } @@ -215,55 +215,62 @@ public class StateHandler extends AbstractRequestHandler { return monitor.status(); } - private JSONObjectWithLegibleException buildJsonForSnapshot(String consumer, MetricSnapshot metricSnapshot) throws JSONException { + private ObjectNode buildJsonForSnapshot(String consumer, MetricSnapshot metricSnapshot) { if (metricSnapshot == null) { - return new JSONObjectWithLegibleException(); + return jsonMapper.createObjectNode(); } - JSONObjectWithLegibleException jsonMetric = new JSONObjectWithLegibleException(); - jsonMetric.put("snapshot", new JSONObjectWithLegibleException() - .put("from", metricSnapshot.getFromTime(TimeUnit.MILLISECONDS) / 1000.0) - .put("to", metricSnapshot.getToTime(TimeUnit.MILLISECONDS) / 1000.0)); + ObjectNode jsonMetric = jsonMapper.createObjectNode(); + jsonMetric.set("snapshot", jsonMapper.createObjectNode() + .put("from", sanitizeDouble(metricSnapshot.getFromTime(TimeUnit.MILLISECONDS) / 1000.0)) + .put("to", sanitizeDouble(metricSnapshot.getToTime(TimeUnit.MILLISECONDS) / 1000.0))); boolean includeDimensions = !consumer.equals(HEALTH_PATH); long periodInMillis = metricSnapshot.getToTime(TimeUnit.MILLISECONDS) - metricSnapshot.getFromTime(TimeUnit.MILLISECONDS); for (Tuple tuple : collapseMetrics(metricSnapshot, consumer)) { - JSONObjectWithLegibleException jsonTuple = new JSONObjectWithLegibleException(); + ObjectNode jsonTuple = jsonMapper.createObjectNode(); jsonTuple.put("name", tuple.key); if (tuple.val instanceof CountMetric) { CountMetric count = (CountMetric)tuple.val; - jsonTuple.put("values", new JSONObjectWithLegibleException() + jsonTuple.set("values", jsonMapper.createObjectNode() .put("count", count.getCount()) - .put("rate", (count.getCount() * 1000.0) / periodInMillis)); + .put("rate", sanitizeDouble(count.getCount() * 1000.0) / periodInMillis)); } else if (tuple.val instanceof GaugeMetric) { GaugeMetric gauge = (GaugeMetric) tuple.val; - JSONObjectWithLegibleException valueFields = new JSONObjectWithLegibleException(); - valueFields.put("average", gauge.getAverage()) - .put("sum", gauge.getSum()) + ObjectNode valueFields = jsonMapper.createObjectNode(); + valueFields.put("average", sanitizeDouble(gauge.getAverage())) + .put("sum", sanitizeDouble(gauge.getSum())) .put("count", gauge.getCount()) - .put("last", gauge.getLast()) - .put("max", gauge.getMax()) - .put("min", gauge.getMin()) - .put("rate", (gauge.getCount() * 1000.0) / periodInMillis); + .put("last", sanitizeDouble(gauge.getLast())) + .put("max", sanitizeDouble(gauge.getMax())) + .put("min", sanitizeDouble(gauge.getMin())) + .put("rate", sanitizeDouble((gauge.getCount() * 1000.0) / periodInMillis)); if (gauge.getPercentiles().isPresent()) { for (Tuple2<String, Double> prefixAndValue : gauge.getPercentiles().get()) { - valueFields.put(prefixAndValue.first + "percentile", prefixAndValue.second.doubleValue()); + valueFields.put(prefixAndValue.first + "percentile", sanitizeDouble(prefixAndValue.second)); } } - jsonTuple.put("values", valueFields); + jsonTuple.set("values", valueFields); } else { throw new UnsupportedOperationException(tuple.val.getClass().getName()); } - Iterator<Map.Entry<String, String>> it = tuple.dim.iterator(); - if (it.hasNext() && includeDimensions) { - JSONObjectWithLegibleException jsonDim = new JSONObjectWithLegibleException(); - while (it.hasNext()) { - Map.Entry<String, String> entry = it.next(); - jsonDim.put(entry.getKey(), entry.getValue()); + if (tuple.dim != null) { + Iterator<Map.Entry<String, String>> it = tuple.dim.iterator(); + if (it.hasNext() && includeDimensions) { + ObjectNode jsonDim = jsonMapper.createObjectNode(); + while (it.hasNext()) { + Map.Entry<String, String> entry = it.next(); + jsonDim.put(entry.getKey(), entry.getValue()); + } + jsonTuple.set("dimensions", jsonDim); } - jsonTuple.put("dimensions", jsonDim); } - jsonMetric.append("values", jsonTuple); + ArrayNode values = (ArrayNode) jsonMetric.get("values"); + if (values == null) { + values = jsonMapper.createArrayNode(); + jsonMetric.set("values", values); + } + values.add(jsonTuple); } return jsonMetric; } @@ -317,6 +324,12 @@ public class StateHandler extends AbstractRequestHandler { return metrics; } + private static byte[] toPrettyString(JsonNode resources) throws JsonProcessingException { + return jsonMapper.writerWithDefaultPrettyPrinter() + .writeValueAsString(resources) + .getBytes(); + } + static class Tuple { final MetricDimensions dim; diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMetricConsumer.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMetricConsumer.java deleted file mode 100644 index dfa791304e0..00000000000 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMetricConsumer.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.jdisc.state; - -import com.yahoo.jdisc.Metric; -import com.yahoo.jdisc.application.MetricConsumer; - -import java.util.Map; - -/** - * @author Simon Thoresen Hult - */ -final class StateMetricConsumer implements MetricConsumer { - - final static Metric.Context NULL_CONTEXT = StateMetricContext.newInstance(null); - private final Object lock = new Object(); - private MetricSnapshot metricSnapshot = new MetricSnapshot(); - - @Override - public void set(String key, Number val, Metric.Context ctx) { - synchronized (lock) { - metricSnapshot.set(dimensionsOrDefault(ctx), key, val); - } - } - - private MetricDimensions dimensionsOrDefault(Metric.Context ctx) { - return (MetricDimensions)(ctx != null ? ctx : NULL_CONTEXT); - } - - @Override - public void add(String key, Number val, Metric.Context ctx) { - synchronized (lock) { - metricSnapshot.add(dimensionsOrDefault(ctx), key, val); - } - } - - @Override - public Metric.Context createContext(Map<String, ?> properties) { - return StateMetricContext.newInstance(properties); - } - - MetricSnapshot createSnapshot() { - MetricSnapshot metricSnapshot; - synchronized (lock) { - metricSnapshot = this.metricSnapshot; - this.metricSnapshot = this.metricSnapshot.createSnapshot(); - } - return metricSnapshot; - } - -} diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java index 40a0ef10fbc..027ce02a2aa 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java @@ -4,25 +4,12 @@ package com.yahoo.container.jdisc.state; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.container.jdisc.config.HealthMonitorConfig; -import com.yahoo.jdisc.Timer; -import com.yahoo.jdisc.application.MetricConsumer; -import com.yahoo.jdisc.core.SystemTimer; -import java.util.Map; -import java.util.Optional; -import java.util.TreeSet; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; /** - * A state monitor keeps track of the current health and metrics state of a container. - * It is used by jDisc to hand out metric update API endpoints to workers through {@link #newMetricConsumer}, - * and to inspect the current accumulated state of metrics through {@link #snapshot}. + * A state monitor keeps track of the current health state of a container. * * @author Simon Thoresen Hult */ @@ -32,44 +19,15 @@ public class StateMonitor extends AbstractComponent { public enum Status {up, down, initializing} - private final CopyOnWriteArrayList<StateMetricConsumer> consumers = new CopyOnWriteArrayList<>(); - private final Optional<ScheduledExecutorService> executor; - private final Timer timer; - private final long snapshotIntervalMs; - private volatile long lastSnapshotTimeMs; - private volatile MetricSnapshot snapshot; private volatile Status status; - private final TreeSet<String> valueNames = new TreeSet<>(); @Inject - public StateMonitor(HealthMonitorConfig config, Timer timer) { - this(config, timer, runnable -> { - Thread thread = new Thread(runnable, "StateMonitor"); - thread.setDaemon(true); - return thread; - }); - } - - public static StateMonitor createForTesting() { - return new StateMonitor(new HealthMonitorConfig.Builder().build(), new SystemTimer()); - } - - /** Non-private only for unit testing this class. */ - StateMonitor(HealthMonitorConfig config, Timer timer, ThreadFactory threadFactory) { - this.timer = timer; - this.snapshotIntervalMs = (long)(config.snapshot_interval() * TimeUnit.SECONDS.toMillis(1)); - this.lastSnapshotTimeMs = timer.currentTimeMillis(); + public StateMonitor(HealthMonitorConfig config) { this.status = Status.valueOf(config.initialStatus()); - this.executor = Optional.ofNullable(threadFactory).map(Executors::newSingleThreadScheduledExecutor); - this.executor.ifPresent(exec -> exec.scheduleAtFixedRate(this::updateSnapshot, snapshotIntervalMs, - snapshotIntervalMs, TimeUnit.MILLISECONDS)); } - /** Returns a metric consumer for jDisc which will write metrics back to this */ - public MetricConsumer newMetricConsumer() { - StateMetricConsumer consumer = new StateMetricConsumer(); - consumers.add(consumer); - return consumer; + public static StateMonitor createForTesting() { + return new StateMonitor(new HealthMonitorConfig.Builder().build()); } public void status(Status status) { @@ -81,60 +39,4 @@ public class StateMonitor extends AbstractComponent { public Status status() { return status; } - /** Returns the last snapshot taken of the metrics in this system */ - public MetricSnapshot snapshot() { - return snapshot; - } - - /** Returns the interval between each metrics snapshot used by this */ - public long getSnapshotIntervalMillis() { return snapshotIntervalMs; } - - /** NOTE: Non-private for unit testing only. **/ - void updateSnapshot() { - long now = timer.currentTimeMillis(); - snapshot = createSnapshot(lastSnapshotTimeMs, now); - lastSnapshotTimeMs = now; - } - - private MetricSnapshot createSnapshot(long fromMillis, long toMillis) { - MetricSnapshot snapshot = new MetricSnapshot(fromMillis, toMillis, TimeUnit.MILLISECONDS); - for (StateMetricConsumer consumer : consumers) { - snapshot.add(consumer.createSnapshot()); - } - updateNames(snapshot); - return snapshot; - } - - private void updateNames(MetricSnapshot current) { - TreeSet<String> seen = new TreeSet<>(); - for (Map.Entry<MetricDimensions, MetricSet> dimensionAndMetric : current) { - for (Map.Entry<String, MetricValue> nameAndMetric : dimensionAndMetric.getValue()) { - seen.add(nameAndMetric.getKey()); - } - } - synchronized (valueNames) { - for (String name : valueNames) { - if (!seen.contains(name)) { - current.add((MetricDimensions) StateMetricConsumer.NULL_CONTEXT, name, 0); - } - } - valueNames.addAll(seen); - } - } - - @Override - public void deconstruct() { - executor.ifPresent(exec -> { - exec.shutdown(); - try { - exec.awaitTermination(5, TimeUnit.SECONDS); - } catch (InterruptedException e) { - } - - if (!exec.isTerminated()) { - log.warning("StateMonitor failed to terminate within 5 seconds of interrupt signal. Ignoring."); - } - }); - } - } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/package-info.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/package-info.java index 4d83d69a402..52b7af2a02c 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/package-info.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/package-info.java @@ -1,12 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +/** + * Metrics implementation for jDisc. This consumes metrics over the jDisc metric API and + * makes these available for in-process consumption, and off-process through a jDisc handler. + */ @ExportPackage package com.yahoo.container.jdisc.state; import com.yahoo.osgi.annotation.ExportPackage; - -/** - * Metrics implementation for jDisc. This consumes metrics over the jDisc metric API - * and makes these available for in-process consumption through - * {@link com.yahoo.container.jdisc.state.StateMonitor#snapshot}, - * and off-process through a jDisc handler. - */ diff --git a/container-core/src/main/java/org/json/package-info.java b/container-core/src/main/java/org/json/package-info.java index 44630ad235a..7ca9fe91e31 100644 --- a/container-core/src/main/java/org/json/package-info.java +++ b/container-core/src/main/java/org/json/package-info.java @@ -1,5 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage package org.json; - +// TODO Vespa 8: stop providing org.json import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-core/src/main/resources/configdefinitions/metrics.metrics-presentation.def b/container-core/src/main/resources/configdefinitions/metrics.metrics-presentation.def index b6c40993ef5..aeb597ed326 100644 --- a/container-core/src/main/resources/configdefinitions/metrics.metrics-presentation.def +++ b/container-core/src/main/resources/configdefinitions/metrics.metrics-presentation.def @@ -2,6 +2,5 @@ namespace=metrics -## Sliding window means present the last n minutes of data, as opposed to -## presenting the newest completed n minute interval. +## TODO Vespa 8: remove slidingwindow bool default=true diff --git a/container-core/src/test/java/com/yahoo/container/handler/metrics/MetricsV2HandlerTest.java b/container-core/src/test/java/com/yahoo/container/handler/metrics/MetricsV2HandlerTest.java index 9020ed91026..ca4bec30322 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/metrics/MetricsV2HandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/metrics/MetricsV2HandlerTest.java @@ -1,17 +1,18 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler.metrics; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.github.tomakehurst.wiremock.junit.WireMockRule; import com.yahoo.container.jdisc.RequestHandlerTestDriver; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; import org.junit.Before; import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import java.io.BufferedReader; +import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.concurrent.Executors; @@ -34,6 +35,8 @@ import static org.junit.Assert.fail; */ public class MetricsV2HandlerTest { + private static final ObjectMapper jsonMapper = new ObjectMapper(); + private static final String URI_BASE = "http://localhost"; private static final String V2_URI = URI_BASE + V2_PATH; @@ -79,29 +82,29 @@ public class MetricsV2HandlerTest { @Test public void v2_response_contains_values_uri() throws Exception { String response = testDriver.sendRequest(V2_URI).readAll(); - JSONObject root = new JSONObject(response); + JsonNode root = jsonMapper.readTree(response); assertTrue(root.has("resources")); - JSONArray resources = root.getJSONArray("resources"); - assertEquals(1, resources.length()); + ArrayNode resources = (ArrayNode) root.get("resources"); + assertEquals(1, resources.size()); - JSONObject valuesUri = resources.getJSONObject(0); - assertEquals(VALUES_URI, valuesUri.getString("url")); + JsonNode valuesUri = resources.get(0); + assertEquals(VALUES_URI, valuesUri.get("url").textValue()); } @Ignore @Test - public void visually_inspect_values_response() throws Exception { - JSONObject responseJson = getResponseAsJson(null); - System.out.println(responseJson.toString(4)); + public void visually_inspect_values_response() { + JsonNode responseJson = getResponseAsJson(null); + System.out.println(responseJson); } @Test public void invalid_path_yields_error_response() throws Exception { String response = testDriver.sendRequest(V2_URI + "/invalid").readAll(); - JSONObject root = new JSONObject(response); + JsonNode root = jsonMapper.readTree(response); assertTrue(root.has("error")); - assertTrue(root.getString("error" ).startsWith("No content")); + assertTrue(root.get("error" ).textValue().startsWith("No content")); } @Test @@ -111,23 +114,23 @@ public class MetricsV2HandlerTest { } @Test - public void consumer_is_propagated_to_metrics_proxy_api() throws JSONException { - JSONObject responseJson = getResponseAsJson(CUSTOM_CONSUMER); + public void consumer_is_propagated_to_metrics_proxy_api() { + JsonNode responseJson = getResponseAsJson(CUSTOM_CONSUMER); - JSONObject firstNodeMetricsValues = - responseJson.getJSONArray("nodes").getJSONObject(0) - .getJSONObject("node") - .getJSONArray("metrics").getJSONObject(0) - .getJSONObject("values"); + JsonNode firstNodeMetricsValues = + responseJson.get("nodes").get(0) + .get("node") + .get("metrics").get(0) + .get("values"); assertTrue(firstNodeMetricsValues.has(REPLACED_CPU_METRIC)); } - private JSONObject getResponseAsJson(String consumer) { + private JsonNode getResponseAsJson(String consumer) { String response = testDriver.sendRequest(VALUES_URI + consumerQuery(consumer)).readAll(); try { - return new JSONObject(response); - } catch (JSONException e) { + return jsonMapper.readTree(response); + } catch (IOException e) { fail("Failed to create json object: " + e.getMessage()); throw new RuntimeException(e); } diff --git a/container-core/src/test/java/com/yahoo/container/handler/metrics/PrometheusV1HandlerTest.java b/container-core/src/test/java/com/yahoo/container/handler/metrics/PrometheusV1HandlerTest.java index a0e8c131c2b..9ffce6d1c28 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/metrics/PrometheusV1HandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/metrics/PrometheusV1HandlerTest.java @@ -1,15 +1,11 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler.metrics; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.github.tomakehurst.wiremock.junit.WireMockRule; import com.yahoo.container.jdisc.RequestHandlerTestDriver; -import java.io.BufferedReader; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.stream.Collectors; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; import org.junit.Before; import org.junit.Ignore; import org.junit.Rule; @@ -26,13 +22,14 @@ import static com.yahoo.container.handler.metrics.MetricsV2Handler.consumerQuery import static com.yahoo.container.handler.metrics.MetricsV2HandlerTest.getFileContents; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; /** * @author gjoranv */ public class PrometheusV1HandlerTest { + private static final ObjectMapper jsonMapper = new ObjectMapper(); + private static final String URI_BASE = "http://localhost"; private static final String V1_URI = URI_BASE + PrometheusV1Handler.V1_PATH; @@ -79,14 +76,14 @@ public class PrometheusV1HandlerTest { @Test public void v1_response_contains_values_uri() throws Exception { String response = testDriver.sendRequest(V1_URI).readAll(); - JSONObject root = new JSONObject(response); + JsonNode root = jsonMapper.readTree(response); assertTrue(root.has("resources")); - JSONArray resources = root.getJSONArray("resources"); - assertEquals(1, resources.length()); + ArrayNode resources = (ArrayNode) root.get("resources"); + assertEquals(1, resources.size()); - JSONObject valuesUri = resources.getJSONObject(0); - assertEquals(VALUES_URI, valuesUri.getString("url")); + JsonNode valuesUri = resources.get(0); + assertEquals(VALUES_URI, valuesUri.get("url").asText()); } @Ignore @@ -99,9 +96,9 @@ public class PrometheusV1HandlerTest { @Test public void invalid_path_yields_error_response() throws Exception { String response = testDriver.sendRequest(V1_URI + "/invalid").readAll(); - JSONObject root = new JSONObject(response); + JsonNode root = jsonMapper.readTree(response); assertTrue(root.has("error")); - assertTrue(root.getString("error" ).startsWith("No content")); + assertTrue(root.get("error" ).textValue().startsWith("No content")); } @Test diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/CoredumpGathererTest.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/CoredumpGathererTest.java index c1f7d790fa5..8a3d0e837c5 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/state/CoredumpGathererTest.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/state/CoredumpGathererTest.java @@ -1,8 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.state; -import org.json.JSONException; -import org.json.JSONObject; +import com.fasterxml.jackson.databind.JsonNode; import org.junit.Test; import java.nio.file.Path; @@ -17,12 +16,12 @@ import static org.junit.Assert.assertEquals; public class CoredumpGathererTest { @Test - public void finds_one_coredump() throws JSONException { - JSONObject packet = CoredumpGatherer.gatherCoredumpMetrics(new MockFileWrapper()); + public void finds_one_coredump() { + JsonNode packet = CoredumpGatherer.gatherCoredumpMetrics(new MockFileWrapper()); - assertEquals("system-coredumps-processing", packet.getString("application")); - assertEquals(1, packet.getInt("status_code")); - assertEquals("Found 1 coredump(s)", packet.getString("status_msg")); + assertEquals("system-coredumps-processing", packet.get("application").textValue()); + assertEquals(1, packet.get("status_code").intValue()); + assertEquals("Found 1 coredump(s)", packet.get("status_msg").textValue()); } diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/HostLifeGathererTest.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/HostLifeGathererTest.java index d025b9662d2..12852c9d54c 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/state/HostLifeGathererTest.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/state/HostLifeGathererTest.java @@ -1,8 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.state; -import org.json.JSONException; -import org.json.JSONObject; +import com.fasterxml.jackson.databind.JsonNode; import org.junit.Test; import java.nio.file.Path; @@ -16,15 +15,15 @@ import static org.junit.Assert.assertEquals; public class HostLifeGathererTest { @Test - public void host_is_alive() throws JSONException { - JSONObject packet = HostLifeGatherer.getHostLifePacket(new MockFileWrapper()); - JSONObject metrics = packet.getJSONObject("metrics"); - assertEquals("host_life", packet.getString("application")); - assertEquals(0, packet.getInt("status_code")); - assertEquals("OK", packet.getString("status_msg")); - - assertEquals(123l, metrics.getLong("uptime")); - assertEquals(1, metrics.getInt("alive")); + public void host_is_alive() { + JsonNode packet = HostLifeGatherer.getHostLifePacket(new MockFileWrapper()); + JsonNode metrics = packet.get("metrics"); + assertEquals("host_life", packet.get("application").textValue()); + assertEquals(0, packet.get("status_code").intValue()); + assertEquals("OK", packet.get("status_msg").textValue()); + + assertEquals(123L, metrics.get("uptime").longValue()); + assertEquals(1, metrics.get("alive").intValue()); } diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricConsumerProviders.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricConsumerProviders.java deleted file mode 100644 index 777f43c5607..00000000000 --- a/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricConsumerProviders.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.jdisc.state; - -import com.google.inject.Provider; -import com.yahoo.jdisc.application.MetricConsumer; - -/** - * @author Simon Thoresen Hult - */ -class MetricConsumerProviders { - - public static Provider<MetricConsumer> wrap(final StateMonitor statetMonitor) { - return new Provider<MetricConsumer>() { - - @Override - public MetricConsumer get() { - return statetMonitor.newMetricConsumer(); - } - }; - } -} diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java index ef700597537..89bde3eeb5b 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java @@ -3,12 +3,13 @@ package com.yahoo.container.jdisc.state; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.jdisc.Metric; +import com.yahoo.container.jdisc.RequestHandlerTestDriver; +import org.junit.Before; import org.junit.Test; import java.util.ArrayList; -import java.util.Collections; import java.util.List; +import java.util.Map; import static com.yahoo.container.jdisc.state.MetricsPacketsHandler.APPLICATION_KEY; import static com.yahoo.container.jdisc.state.MetricsPacketsHandler.DIMENSIONS_KEY; @@ -25,9 +26,21 @@ import static org.junit.Assert.assertTrue; */ public class MetricsPacketsHandlerTest extends StateHandlerTestBase { + private static final String APPLICATION_NAME = "state-handler-test-base"; + + private static MetricsPacketsHandler metricsPacketsHandler; + + @Before + public void setupHandler() { + metricsPacketsHandlerConfig = new MetricsPacketsHandlerConfig(new MetricsPacketsHandlerConfig.Builder() + .application(APPLICATION_NAME)); + metricsPacketsHandler = new MetricsPacketsHandler(monitor, timer, snapshotProviderRegistry, + metricsPresentationConfig, metricsPacketsHandlerConfig); + testDriver = new RequestHandlerTestDriver(metricsPacketsHandler); + } + @Test - public void only_status_packet_is_returned_prior_to_first_snapshot() throws Exception { - metric.add("not_included", 1, null); + public void status_packet_is_returned_prior_to_first_snapshot() throws Exception { String response = requestAsString("http://localhost/metrics-packets"); List<JsonNode> packets = toJsonPackets(response); @@ -41,7 +54,7 @@ public class MetricsPacketsHandlerTest extends StateHandlerTestBase { @Test public void metrics_are_included_after_snapshot() throws Exception { - metric.add("counter", 1, null); + createSnapshotWithCountMetric("counter", 1, null); List<JsonNode> packets = incrementTimeAndGetJsonPackets(); assertEquals(2, packets.size()); @@ -51,7 +64,7 @@ public class MetricsPacketsHandlerTest extends StateHandlerTestBase { @Test public void metadata_is_included_in_each_metrics_packet() throws Exception { - metric.add("counter", 1, null); + createSnapshotWithCountMetric("counter", 1, null); List<JsonNode> packets = incrementTimeAndGetJsonPackets(); JsonNode counterPacket = packets.get(1); @@ -62,7 +75,7 @@ public class MetricsPacketsHandlerTest extends StateHandlerTestBase { @Test public void timestamp_resolution_is_in_seconds() throws Exception { - metric.add("counter", 1, null); + createSnapshotWithCountMetric("counter", 1, null); List<JsonNode> packets = incrementTimeAndGetJsonPackets(); JsonNode counterPacket = packets.get(1); @@ -71,8 +84,10 @@ public class MetricsPacketsHandlerTest extends StateHandlerTestBase { @Test public void expected_aggregators_are_output_for_gauge_metrics() throws Exception{ - Metric.Context context = metric.createContext(Collections.singletonMap("dim1", "value1")); - metric.set("gauge", 0.2, null); + var context = StateMetricContext.newInstance(Map.of("dim1", "value1")); + var snapshot = new MetricSnapshot(); + snapshot.set(context, "gauge", 0.2); + snapshotProvider.setSnapshot(snapshot); List<JsonNode> packets = incrementTimeAndGetJsonPackets(); JsonNode gaugeMetric = packets.get(1).get(METRICS_KEY); @@ -84,8 +99,8 @@ public class MetricsPacketsHandlerTest extends StateHandlerTestBase { @Test public void dimensions_from_context_are_included() throws Exception { - Metric.Context context = metric.createContext(Collections.singletonMap("dim1", "value1")); - metric.add("counter", 1, context); + var context = StateMetricContext.newInstance(Map.of("dim1", "value1")); + createSnapshotWithCountMetric("counter", 1, context); List<JsonNode> packets = incrementTimeAndGetJsonPackets(); JsonNode counterPacket = packets.get(1); @@ -97,9 +112,11 @@ public class MetricsPacketsHandlerTest extends StateHandlerTestBase { @Test public void metrics_with_identical_dimensions_are_contained_in_the_same_packet() throws Exception { - Metric.Context context = metric.createContext(Collections.singletonMap("dim1", "value1")); - metric.add("counter1", 1, context); - metric.add("counter2", 2, context); + var context = StateMetricContext.newInstance(Map.of("dim1", "value1")); + var snapshot = new MetricSnapshot(); + snapshot.add(context, "counter1", 1); + snapshot.add(context, "counter2", 2); + snapshotProvider.setSnapshot(snapshot); List<JsonNode> packets = incrementTimeAndGetJsonPackets(); assertEquals(2, packets.size()); @@ -112,10 +129,12 @@ public class MetricsPacketsHandlerTest extends StateHandlerTestBase { @Test public void metrics_with_different_dimensions_get_separate_packets() throws Exception { - Metric.Context context1 = metric.createContext(Collections.singletonMap("dim1", "value1")); - Metric.Context context2 = metric.createContext(Collections.singletonMap("dim2", "value2")); - metric.add("counter1", 1, context1); - metric.add("counter2", 1, context2); + var context1 = StateMetricContext.newInstance(Map.of("dim1", "value1")); + var context2 = StateMetricContext.newInstance(Map.of("dim2", "value2")); + var snapshot = new MetricSnapshot(); + snapshot.add(context1, "counter1", 1); + snapshot.add(context2, "counter2", 2); + snapshotProvider.setSnapshot(snapshot); List<JsonNode> packets = incrementTimeAndGetJsonPackets(); assertEquals(3, packets.size()); @@ -145,4 +164,10 @@ public class MetricsPacketsHandlerTest extends StateHandlerTestBase { assertEquals(expected, counterMetrics.get(metricName).asLong()); } + private void createSnapshotWithCountMetric(String name, Number value, MetricDimensions context) { + var snapshot = new MetricSnapshot(); + snapshot.add(context, name, value); + snapshotProvider.setSnapshot(snapshot); + } + } diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/MockSnapshotProvider.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/MockSnapshotProvider.java new file mode 100644 index 00000000000..5cc65be855d --- /dev/null +++ b/container-core/src/test/java/com/yahoo/container/jdisc/state/MockSnapshotProvider.java @@ -0,0 +1,23 @@ +package com.yahoo.container.jdisc.state; + +import java.io.PrintStream; + +/** + * @author gjoranv + */ +public class MockSnapshotProvider implements SnapshotProvider { + + private MetricSnapshot snapshot; + + public void setSnapshot(MetricSnapshot snapshot) { + this.snapshot = snapshot; + } + + @Override + public MetricSnapshot latestSnapshot() { + return snapshot; + } + + @Override + public void histogram(PrintStream output) { } +} diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTest.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTest.java index 26a2f817acc..385eb627427 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTest.java @@ -3,14 +3,13 @@ package com.yahoo.container.jdisc.state; import com.fasterxml.jackson.databind.JsonNode; import com.yahoo.component.Vtag; -import com.yahoo.jdisc.Metric; +import com.yahoo.container.jdisc.RequestHandlerTestDriver; import com.yahoo.vespa.defaults.Defaults; +import org.junit.Before; import org.junit.Test; -import java.util.Collections; -import java.util.HashMap; import java.util.Map; -import java.util.TreeMap; +import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -21,237 +20,63 @@ import static org.junit.Assert.assertTrue; */ public class StateHandlerTest extends StateHandlerTestBase { + private static final String V1_URI = URI_BASE + "/state/v1/"; + private static StateHandler stateHandler; + + @Before + public void setupHandler() { + stateHandler = new StateHandler(monitor, timer, applicationMetadataConfig, snapshotProviderRegistry , metricsPresentationConfig); + testDriver = new RequestHandlerTestDriver(stateHandler); + } + @Test - public void testReportPriorToFirstSnapshot() throws Exception { - metric.add("foo", 1, null); - metric.set("bar", 4, null); - JsonNode json = requestAsJson("http://localhost/state/v1/all"); + public void testStatusReportPriorToFirstSnapshot() throws Exception { + JsonNode json = requestAsJson(V1_URI + "/all"); assertEquals(json.toString(), "up", json.get("status").get("code").asText()); assertFalse(json.toString(), json.get("metrics").has("values")); } @Test public void testReportIncludesMetricsAfterSnapshot() throws Exception { - metric.add("foo", 1, null); - metric.set("bar", 4, null); - advanceToNextSnapshot(); - JsonNode json1 = requestAsJson("http://localhost/state/v1/metrics"); + var snapshot = new MetricSnapshot(); + snapshot.add(null, "foo", 1); + snapshot.set(null, "bar", 4); + snapshotProvider.setSnapshot(snapshot); + + JsonNode json1 = requestAsJson(V1_URI + "metrics"); assertEquals(json1.toString(), "up", json1.get("status").get("code").asText()); assertEquals(json1.toString(), 2, json1.get("metrics").get("values").size()); - - metric.add("fuz", 1, metric.createContext(new HashMap<>(0))); - advanceToNextSnapshot(); - JsonNode json2 = requestAsJson("http://localhost/state/v1/metrics"); - assertEquals(json2.toString(), "up", json2.get("status").get("code").asText()); - assertEquals(json2.toString(), 3, json2.get("metrics").get("values").size()); - } - - /** - * Tests that we restart an metric when it changes type from gauge to counter or back. - * This may happen in practice on config reloads. - */ - @Test - public void testMetricTypeChangeIsAllowed() { - String metricName = "myMetric"; - Metric.Context metricContext = null; - - { - // Add a count metric - metric.add(metricName, 1, metricContext); - metric.add(metricName, 2, metricContext); - // Change it to a gauge metric - metric.set(metricName, 9, metricContext); - advanceToNextSnapshot(); - MetricValue resultingMetric = monitor.snapshot().iterator().next().getValue().get(metricName); - assertEquals(GaugeMetric.class, resultingMetric.getClass()); - assertEquals("Value was reset and produces the last gauge value", - 9.0, ((GaugeMetric) resultingMetric).getLast(), 0.000001); - } - - { - // Add a gauge metric - metric.set(metricName, 9, metricContext); - // Change it to a count metric - metric.add(metricName, 1, metricContext); - metric.add(metricName, 2, metricContext); - advanceToNextSnapshot(); - MetricValue resultingMetric = monitor.snapshot().iterator().next().getValue().get(metricName); - assertEquals(CountMetric.class, resultingMetric.getClass()); - assertEquals("Value was reset, and changed to add semantics giving 1+2", - 3, ((CountMetric) resultingMetric).getCount()); - } - } - - @Test - public void testAverageAggregationOfValues() throws Exception { - metric.set("bar", 4, null); - metric.set("bar", 5, null); - metric.set("bar", 7, null); - metric.set("bar", 2, null); - advanceToNextSnapshot(); - JsonNode json = requestAsJson("http://localhost/state/v1/all"); - assertEquals(json.toString(), "up", json.get("status").get("code").asText()); - assertEquals(json.toString(), 1, json.get("metrics").get("values").size()); - assertEquals(json.toString(), 4.5, - json.get("metrics").get("values").get(0).get("values").get("average").asDouble(), 0.001); } @Test - public void testSumAggregationOfCounts() throws Exception { - metric.add("foo", 1, null); - metric.add("foo", 1, null); - metric.add("foo", 2, null); - metric.add("foo", 1, null); - advanceToNextSnapshot(); - JsonNode json = requestAsJson("http://localhost/state/v1/all"); - assertEquals(json.toString(), "up", json.get("status").get("code").asText()); - assertEquals(json.toString(), 1, json.get("metrics").get("values").size()); - assertEquals(json.toString(), 5, - json.get("metrics").get("values").get(0).get("values").get("count").asDouble(), 0.001); - } + public void testCountMetricCount() throws Exception { + var snapshot = new MetricSnapshot(); + snapshot.add(null, "foo", 4); + snapshot.add(null, "foo", 2); + snapshotProvider.setSnapshot(snapshot); - @Test - public void testReadabilityOfJsonReport() throws Exception { - metric.add("foo", 1, null); - advanceToNextSnapshot(); - assertEquals("{\n" + - " \"metrics\": {\n" + - " \"snapshot\": {\n" + - " \"from\": 0,\n" + - " \"to\": 300\n" + - " },\n" + - " \"values\": [{\n" + - " \"name\": \"foo\",\n" + - " \"values\": {\n" + - " \"count\": 1,\n" + - " \"rate\": 0.0033333333333333335\n" + - " }\n" + - " }]\n" + - " },\n" + - " \"status\": {\"code\": \"up\"},\n" + - " \"time\": 300000\n" + - "}", - requestAsString("http://localhost/state/v1/all")); - - Metric.Context ctx = metric.createContext(Collections.singletonMap("component", "test")); - metric.set("bar", 2, ctx); - metric.set("bar", 3, ctx); - metric.set("bar", 4, ctx); - metric.set("bar", 5, ctx); - advanceToNextSnapshot(); - assertEquals("{\n" + - " \"metrics\": {\n" + - " \"snapshot\": {\n" + - " \"from\": 300,\n" + - " \"to\": 600\n" + - " },\n" + - " \"values\": [\n" + - " {\n" + - " \"name\": \"foo\",\n" + - " \"values\": {\n" + - " \"count\": 0,\n" + - " \"rate\": 0\n" + - " }\n" + - " },\n" + - " {\n" + - " \"dimensions\": {\"component\": \"test\"},\n" + - " \"name\": \"bar\",\n" + - " \"values\": {\n" + - " \"average\": 3.5,\n" + - " \"count\": 4,\n" + - " \"last\": 5,\n" + - " \"max\": 5,\n" + - " \"min\": 2,\n" + - " \"rate\": 0.013333333333333334,\n" + - " \"sum\": 14\n" + - " }\n" + - " }\n" + - " ]\n" + - " },\n" + - " \"status\": {\"code\": \"up\"},\n" + - " \"time\": 600000\n" + - "}", - requestAsString("http://localhost/state/v1/all")); - } - - @Test - public void testNotAggregatingCountsBeyondSnapshots() throws Exception { - metric.add("foo", 1, null); - metric.add("foo", 1, null); - advanceToNextSnapshot(); - metric.add("foo", 2, null); - metric.add("foo", 1, null); - advanceToNextSnapshot(); - JsonNode json = requestAsJson("http://localhost/state/v1/all"); + JsonNode json = requestAsJson(V1_URI + "all"); assertEquals(json.toString(), "up", json.get("status").get("code").asText()); assertEquals(json.toString(), 1, json.get("metrics").get("values").size()); - assertEquals(json.toString(), 3, + assertEquals(json.toString(), 6, json.get("metrics").get("values").get(0).get("values").get("count").asDouble(), 0.001); } @Test - public void testSnapshottingTimes() throws Exception { - metric.add("foo", 1, null); - metric.set("bar", 3, null); - // At this time we should not have done any snapshotting - { - JsonNode json = requestAsJson("http://localhost/state/v1/all"); - assertFalse(json.toString(), json.get("metrics").has("snapshot")); - } - // At this time first snapshot should have been generated - advanceToNextSnapshot(); - { - JsonNode json = requestAsJson("http://localhost/state/v1/all"); - assertTrue(json.toString(), json.get("metrics").has("snapshot")); - assertEquals(0.0, json.get("metrics").get("snapshot").get("from").asDouble(), 0.00001); - assertEquals(300.0, json.get("metrics").get("snapshot").get("to").asDouble(), 0.00001); - } - // No new snapshot - { - JsonNode json = requestAsJson("http://localhost/state/v1/all"); - assertTrue(json.toString(), json.get("metrics").has("snapshot")); - assertEquals(0.0, json.get("metrics").get("snapshot").get("from").asDouble(), 0.00001); - assertEquals(300.0, json.get("metrics").get("snapshot").get("to").asDouble(), 0.00001); - } - // A new snapshot - advanceToNextSnapshot(); - { - JsonNode json = requestAsJson("http://localhost/state/v1/all"); - assertTrue(json.toString(), json.get("metrics").has("snapshot")); - assertEquals(300.0, json.get("metrics").get("snapshot").get("from").asDouble(), 0.00001); - assertEquals(600.0, json.get("metrics").get("snapshot").get("to").asDouble(), 0.00001); - } - } - - @Test - public void testFreshStartOfValuesBeyondSnapshot() throws Exception { - metric.set("bar", 4, null); - metric.set("bar", 5, null); - advanceToNextSnapshot(); - metric.set("bar", 4, null); - metric.set("bar", 2, null); - advanceToNextSnapshot(); - JsonNode json = requestAsJson("http://localhost/state/v1/all"); - assertEquals(json.toString(), "up", json.get("status").get("code").asText()); - assertEquals(json.toString(), 1, json.get("metrics").get("values").size()); - assertEquals(json.toString(), 3, - json.get("metrics").get("values").get(0).get("values").get("average").asDouble(), 0.001); - } + public void gaugeSnapshotsTracksCountMinMaxAvgPerPeriod() throws Exception { + var snapshot = new MetricSnapshot(); + snapshot.set(null, "bar", 20); + snapshot.set(null, "bar", 40); + snapshotProvider.setSnapshot(snapshot); - @Test - public void snapshotsPreserveLastGaugeValue() throws Exception { - metric.set("bar", 4, null); - advanceToNextSnapshot(); - advanceToNextSnapshot(); - JsonNode json = requestAsJson("http://localhost/state/v1/all"); + JsonNode json = requestAsJson(V1_URI + "all"); JsonNode metricValues = getFirstMetricValueNode(json); - assertEquals(json.toString(), 4, metricValues.get("last").asDouble(), 0.001); - // Use 'last' as avg/min/max when none has been set explicitly during snapshot period - assertEquals(json.toString(), 4, metricValues.get("average").asDouble(), 0.001); - assertEquals(json.toString(), 4, metricValues.get("min").asDouble(), 0.001); - assertEquals(json.toString(), 4, metricValues.get("max").asDouble(), 0.001); - // Count is tracked per period. - assertEquals(json.toString(), 0, metricValues.get("count").asInt()); + assertEquals(json.toString(), 40, metricValues.get("last").asDouble(), 0.001); + // Last snapshot had explicit values set + assertEquals(json.toString(), 30, metricValues.get("average").asDouble(), 0.001); + assertEquals(json.toString(), 20, metricValues.get("min").asDouble(), 0.001); + assertEquals(json.toString(), 40, metricValues.get("max").asDouble(), 0.001); + assertEquals(json.toString(), 2, metricValues.get("count").asInt()); } private JsonNode getFirstMetricValueNode(JsonNode root) { @@ -262,38 +87,65 @@ public class StateHandlerTest extends StateHandlerTestBase { } @Test - public void gaugeSnapshotsTracksCountMinMaxAvgPerPeriod() throws Exception { - metric.set("bar", 10000, null); // Ensure any cross-snapshot noise is visible - advanceToNextSnapshot(); - metric.set("bar", 20, null); - metric.set("bar", 40, null); + public void testReadabilityOfJsonReport() { + var snapshot = new MetricSnapshot(0L, SNAPSHOT_INTERVAL, TimeUnit.MILLISECONDS); + snapshot.add(null, "foo", 1); + var ctx = StateMetricContext.newInstance(Map.of("component", "test")); + snapshot.set(ctx, "bar", 2); + snapshot.set(ctx, "bar", 3); + snapshot.set(ctx, "bar", 4); + snapshot.set(ctx, "bar", 5); + snapshotProvider.setSnapshot(snapshot); advanceToNextSnapshot(); - JsonNode json = requestAsJson("http://localhost/state/v1/all"); - JsonNode metricValues = getFirstMetricValueNode(json); - assertEquals(json.toString(), 40, metricValues.get("last").asDouble(), 0.001); - // Last snapshot had explicit values set - assertEquals(json.toString(), 30, metricValues.get("average").asDouble(), 0.001); - assertEquals(json.toString(), 20, metricValues.get("min").asDouble(), 0.001); - assertEquals(json.toString(), 40, metricValues.get("max").asDouble(), 0.001); - assertEquals(json.toString(), 2, metricValues.get("count").asInt()); + assertEquals("{\n" + + " \"time\" : 300000,\n" + + " \"status\" : {\n" + + " \"code\" : \"up\"\n" + + " },\n" + + " \"metrics\" : {\n" + + " \"snapshot\" : {\n" + + " \"from\" : 0.0,\n" + + " \"to\" : 300.0\n" + + " },\n" + + " \"values\" : [ {\n" + + " \"name\" : \"foo\",\n" + + " \"values\" : {\n" + + " \"count\" : 1,\n" + + " \"rate\" : 0.0033333333333333335\n" + + " }\n" + + " }, {\n" + + " \"name\" : \"bar\",\n" + + " \"values\" : {\n" + + " \"average\" : 3.5,\n" + + " \"sum\" : 14.0,\n" + + " \"count\" : 4,\n" + + " \"last\" : 5.0,\n" + + " \"max\" : 5.0,\n" + + " \"min\" : 2.0,\n" + + " \"rate\" : 0.013333333333333334\n" + + " },\n" + + " \"dimensions\" : {\n" + + " \"component\" : \"test\"\n" + + " }\n" + + " } ]\n" + + " }\n" + + "}", + requestAsString(V1_URI + "all")); } @Test public void testHealthAggregation() throws Exception { - Map<String, String> dimensions1 = new TreeMap<>(); - dimensions1.put("port", String.valueOf(Defaults.getDefaults().vespaWebServicePort())); - Metric.Context context1 = metric.createContext(dimensions1); - Map<String, String> dimensions2 = new TreeMap<>(); - dimensions2.put("port", "80"); - Metric.Context context2 = metric.createContext(dimensions2); - - metric.add("serverNumSuccessfulResponses", 4, context1); - metric.add("serverNumSuccessfulResponses", 2, context2); - metric.set("serverTotalSuccessfulResponseLatency", 20, context1); - metric.set("serverTotalSuccessfulResponseLatency", 40, context2); - metric.add("random", 3, context1); - advanceToNextSnapshot(); - JsonNode json = requestAsJson("http://localhost/state/v1/health"); + var context1 = StateMetricContext.newInstance(Map.of("port", Defaults.getDefaults().vespaWebServicePort())); + var context2 = StateMetricContext.newInstance(Map.of("port", 80)); + var snapshot = new MetricSnapshot(); + snapshot.add(context1, "serverNumSuccessfulResponses", 4); + snapshot.add(context2, "serverNumSuccessfulResponses", 2); + snapshot.set(context1, "serverTotalSuccessfulResponseLatency", 20); + snapshot.set(context2, "serverTotalSuccessfulResponseLatency", 40); + snapshot.add(context1, "random", 3); + snapshotProvider.setSnapshot(snapshot); + + JsonNode json = requestAsJson(V1_URI + "health"); assertEquals(json.toString(), "up", json.get("status").get("code").asText()); assertEquals(json.toString(), 2, json.get("metrics").get("values").size()); assertEquals(json.toString(), "requestsPerSecond", @@ -308,7 +160,7 @@ public class StateHandlerTest extends StateHandlerTestBase { @Test public void testStateConfig() throws Exception { - JsonNode root = requestAsJson("http://localhost/state/v1/config"); + JsonNode root = requestAsJson(V1_URI + "config"); JsonNode config = root.get("config"); JsonNode container = config.get("container"); @@ -317,7 +169,7 @@ public class StateHandlerTest extends StateHandlerTestBase { @Test public void testStateVersion() throws Exception { - JsonNode root = requestAsJson("http://localhost/state/v1/version"); + JsonNode root = requestAsJson(V1_URI + "version"); JsonNode version = root.get("version"); assertEquals(Vtag.currentVersion.toString(), version.asText()); diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java index 57a9ba4abdb..68c1ae10ec6 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateHandlerTestBase.java @@ -3,103 +3,63 @@ package com.yahoo.container.jdisc.state; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.inject.AbstractModule; +import com.yahoo.component.ComponentId; +import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.core.ApplicationMetadataConfig; +import com.yahoo.container.jdisc.RequestHandlerTestDriver; import com.yahoo.container.jdisc.config.HealthMonitorConfig; -import com.yahoo.jdisc.Metric; -import com.yahoo.jdisc.Response; import com.yahoo.jdisc.Timer; -import com.yahoo.jdisc.application.ContainerBuilder; -import com.yahoo.jdisc.application.MetricConsumer; -import com.yahoo.jdisc.handler.BufferedContentChannel; -import com.yahoo.jdisc.handler.ContentChannel; -import com.yahoo.jdisc.handler.ResponseHandler; -import com.yahoo.jdisc.test.TestDriver; import com.yahoo.metrics.MetricsPresentationConfig; -import org.junit.After; import org.junit.Before; +import org.junit.BeforeClass; -import java.io.InputStreamReader; -import java.io.Reader; -import java.nio.charset.StandardCharsets; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; /** - * @author Simon Thoresen Hult * @author gjoranv */ public class StateHandlerTestBase { final static long SNAPSHOT_INTERVAL = TimeUnit.SECONDS.toMillis(300); final static long META_GENERATION = 69; - static final String APPLICATION_NAME = "state-handler-test-base"; - TestDriver driver; - StateMonitor monitor; - Metric metric; + + static final String URI_BASE = "http://localhost"; + + static StateMonitor monitor; + static RequestHandlerTestDriver testDriver; + + static HealthMonitorConfig healthMonitorConfig; + static ApplicationMetadataConfig applicationMetadataConfig; + static MetricsPresentationConfig metricsPresentationConfig; + static MetricsPacketsHandlerConfig metricsPacketsHandlerConfig; + final AtomicLong currentTimeMillis = new AtomicLong(0); + Timer timer; - @Before - public void startTestDriver() { - Timer timer = this.currentTimeMillis::get; - this.driver = TestDriver.newSimpleApplicationInstanceWithoutOsgi(new AbstractModule() { - @Override - protected void configure() { - bind(Timer.class).toInstance(timer); - } - }); - ContainerBuilder builder = driver.newContainerBuilder(); - HealthMonitorConfig healthMonitorConfig = - new HealthMonitorConfig( - new HealthMonitorConfig.Builder() - .snapshot_interval(TimeUnit.MILLISECONDS.toSeconds(SNAPSHOT_INTERVAL)) - .initialStatus("up")); - this.monitor = new StateMonitor(healthMonitorConfig, timer, null); - builder.guiceModules().install(new AbstractModule() { + MockSnapshotProvider snapshotProvider; + ComponentRegistry<SnapshotProvider> snapshotProviderRegistry; - @Override - protected void configure() { - bind(StateMonitor.class).toInstance(monitor); - bind(MetricConsumer.class).toProvider(MetricConsumerProviders.wrap(monitor)); - bind(ApplicationMetadataConfig.class).toInstance(new ApplicationMetadataConfig( - new ApplicationMetadataConfig.Builder().generation(META_GENERATION))); - bind(MetricsPresentationConfig.class) - .toInstance(new MetricsPresentationConfig(new MetricsPresentationConfig.Builder())); - bind(MetricsPacketsHandlerConfig.class).toInstance(new MetricsPacketsHandlerConfig( - new MetricsPacketsHandlerConfig.Builder().application(APPLICATION_NAME))); - } - }); - builder.serverBindings().bind("http://*/*", builder.getInstance(StateHandler.class)); - builder.serverBindings().bind("http://*/metrics-packets", builder.getInstance(MetricsPacketsHandler.class)); - driver.activateContainer(builder); - metric = builder.getInstance(Metric.class); + @BeforeClass + public static void setupClass() { + metricsPresentationConfig = new MetricsPresentationConfig(new MetricsPresentationConfig.Builder()); + healthMonitorConfig = new HealthMonitorConfig(new HealthMonitorConfig.Builder() + .initialStatus("up")); + applicationMetadataConfig = new ApplicationMetadataConfig(new ApplicationMetadataConfig.Builder() + .generation(META_GENERATION)); } - @After - public void stopTestDriver() { - assertTrue(driver.close()); + @Before + public void setupSnapshotProvider() { + timer = currentTimeMillis::get; + snapshotProvider = new MockSnapshotProvider(); + snapshotProviderRegistry = new ComponentRegistry<>(); + snapshotProviderRegistry.register(new ComponentId("foo"), snapshotProvider); + monitor = new StateMonitor(healthMonitorConfig); } - String requestAsString(String requestUri) throws Exception { - final BufferedContentChannel content = new BufferedContentChannel(); - Response response = driver.dispatchRequest(requestUri, new ResponseHandler() { - - @Override - public ContentChannel handleResponse(Response response) { - return content; - } - }).get(60, TimeUnit.SECONDS); - assertNotNull(response); - assertEquals(Response.Status.OK, response.getStatus()); - StringBuilder str = new StringBuilder(); - Reader in = new InputStreamReader(content.toStream(), StandardCharsets.UTF_8); - for (int c; (c = in.read()) != -1; ) { - str.append((char)c); - } - return str.toString(); + String requestAsString(String requestUri) { + return testDriver.sendRequest(requestUri).readAll(); } JsonNode requestAsJson(String requestUri) throws Exception { @@ -109,7 +69,6 @@ public class StateHandlerTestBase { void advanceToNextSnapshot() { currentTimeMillis.addAndGet(SNAPSHOT_INTERVAL); - monitor.updateSnapshot(); } } diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateMonitorBenchmarkTest.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/StateMonitorBenchmarkTest.java deleted file mode 100644 index 22a2fc274c7..00000000000 --- a/container-core/src/test/java/com/yahoo/container/jdisc/state/StateMonitorBenchmarkTest.java +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.jdisc.state; - -import com.google.inject.Provider; -import com.yahoo.jdisc.Metric; -import com.yahoo.jdisc.application.ContainerThread; -import com.yahoo.jdisc.application.MetricConsumer; -import com.yahoo.jdisc.application.MetricProvider; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertTrue; - -/** - * @author Simon Thoresen Hult - */ -public class StateMonitorBenchmarkTest { - - private final static int NUM_THREADS = 32; - private final static int NUM_UPDATES = 1000;//0000; - - @Test - public void requireThatHealthMonitorDoesNotBlockMetricThreads() throws Exception { - StateMonitor monitor = StateMonitor.createForTesting(); - Provider<MetricConsumer> provider = MetricConsumerProviders.wrap(monitor); - performUpdates(provider, 8); - for (int i = 1; i <= NUM_THREADS; i *= 2) { - long millis = performUpdates(provider, i); - System.err.format("%2d threads, %5d millis => %9d ups\n", - i, millis, (int)((i * NUM_UPDATES) / (millis / 1000.0))); - } - monitor.deconstruct(); - } - - private long performUpdates(Provider<MetricConsumer> metricProvider, int numThreads) throws Exception { - ThreadFactory threadFactory = new ContainerThread.Factory(metricProvider); - ExecutorService executor = Executors.newFixedThreadPool(numThreads, threadFactory); - List<Callable<Boolean>> tasks = new ArrayList<>(numThreads); - for (int i = 0; i < numThreads; ++i) { - tasks.add(new UpdateTask(new MetricProvider(metricProvider).get())); - } - long before = System.nanoTime(); - List<Future<Boolean>> results = executor.invokeAll(tasks); - long after = System.nanoTime(); - for (Future<Boolean> result : results) { - assertTrue(result.get()); - } - return TimeUnit.NANOSECONDS.toMillis(after - before); - } - - public static class UpdateTask implements Callable<Boolean> { - - final Metric metric; - - UpdateTask(Metric metric) { - this.metric = metric; - } - - @Override - public Boolean call() throws Exception { - Metric.Context ctx = metric.createContext(Collections.<String, Object>emptyMap()); - for (int i = 0; i < NUM_UPDATES; ++i) { - metric.add("foo", 69L, ctx); - } - return true; - } - } -} diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index 8691d9a7ffb..08d6e0103bf 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -313,7 +313,7 @@ <artifactId>javassist</artifactId> <version>${javassist.version}</version> </dependency> - <dependency> <!-- TODO Vespa 8: upgrade to newest version. Consider removing as provided dependency --> + <dependency> <!-- TODO Vespa 8: remove as provided dependency --> <groupId>org.json</groupId> <artifactId>json</artifactId> <version>${org.json.version}</version> diff --git a/container-di/src/main/java/com/yahoo/container/di/CloudSubscriberFactory.java b/container-di/src/main/java/com/yahoo/container/di/CloudSubscriberFactory.java index d43f96c9b4a..065733a719a 100644 --- a/container-di/src/main/java/com/yahoo/container/di/CloudSubscriberFactory.java +++ b/container-di/src/main/java/com/yahoo/container/di/CloudSubscriberFactory.java @@ -114,7 +114,8 @@ public class CloudSubscriberFactory implements SubscriberFactory { catch (IllegalArgumentException e) { numExceptions++; log.log(Level.WARNING, "Got exception from the config system (ignore if you just removed a " + - "component from your application that used the mentioned config): ", e); + "component from your application that used the mentioned config) Subscriber info: " + + subscriber.toString(), e); if (numExceptions >= 5) throw new IllegalArgumentException("Failed retrieving the next config generation", e); } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProvider.java index a300364d848..9b907639e57 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProvider.java @@ -4,10 +4,7 @@ package com.yahoo.container.jdisc.metric; import com.google.inject.Inject; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.jdisc.MetricConsumerFactory; -import com.yahoo.container.jdisc.metric.state.StateMetricConsumerFactory; -import com.yahoo.container.jdisc.state.StateMonitor; import com.yahoo.jdisc.application.MetricConsumer; -import com.yahoo.metrics.MetricsPresentationConfig; /** @@ -27,17 +24,9 @@ public class MetricConsumerProvider { private final MetricConsumerFactory[] factories; @Inject - public MetricConsumerProvider(ComponentRegistry<MetricConsumerFactory> factoryRegistry, - MetricsPresentationConfig presentationConfig, - StateMonitor stateMonitor) { - MetricConsumerFactory[] factories; - if (factoryRegistry.getComponentCount() == 0 || ! presentationConfig.slidingwindow()) { - factories = new MetricConsumerFactory[1]; - factories[0] = new StateMetricConsumerFactory(stateMonitor); - } else { - factories = new MetricConsumerFactory[factoryRegistry.getComponentCount()]; - factoryRegistry.allComponents().toArray(factories); - } + public MetricConsumerProvider(ComponentRegistry<MetricConsumerFactory> factoryRegistry) { + MetricConsumerFactory[] factories = new MetricConsumerFactory[factoryRegistry.getComponentCount()]; + factoryRegistry.allComponents().toArray(factories); this.factories = factories; } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProviderProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProviderProvider.java index a44650a153d..ca3d47bc338 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProviderProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricConsumerProviderProvider.java @@ -20,10 +20,8 @@ public class MetricConsumerProviderProvider implements Provider<MetricConsumerPr private final MetricConsumerProvider provided; @Inject - public MetricConsumerProviderProvider(ComponentRegistry<MetricConsumerFactory> factoryRegistry, - MetricsPresentationConfig presentationConfig, - StateMonitor stateMonitor) { - provided = new MetricConsumerProvider(factoryRegistry, presentationConfig, stateMonitor); + public MetricConsumerProviderProvider(ComponentRegistry<MetricConsumerFactory> factoryRegistry) { + provided = new MetricConsumerProvider(factoryRegistry); } @Override diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/state/StateMetricConsumerFactory.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/state/StateMetricConsumerFactory.java deleted file mode 100644 index d1cf660f571..00000000000 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/state/StateMetricConsumerFactory.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.container.jdisc.metric.state; - -import com.yahoo.container.jdisc.MetricConsumerFactory; -import com.yahoo.container.jdisc.state.StateMonitor; -import com.yahoo.jdisc.application.MetricConsumer; - -/** - * @author Simon Thoresen Hult - */ -public class StateMetricConsumerFactory implements MetricConsumerFactory { - - private final StateMonitor stateMonitor; - - public StateMetricConsumerFactory(StateMonitor stateMonitor) { - this.stateMonitor = stateMonitor; - } - - @Override - public MetricConsumer newInstance() { - return stateMonitor.newMetricConsumer(); - } - -} diff --git a/container-disc/src/main/java/com/yahoo/container/usability/BindingsOverviewHandler.java b/container-disc/src/main/java/com/yahoo/container/usability/BindingsOverviewHandler.java index 709441999d0..df7cacdc768 100644 --- a/container-disc/src/main/java/com/yahoo/container/usability/BindingsOverviewHandler.java +++ b/container-disc/src/main/java/com/yahoo/container/usability/BindingsOverviewHandler.java @@ -1,9 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.usability; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.inject.Inject; import com.yahoo.component.ComponentId; -import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.Container; import com.yahoo.container.jdisc.JdiscBindingsConfig; import com.yahoo.jdisc.handler.AbstractRequestHandler; @@ -15,16 +19,11 @@ import com.yahoo.jdisc.handler.ResponseDispatch; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.jdisc.http.HttpRequest.Method; - -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; import org.osgi.framework.Bundle; import org.osgi.framework.FrameworkUtil; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; @@ -33,6 +32,8 @@ import java.util.Map; */ public class BindingsOverviewHandler extends AbstractRequestHandler { + private static final ObjectMapper jsonMapper = new ObjectMapper(); + private final JdiscBindingsConfig bindingsConfig; @Inject @@ -42,7 +43,7 @@ public class BindingsOverviewHandler extends AbstractRequestHandler { @Override public ContentChannel handleRequest(com.yahoo.jdisc.Request request, ResponseHandler handler) { - JSONObject json; + JsonNode json; int statusToReturn; if (request instanceof HttpRequest && ((HttpRequest) request).getMethod() != Method.GET) { @@ -63,7 +64,9 @@ public class BindingsOverviewHandler extends AbstractRequestHandler { }.connect(handler)); try { - writer.write(json.toString()); + writer.write(jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsBytes(json)); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); } finally { writer.close(); } @@ -71,63 +74,58 @@ public class BindingsOverviewHandler extends AbstractRequestHandler { return new IgnoredContent(); } - private JSONObject errorMessageInJson() { - JSONObject error = new JSONObject(); - try { - error.put("error", "This API, " - + this.getClass().getSimpleName() - + ", only supports HTTP GET." - + " You are probably looking for another API/path."); - } catch (org.json.JSONException e) { - // just ignore it - } + private static JsonNode errorMessageInJson() { + ObjectNode error = jsonMapper.createObjectNode(); + error.put("error", "This API, " + + BindingsOverviewHandler.class.getSimpleName() + + ", only supports HTTP GET." + + " You are probably looking for another API/path."); return error; } - static JSONArray renderRequestHandlers(JdiscBindingsConfig bindingsConfig, + static ArrayNode renderRequestHandlers(JdiscBindingsConfig bindingsConfig, Map<ComponentId, ? extends RequestHandler> handlersById) { - JSONArray ret = new JSONArray(); + ArrayNode ret = jsonMapper.createArrayNode(); for (Map.Entry<ComponentId, ? extends RequestHandler> handlerEntry : handlersById.entrySet()) { String id = handlerEntry.getKey().stringValue(); RequestHandler handler = handlerEntry.getValue(); - JSONObject handlerJson = renderComponent(handler, handlerEntry.getKey()); + ObjectNode handlerJson = renderComponent(handler, handlerEntry.getKey()); addBindings(bindingsConfig, id, handlerJson); - ret.put(handlerJson); + ret.add(handlerJson); } return ret; } - private static void addBindings(JdiscBindingsConfig bindingsConfig, String id, JSONObject handlerJson) { + private static void addBindings(JdiscBindingsConfig bindingsConfig, String id, ObjectNode handlerJson) { List<String> serverBindings = new ArrayList<>(); JdiscBindingsConfig.Handlers handlerConfig = bindingsConfig.handlers(id); if (handlerConfig != null) { serverBindings = handlerConfig.serverBindings(); } - putJson(handlerJson, "serverBindings", renderBindings(serverBindings)); + handlerJson.set("serverBindings", renderBindings(serverBindings)); } - private static JSONArray renderBindings(List<String> bindings) { - JSONArray array = new JSONArray(); + private static JsonNode renderBindings(List<String> bindings) { + ArrayNode array = jsonMapper.createArrayNode(); for (String binding : bindings) - array.put(binding); + array.add(binding); return array; } - private static JSONObject renderComponent(Object component, ComponentId id) { - JSONObject jc = new JSONObject(); - putJson(jc, "id", id.stringValue()); + private static ObjectNode renderComponent(Object component, ComponentId id) { + ObjectNode jc = jsonMapper.createObjectNode(); + jc.put("id", id.stringValue()); addBundleInfo(jc, component); return jc; } - private static void addBundleInfo(JSONObject jsonObject, Object component) { + private static void addBundleInfo(ObjectNode jsonObject, Object component) { BundleInfo bundleInfo = bundleInfo(component); - putJson(jsonObject, "class", bundleInfo.className); - putJson(jsonObject, "bundle", bundleInfo.bundleName); - + jsonObject.put("class", bundleInfo.className); + jsonObject.put("bundle", bundleInfo.bundleName); } private static BundleInfo bundleInfo(Object component) { @@ -143,15 +141,6 @@ public class BindingsOverviewHandler extends AbstractRequestHandler { } } - private static void putJson(JSONObject json, String key, Object value) { - try { - json.put(key, value); - } catch (JSONException e) { - // The original JSONException lacks key-value info. - throw new RuntimeException("Trying to add invalid JSON object with key '" + key + "' and value '" + value + "' - " + e.getMessage(), e); - } - } - static final class BundleInfo { public final String className; @@ -172,10 +161,10 @@ public class BindingsOverviewHandler extends AbstractRequestHandler { this.bindingsConfig = bindingsConfig; } - public JSONObject render() { - JSONObject root = new JSONObject(); + public JsonNode render() { + ObjectNode root = jsonMapper.createObjectNode(); - putJson(root, "handlers", + root.set("handlers", renderRequestHandlers(bindingsConfig, Container.get().getRequestHandlerRegistry().allComponentsById())); return root; diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricConsumerProviderTest.java b/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricConsumerProviderTest.java index 5e3a49c2dda..0acbb701612 100644 --- a/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricConsumerProviderTest.java +++ b/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricConsumerProviderTest.java @@ -34,10 +34,4 @@ public class MetricConsumerProviderTest { Mockito.verify(bar, Mockito.times(1)).add("foo", 6, null); } - @Test - public void requireThatDefaultConsumerFactoryIsStateMetric() { - MetricConsumer consumer = MetricConsumerProviders.newDefaultInstance().newInstance(); - assertEquals("StateMetricConsumer", consumer.getClass().getSimpleName()); - } - } diff --git a/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricConsumerProviders.java b/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricConsumerProviders.java index dd96414bbc7..4e576d2cd8c 100644 --- a/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricConsumerProviders.java +++ b/container-disc/src/test/java/com/yahoo/container/jdisc/metric/MetricConsumerProviders.java @@ -22,15 +22,7 @@ class MetricConsumerProviders { } public static MetricConsumerProvider newInstance(MetricConsumerFactory... factories) { - return new MetricConsumerProvider(newComponentRegistry(factories), - new MetricsPresentationConfig(new MetricsPresentationConfig.Builder()), - StateMonitor.createForTesting()); - } - - public static MetricConsumerProvider newDefaultInstance() { - return new MetricConsumerProvider(newComponentRegistry(), - new MetricsPresentationConfig(new MetricsPresentationConfig.Builder()), - StateMonitor.createForTesting()); + return new MetricConsumerProvider(newComponentRegistry(factories)); } public static ComponentRegistry<MetricConsumerFactory> newComponentRegistry(MetricConsumerFactory... factories) { diff --git a/container-search-and-docproc/src/main/java/com/yahoo/container/handler/observability/ApplicationStatusHandler.java b/container-search-and-docproc/src/main/java/com/yahoo/container/handler/observability/ApplicationStatusHandler.java index 57c5e768cfb..943eef1e0bf 100644 --- a/container-search-and-docproc/src/main/java/com/yahoo/container/handler/observability/ApplicationStatusHandler.java +++ b/container-search-and-docproc/src/main/java/com/yahoo/container/handler/observability/ApplicationStatusHandler.java @@ -1,6 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler.observability; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.component.ComponentId; @@ -29,15 +34,11 @@ import com.yahoo.processing.execution.chain.ChainRegistry; import com.yahoo.processing.handler.ProcessingHandler; import com.yahoo.search.handler.SearchHandler; import com.yahoo.search.searchchain.SearchChainRegistry; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; import org.osgi.framework.Bundle; import org.osgi.framework.FrameworkUtil; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Arrays; import java.util.Date; import java.util.Iterator; import java.util.List; @@ -51,11 +52,13 @@ import java.util.Map; */ public class ApplicationStatusHandler extends AbstractRequestHandler { - private final JSONObject applicationJson; - private final JSONArray clientsJson; - private final JSONArray serversJson; - private final JSONArray requestFiltersJson; - private final JSONArray responseFiltersJson; + private static final ObjectMapper jsonMapper = new ObjectMapper(); + + private final JsonNode applicationJson; + private final JsonNode clientsJson; + private final JsonNode serversJson; + private final JsonNode requestFiltersJson; + private final JsonNode responseFiltersJson; private final JdiscBindingsConfig bindingsConfig; @Inject @@ -78,7 +81,7 @@ public class ApplicationStatusHandler extends AbstractRequestHandler { @Override public ContentChannel handleRequest(com.yahoo.jdisc.Request request, ResponseHandler handler) { - JSONObject json = new StatusResponse(applicationJson, clientsJson, serversJson, + JsonNode json = new StatusResponse(applicationJson, clientsJson, serversJson, requestFiltersJson, responseFiltersJson, bindingsConfig) .render(); @@ -91,62 +94,66 @@ public class ApplicationStatusHandler extends AbstractRequestHandler { } }.connect(handler)); - writer.write(json.toString()); + try { + writer.write(jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsBytes(json)); + } catch (JsonProcessingException e) { + throw new RuntimeException("Invalid JSON: " + e.getMessage(), e); + } writer.close(); return new IgnoredContent(); } - static JSONObject renderApplicationConfigs(ApplicationMetadataConfig metaConfig, + static JsonNode renderApplicationConfigs(ApplicationMetadataConfig metaConfig, ApplicationUserdataConfig userConfig) { - JSONObject vespa = new JSONObject(); - putJson(vespa, "version", Vtag.currentVersion); - - JSONObject meta = new JSONObject(); - putJson(meta, "name", metaConfig.name()); - putJson(meta, "user", metaConfig.user()); - putJson(meta, "path", metaConfig.path()); - putJson(meta, "generation", metaConfig.generation()); - putJson(meta, "timestamp", metaConfig.timestamp()); - putJson(meta, "date", new Date(metaConfig.timestamp()).toString()); - putJson(meta, "checksum", metaConfig.checksum()); - - JSONObject user = new JSONObject(); - putJson(user, "version", userConfig.version()); - - JSONObject application = new JSONObject(); - putJson(application, "vespa", vespa); - putJson(application, "meta", meta); - putJson(application, "user", user); + ObjectNode vespa = jsonMapper.createObjectNode(); + vespa.put("version", Vtag.currentVersion.toString()); + + ObjectNode meta = jsonMapper.createObjectNode(); + meta.put("name", metaConfig.name()); + meta.put("user", metaConfig.user()); + meta.put("path", metaConfig.path()); + meta.put("generation", metaConfig.generation()); + meta.put("timestamp", metaConfig.timestamp()); + meta.put("date", new Date(metaConfig.timestamp()).toString()); + meta.put("checksum", metaConfig.checksum()); + + ObjectNode user = jsonMapper.createObjectNode(); + user.put("version", userConfig.version()); + + ObjectNode application = jsonMapper.createObjectNode(); + application.set("vespa", vespa); + application.set("meta", meta); + application.set("user", user); return application; } - static JSONArray renderObjectComponents(Map<ComponentId, ?> componentsById) { - JSONArray ret = new JSONArray(); + static JsonNode renderObjectComponents(Map<ComponentId, ?> componentsById) { + ArrayNode ret = jsonMapper.createArrayNode(); for (Map.Entry<ComponentId, ?> componentEntry : componentsById.entrySet()) { - JSONObject jc = renderComponent(componentEntry.getValue(), componentEntry.getKey()); - ret.put(jc); + JsonNode jc = renderComponent(componentEntry.getValue(), componentEntry.getKey()); + ret.add(jc); } return ret; } - static JSONArray renderRequestHandlers(JdiscBindingsConfig bindingsConfig, + static JsonNode renderRequestHandlers(JdiscBindingsConfig bindingsConfig, Map<ComponentId, ? extends RequestHandler> handlersById) { - JSONArray ret = new JSONArray(); + ArrayNode ret = jsonMapper.createArrayNode(); for (Map.Entry<ComponentId, ? extends RequestHandler> handlerEntry : handlersById.entrySet()) { String id = handlerEntry.getKey().stringValue(); RequestHandler handler = handlerEntry.getValue(); - JSONObject handlerJson = renderComponent(handler, handlerEntry.getKey()); + ObjectNode handlerJson = renderComponent(handler, handlerEntry.getKey()); addBindings(bindingsConfig, id, handlerJson); - ret.put(handlerJson); + ret.add(handlerJson); } return ret; } - private static void addBindings(JdiscBindingsConfig bindingsConfig, String id, JSONObject handlerJson) { + private static void addBindings(JdiscBindingsConfig bindingsConfig, String id, ObjectNode handlerJson) { List<String> serverBindings = new ArrayList<>(); List<String> clientBindings = new ArrayList<>(); @@ -155,40 +162,40 @@ public class ApplicationStatusHandler extends AbstractRequestHandler { serverBindings = handlerConfig.serverBindings(); clientBindings = handlerConfig.clientBindings(); } - putJson(handlerJson, "serverBindings", renderBindings(serverBindings)); - putJson(handlerJson, "clientBindings", renderBindings(clientBindings)); + handlerJson.set("serverBindings", renderBindings(serverBindings)); + handlerJson.set("clientBindings", renderBindings(clientBindings)); } - private static JSONArray renderBindings(List<String> bindings) { - JSONArray ret = new JSONArray(); + private static JsonNode renderBindings(List<String> bindings) { + ArrayNode ret = jsonMapper.createArrayNode(); for (String binding : bindings) - ret.put(binding); + ret.add(binding); return ret; } - private static JSONArray renderAbstractComponents(List<? extends AbstractComponent> components) { - JSONArray ret = new JSONArray(); + private static JsonNode renderAbstractComponents(List<? extends AbstractComponent> components) { + ArrayNode ret = jsonMapper.createArrayNode(); for (AbstractComponent c : components) { - JSONObject jc = renderComponent(c, c.getId()); - ret.put(jc); + JsonNode jc = renderComponent(c, c.getId()); + ret.add(jc); } return ret; } - private static JSONObject renderComponent(Object component, ComponentId id) { - JSONObject jc = new JSONObject(); - putJson(jc, "id", id.stringValue()); + private static ObjectNode renderComponent(Object component, ComponentId id) { + ObjectNode jc = jsonMapper.createObjectNode(); + jc.put("id", id.stringValue()); addBundleInfo(jc, component); return jc; } - private static void addBundleInfo(JSONObject jsonObject, Object component) { + private static void addBundleInfo(ObjectNode jsonObject, Object component) { BundleInfo bundleInfo = bundleInfo(component); - putJson(jsonObject, "class", bundleInfo.className); - putJson(jsonObject, "bundle", bundleInfo.bundleName); + jsonObject.put("class", bundleInfo.className); + jsonObject.put("bundle", bundleInfo.bundleName); } @@ -205,15 +212,6 @@ public class ApplicationStatusHandler extends AbstractRequestHandler { } } - private static void putJson(JSONObject json, String key, Object value) { - try { - json.put(key, value); - } catch (JSONException e) { - // The original JSONException lacks key-value info. - throw new RuntimeException("Trying to add invalid JSON object with key '" + key + "' and value '" + value + "' - " + e.getMessage(), e); - } - } - static final class BundleInfo { public final String className; public final String bundleName; @@ -224,18 +222,18 @@ public class ApplicationStatusHandler extends AbstractRequestHandler { } static final class StatusResponse { - private final JSONObject applicationJson; - private final JSONArray clientsJson; - private final JSONArray serversJson; - private final JSONArray requestFiltersJson; - private final JSONArray responseFiltersJson; + private final JsonNode applicationJson; + private final JsonNode clientsJson; + private final JsonNode serversJson; + private final JsonNode requestFiltersJson; + private final JsonNode responseFiltersJson; private final JdiscBindingsConfig bindingsConfig; - StatusResponse(JSONObject applicationJson, - JSONArray clientsJson, - JSONArray serversJson, - JSONArray requestFiltersJson, - JSONArray responseFiltersJson, + StatusResponse(JsonNode applicationJson, + JsonNode clientsJson, + JsonNode serversJson, + JsonNode requestFiltersJson, + JsonNode responseFiltersJson, JdiscBindingsConfig bindingsConfig) { this.applicationJson = applicationJson; this.clientsJson = clientsJson; @@ -245,52 +243,52 @@ public class ApplicationStatusHandler extends AbstractRequestHandler { this.bindingsConfig = bindingsConfig; } - public JSONObject render() { - JSONObject root = new JSONObject(); + public JsonNode render() { + ObjectNode root = jsonMapper.createObjectNode(); - putJson(root, "application", applicationJson); - putJson(root, "abstractComponents", + root.set("application", applicationJson); + root.set("abstractComponents", renderAbstractComponents(Container.get().getComponentRegistry().allComponents())); - putJson(root, "handlers", + root.set("handlers", renderRequestHandlers(bindingsConfig, Container.get().getRequestHandlerRegistry().allComponentsById())); - putJson(root, "clients", clientsJson); - putJson(root, "servers", serversJson); - putJson(root, "httpRequestFilters", requestFiltersJson); - putJson(root, "httpResponseFilters", responseFiltersJson); + root.set("clients", clientsJson); + root.set("servers", serversJson); + root.set("httpRequestFilters", requestFiltersJson); + root.set("httpResponseFilters", responseFiltersJson); - putJson(root, "searchChains", renderSearchChains(Container.get())); - putJson(root, "docprocChains", renderDocprocChains(Container.get())); - putJson(root, "processingChains", renderProcessingChains(Container.get())); + root.set("searchChains", renderSearchChains(Container.get())); + root.set("docprocChains", renderDocprocChains(Container.get())); + root.set("processingChains", renderProcessingChains(Container.get())); return root; } - private static JSONObject renderSearchChains(Container container) { + private static JsonNode renderSearchChains(Container container) { for (RequestHandler h : container.getRequestHandlerRegistry().allComponents()) { if (h instanceof SearchHandler) { SearchChainRegistry scReg = ((SearchHandler) h).getSearchChainRegistry(); return renderChains(scReg); } } - return new JSONObject(); + return jsonMapper.createObjectNode(); } - private static JSONObject renderDocprocChains(Container container) { - JSONObject ret = new JSONObject(); + private static JsonNode renderDocprocChains(Container container) { + ObjectNode ret = jsonMapper.createObjectNode(); for (RequestHandler h : container.getRequestHandlerRegistry().allComponents()) { if (h instanceof DocumentProcessingHandler) { ComponentRegistry<DocprocService> registry = ((DocumentProcessingHandler) h).getDocprocServiceRegistry(); for (DocprocService service : registry.allComponents()) { - putJson(ret, service.getId().stringValue(), renderCalls(service.getCallStack().iterator())); + ret.set(service.getId().stringValue(), renderCalls(service.getCallStack().iterator())); } } } return ret; } - private static JSONObject renderProcessingChains(Container container) { - JSONObject ret = new JSONObject(); + private static JsonNode renderProcessingChains(Container container) { + JsonNode ret = jsonMapper.createObjectNode(); for (RequestHandler h : container.getRequestHandlerRegistry().allComponents()) { if (h instanceof ProcessingHandler) { ChainRegistry<Processor> registry = ((ProcessingHandler) h).getChainRegistry(); @@ -301,20 +299,20 @@ public class ApplicationStatusHandler extends AbstractRequestHandler { } // Note the generic param here! The key to make this work is '? extends Chain', but why? - static JSONObject renderChains(ComponentRegistry<? extends Chain<?>> chains) { - JSONObject ret = new JSONObject(); + static JsonNode renderChains(ComponentRegistry<? extends Chain<?>> chains) { + ObjectNode ret = jsonMapper.createObjectNode(); for (Chain<?> chain : chains.allComponents()) { - putJson(ret, chain.getId().stringValue(), renderAbstractComponents(chain.components())); + ret.set(chain.getId().stringValue(), renderAbstractComponents(chain.components())); } return ret; } - private static JSONArray renderCalls(Iterator<Call> components) { - JSONArray ret = new JSONArray(); + private static JsonNode renderCalls(Iterator<Call> components) { + ArrayNode ret = jsonMapper.createArrayNode(); while (components.hasNext()) { Call c = components.next(); - JSONObject jc = renderComponent(c.getDocumentProcessor(), c.getDocumentProcessor().getId()); - ret.put(jc); + JsonNode jc = renderComponent(c.getDocumentProcessor(), c.getDocumentProcessor().getId()); + ret.add(jc); } return ret; } diff --git a/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java b/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java index 9ddcc7a7e69..3132f3744c9 100644 --- a/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java +++ b/container-search-gui/src/main/java/com/yahoo/search/query/gui/GUIHandler.java @@ -1,8 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.query.gui; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.inject.Inject; - import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -11,6 +13,7 @@ import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.querytransform.RecallSearcher; import com.yahoo.restapi.Path; import com.yahoo.search.Query; +import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.query.Model; import com.yahoo.search.query.Presentation; import com.yahoo.search.query.Ranking; @@ -19,19 +22,12 @@ import com.yahoo.search.query.ranking.MatchPhase; import com.yahoo.search.query.restapi.ErrorResponse; import com.yahoo.search.yql.MinimalQueryInserter; import com.yahoo.vespa.config.search.RankProfilesConfig; -import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.yolean.Exceptions; -import org.json.JSONException; -import org.json.JSONObject; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; import java.util.logging.Level; @@ -42,6 +38,8 @@ import java.util.logging.Level; */ public class GUIHandler extends LoggingRequestHandler { + private static final ObjectMapper jsonMapper = new ObjectMapper(); + private final IndexModel indexModel; private final RankProfilesConfig rankProfilesConfig; @@ -112,7 +110,7 @@ public class GUIHandler extends LoggingRequestHandler { InputStream is; if (this.path.equals("config.json")){ String json = "{}"; - try { json = getGUIConfig(); } catch (JSONException e) { /*Something happened while parsing JSON */ } + try { json = getGUIConfig(); } catch (IOException e) { /*Something happened while parsing JSON */ } is = new ByteArrayInputStream(json.getBytes()); } else{ is = GUIHandler.class.getClassLoader().getResourceAsStream("gui/"+this.path); @@ -160,50 +158,50 @@ public class GUIHandler extends LoggingRequestHandler { return "text/html"; } - private String getGUIConfig() throws JSONException { - JSONObject json = new JSONObject(); - json.put("ranking_properties", Arrays.asList("propertyname")); - json.put("ranking_features", Arrays.asList("featurename")); + private String getGUIConfig() throws IOException { + ObjectNode json = jsonMapper.createObjectNode(); + json.set("ranking_properties", jsonMapper.createArrayNode().add("propertyname")); + json.set("ranking_features", jsonMapper.createArrayNode().add("featurename")); - List<String> sources = new ArrayList<>(); + ArrayNode sources = jsonMapper.createArrayNode(); try { - sources = new ArrayList<>(indexModel.getMasterClusters().keySet()); + indexModel.getMasterClusters().keySet().forEach(sources::add); } catch (NullPointerException ex){ /* clusters are not set */ } - json.put("model_sources", sources); + json.set("model_sources", sources); - List<String> rankProfiles = new ArrayList<>(); + ArrayNode rankProfiles = jsonMapper.createArrayNode(); try { rankProfilesConfig.rankprofile().forEach(rankProfile -> rankProfiles.add(rankProfile.name())); } catch (NullPointerException ex){ /* rankprofiles are not set*/ } - json.put("ranking_profile", rankProfiles); + json.set("ranking_profile", rankProfiles); // Creating map from parent to children for GUI: parameter --> child-parameters - HashMap<String, List<String>> childMap = new HashMap<>(); - childMap.put(Model.MODEL, Arrays.asList(Model.DEFAULT_INDEX, Model.ENCODING, Model.LANGUAGE, Model.QUERY_STRING, Model.RESTRICT, Model.SEARCH_PATH, Model.SOURCES, Model.TYPE)); - childMap.put(Ranking.RANKING, Arrays.asList(Ranking.LOCATION, Ranking.FEATURES, Ranking.LIST_FEATURES, Ranking.PROFILE, Ranking.PROPERTIES, Ranking.SORTING, Ranking.FRESHNESS, Ranking.QUERYCACHE, Ranking.MATCH_PHASE)); - childMap.put(Ranking.RANKING +"."+ Ranking.MATCH_PHASE, Arrays.asList(MatchPhase.MAX_HITS, MatchPhase.ATTRIBUTE, MatchPhase.ASCENDING, Ranking.DIVERSITY)); - childMap.put(Ranking.RANKING +"."+ Ranking.MATCH_PHASE +"."+Ranking.DIVERSITY, Arrays.asList(Diversity.ATTRIBUTE, Diversity.MINGROUPS)); - childMap.put(Presentation.PRESENTATION, Arrays.asList(Presentation.BOLDING, Presentation.FORMAT, Presentation.SUMMARY, "template", Presentation.TIMING )); - childMap.put("trace", Arrays.asList("timestamps")); - childMap.put("tracelevel", Arrays.asList("rules")); - childMap.put("metrics", Arrays.asList("ignore")); - childMap.put("collapse", Arrays.asList("summary")); - childMap.put("pos", Arrays.asList("ll", "radius", "bb", "attribute")); - childMap.put("streaming", Arrays.asList("userid", "groupname", "selection", "priority", "maxbucketspervisitor")); - childMap.put("rules", Arrays.asList("off", "rulebase")); - json.put("childMap", childMap); - - List<String> levelZeroParameters = Arrays.asList(MinimalQueryInserter.YQL.toString(), Query.HITS.toString(), Query.OFFSET.toString(), - "queryProfile", Query.NO_CACHE.toString(), Query.GROUPING_SESSION_CACHE.toString(), - Query.SEARCH_CHAIN.toString(), Query.TIMEOUT.toString(), "trace", "tracelevel", - Query.TRACE_LEVEL.toString(), Query.EXPLAIN_LEVEL.toString(), "explainlevel", Model.MODEL, Ranking.RANKING, "collapse", "collapsesize","collapsefield", - Presentation.PRESENTATION, "pos", "streaming", "rules", RecallSearcher.recallName.toString(), "user", - "metrics", ""); - json.put("levelZeroParameters", levelZeroParameters); - - return json.toString(); + ObjectNode childMap = jsonMapper.createObjectNode(); + childMap.set(Model.MODEL, jsonMapper.createArrayNode().add(Model.DEFAULT_INDEX).add(Model.ENCODING).add(Model.LANGUAGE).add(Model.QUERY_STRING).add(Model.RESTRICT).add(Model.SEARCH_PATH).add(Model.SOURCES).add(Model.TYPE)); + childMap.set(Ranking.RANKING, jsonMapper.createArrayNode().add(Ranking.LOCATION).add(Ranking.FEATURES).add(Ranking.LIST_FEATURES).add(Ranking.PROFILE).add(Ranking.PROPERTIES).add(Ranking.SORTING).add(Ranking.FRESHNESS).add(Ranking.QUERYCACHE).add(Ranking.MATCH_PHASE)); + childMap.set(Ranking.RANKING +"."+ Ranking.MATCH_PHASE, jsonMapper.createArrayNode().add(MatchPhase.MAX_HITS).add(MatchPhase.ATTRIBUTE).add(MatchPhase.ASCENDING).add(Ranking.DIVERSITY)); + childMap.set(Ranking.RANKING +"."+ Ranking.MATCH_PHASE +"."+Ranking.DIVERSITY, jsonMapper.createArrayNode().add(Diversity.ATTRIBUTE).add(Diversity.MINGROUPS)); + childMap.set(Presentation.PRESENTATION, jsonMapper.createArrayNode().add(Presentation.BOLDING).add(Presentation.FORMAT).add(Presentation.SUMMARY).add("template").add(Presentation.TIMING )); + childMap.set("trace", jsonMapper.createArrayNode().add("timestamps")); + childMap.set("tracelevel", jsonMapper.createArrayNode().add("rules")); + childMap.set("metrics", jsonMapper.createArrayNode().add("ignore")); + childMap.set("collapse", jsonMapper.createArrayNode().add("summary")); + childMap.set("pos", jsonMapper.createArrayNode().add("ll").add("radius").add("bb").add("attribute")); + childMap.set("streaming", jsonMapper.createArrayNode().add("userid").add("groupname").add("selection").add("priority").add("maxbucketspervisitor")); + childMap.set("rules", jsonMapper.createArrayNode().add("off").add("rulebase")); + json.set("childMap", childMap); + + ArrayNode levelZeroParameters = jsonMapper.createArrayNode().add(MinimalQueryInserter.YQL.toString()).add(Query.HITS.toString()).add(Query.OFFSET.toString()) + .add("queryProfile").add(Query.NO_CACHE.toString()).add(Query.GROUPING_SESSION_CACHE.toString()) + .add(Query.SEARCH_CHAIN.toString()).add(Query.TIMEOUT.toString()).add("trace").add("tracelevel") + .add(Query.TRACE_LEVEL.toString()).add(Query.EXPLAIN_LEVEL.toString()).add("explainlevel").add(Model.MODEL).add(Ranking.RANKING).add("collapse").add("collapsesize").add("collapsefield") + .add(Presentation.PRESENTATION).add("pos").add("streaming").add("rules").add(RecallSearcher.recallName.toString()).add("user") + .add("metrics").add(""); + json.set("levelZeroParameters", levelZeroParameters); + + return jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsString(json); } } }
\ No newline at end of file diff --git a/container-search/pom.xml b/container-search/pom.xml index 074f5827122..014b7dda14f 100644 --- a/container-search/pom.xml +++ b/container-search/pom.xml @@ -150,6 +150,11 @@ <version>${project.version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>org.assertj</groupId> + <artifactId>assertj-core</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> <plugins> diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java index 209bfd08e6b..55438aa35ba 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/JSONString.java @@ -21,6 +21,7 @@ import java.util.Iterator; * * @author Steinar Knutsen */ +// TODO Vespa 8: remove methods leaking org.json types (replace with Slime equivalent?) public class JSONString implements Inspectable { private Inspector value; @@ -436,6 +437,8 @@ public class JSONString implements Inspectable { return content; } + /** @deprecated Use {@link #getContent()} instead and parse content yourself */ + @Deprecated(forRemoval = true, since = "7") public Object getParsedJSON() { initContent(); if (parsedJSON == null) { @@ -444,6 +447,7 @@ public class JSONString implements Inspectable { return parsedJSON; } + @Deprecated(forRemoval = true, since = "7") public void setParsedJSON(Object parsedJSON) { this.parsedJSON = parsedJSON; } diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index 31f8194b3b7..c4f850307ae 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -45,8 +45,6 @@ import com.yahoo.search.result.Hit; import com.yahoo.search.result.HitGroup; import com.yahoo.search.result.NanNumber; import com.yahoo.tensor.Tensor; -import org.json.JSONArray; -import org.json.JSONObject; import java.io.IOException; import java.io.OutputStream; @@ -671,14 +669,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { } else if (field instanceof FieldValue) { // the null below is the field which has already been written ((FieldValue) field).serialize(null, new JsonWriter(generator)); - } else if (field instanceof JSONArray || field instanceof JSONObject) { - // org.json returns null if the object would not result in syntactically correct JSON - String s = field.toString(); - if (s == null) { - generator.writeNull(); - } else { - generator.writeRawValue(s); - } } else { generator.writeString(field.toString()); } diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java index d1399cabc75..49df321e581 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/SlimeSummaryTestCase.java @@ -4,16 +4,21 @@ package com.yahoo.prelude.fastsearch; import com.google.common.collect.ImmutableSet; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.data.access.slime.SlimeAdapter; +import com.yahoo.prelude.hitfield.JSONString; import com.yahoo.prelude.hitfield.RawData; import com.yahoo.prelude.hitfield.XMLString; -import com.yahoo.prelude.hitfield.JSONString; import com.yahoo.search.result.FeatureData; import com.yahoo.search.result.Hit; -import com.yahoo.search.result.NanNumber; import com.yahoo.search.result.StructuredData; +import com.yahoo.slime.BinaryFormat; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.serialization.TypedBinaryFormat; +import org.junit.Test; + import java.nio.ByteBuffer; import java.nio.ByteOrder; - import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Iterator; @@ -21,14 +26,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import com.yahoo.slime.BinaryFormat; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.Slime; -import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.serialization.TypedBinaryFormat; -import org.junit.Test; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -102,7 +99,7 @@ public class SlimeSummaryTestCase { if (hit.getField("jsonstring_field") instanceof JSONString) { JSONString jstr = (JSONString) hit.getField("jsonstring_field"); assertEquals("{\"foo\":1,\"bar\":2}", jstr.getContent()); - assertNotNull(jstr.getParsedJSON()); + assertNotNull(getParsedJSON(jstr)); com.yahoo.data.access.Inspector value = jstr.inspect(); assertEquals(1L, value.field("foo").asLong()); @@ -126,6 +123,8 @@ public class SlimeSummaryTestCase { assertEquals(tensor2, featureData.getTensor("tensor2_feature")); } + @SuppressWarnings("removal") private static Object getParsedJSON(JSONString jstr) { return jstr.getParsedJSON(); } + @Test public void testFieldAccessAPI() { DocsumDefinitionSet partialDocsum1 = createDocsumDefinitionSet(partial_summary1_cf); diff --git a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java index 3cca053d0e5..80e629ca4cb 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/handler/test/JSONSearchHandlerTestCase.java @@ -1,10 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.handler.test; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.yahoo.container.Container; import com.yahoo.container.core.config.testutil.HandlersConfigurerTestWrapper; import com.yahoo.container.jdisc.HttpRequest; - import com.yahoo.container.jdisc.RequestHandlerTestDriver; import com.yahoo.container.protect.Error; import com.yahoo.io.IOUtils; @@ -13,8 +16,8 @@ import com.yahoo.search.handler.SearchHandler; import com.yahoo.search.searchchain.config.test.SearchChainConfigurerTestCase; import com.yahoo.slime.Inspector; import com.yahoo.slime.SlimeUtils; -import org.json.JSONArray; -import org.json.JSONObject; +import com.yahoo.test.json.JsonTestHelper; +import org.assertj.core.api.Assertions; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -23,13 +26,19 @@ import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; -import java.util.Map; +import java.nio.charset.StandardCharsets; import java.util.HashMap; +import java.util.Map; import static com.yahoo.jdisc.http.HttpRequest.Method.GET; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * Tests submitting the query as JSON. @@ -38,6 +47,8 @@ import static org.junit.Assert.*; */ public class JSONSearchHandlerTestCase { + private static final ObjectMapper jsonMapper = new ObjectMapper(); + private static final String testDir = "src/test/java/com/yahoo/search/handler/test/config"; private static final String myHostnameHeader = "my-hostname-header"; private static final String selfHostname = HostName.getLocalhost(); @@ -97,8 +108,8 @@ public class JSONSearchHandlerTestCase { } @Test - public void testFailing() throws Exception { - JSONObject json = new JSONObject(); + public void testFailing() { + ObjectNode json = jsonMapper.createObjectNode(); json.put("query", "test"); json.put("searchChain", "classLoadingError"); assertTrue(driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE).readAll().contains("NoClassDefFoundError")); @@ -106,16 +117,16 @@ public class JSONSearchHandlerTestCase { @Test - public synchronized void testPluginError() throws Exception { - JSONObject json = new JSONObject(); + public synchronized void testPluginError() { + ObjectNode json = jsonMapper.createObjectNode(); json.put("query", "test"); json.put("searchChain", "exceptionInPlugin"); assertTrue(driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE).readAll().contains("NullPointerException")); } @Test - public synchronized void testWorkingReconfiguration() throws Exception { - JSONObject json = new JSONObject(); + public synchronized void testWorkingReconfiguration() throws IOException { + ObjectNode json = jsonMapper.createObjectNode(); json.put("query", "abc"); assertJsonResult(json, driver); @@ -135,7 +146,7 @@ public class JSONSearchHandlerTestCase { } @Test - public void testInvalidYqlQuery() throws Exception { + public void testInvalidYqlQuery() throws IOException { IOUtils.copyDirectory(new File(testDir, "config_yql"), new File(tempDir), 1); generateComponentsConfigForActive(); configurer.reloadConfig(); @@ -143,7 +154,7 @@ public class JSONSearchHandlerTestCase { SearchHandler newSearchHandler = fetchSearchHandler(configurer); assertTrue("Do I have a new instance of the search handler?", searchHandler != newSearchHandler); try (RequestHandlerTestDriver newDriver = new RequestHandlerTestDriver(newSearchHandler)) { - JSONObject json = new JSONObject(); + ObjectNode json = jsonMapper.createObjectNode(); json.put("yql", "select * from foo where bar > 1453501295"); RequestHandlerTestDriver.MockResponseHandler responseHandler = newDriver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE); responseHandler.readAll(); @@ -153,14 +164,14 @@ public class JSONSearchHandlerTestCase { // Query handling takes a different code path when a query profile is active, so we test both paths. @Test - public void testInvalidQueryParamWithQueryProfile() throws Exception { + public void testInvalidQueryParamWithQueryProfile() throws IOException { try (RequestHandlerTestDriver newDriver = driverWithConfig("config_invalid_param")) { testInvalidQueryParam(newDriver); } } - private void testInvalidQueryParam(final RequestHandlerTestDriver testDriver) throws Exception { - JSONObject json = new JSONObject(); + private void testInvalidQueryParam(final RequestHandlerTestDriver testDriver) { + ObjectNode json = jsonMapper.createObjectNode(); json.put("query", "status_code:0"); json.put("hits", 20); json.put("offset", -20); @@ -173,16 +184,16 @@ public class JSONSearchHandlerTestCase { } @Test - public void testNormalResultJsonAliasRendering() throws Exception { - JSONObject json = new JSONObject(); + public void testNormalResultJsonAliasRendering() { + ObjectNode json = jsonMapper.createObjectNode(); json.put("format", "json"); json.put("query", "abc"); assertJsonResult(json, driver); } @Test - public void testNullQuery() throws Exception { - JSONObject json = new JSONObject(); + public void testNullQuery() { + ObjectNode json = jsonMapper.createObjectNode(); json.put("format", "xml"); assertEquals("<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n" + @@ -195,8 +206,8 @@ public class JSONSearchHandlerTestCase { } @Test - public void testWebServiceStatus() throws Exception { - JSONObject json = new JSONObject(); + public void testWebServiceStatus() { + ObjectNode json = jsonMapper.createObjectNode(); json.put("query", "web_service_status_code"); RequestHandlerTestDriver.MockResponseHandler responseHandler = driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE); @@ -206,39 +217,39 @@ public class JSONSearchHandlerTestCase { } @Test - public void testNormalResultImplicitDefaultRendering() throws Exception { - JSONObject json = new JSONObject(); + public void testNormalResultImplicitDefaultRendering() { + ObjectNode json = jsonMapper.createObjectNode(); json.put("query", "abc"); assertJsonResult(json, driver); } @Test - public void testNormalResultExplicitDefaultRendering() throws Exception { - JSONObject json = new JSONObject(); + public void testNormalResultExplicitDefaultRendering() { + ObjectNode json = jsonMapper.createObjectNode(); json.put("query", "abc"); json.put("format", "default"); assertJsonResult(json, driver); } @Test - public void testNormalResultXmlAliasRendering() throws Exception { - JSONObject json = new JSONObject(); + public void testNormalResultXmlAliasRendering() { + ObjectNode json = jsonMapper.createObjectNode(); json.put("query", "abc"); json.put("format", "xml"); assertXmlResult(json, driver); } @Test - public void testNormalResultExplicitDefaultRenderingFullRendererName1() throws Exception { - JSONObject json = new JSONObject(); + public void testNormalResultExplicitDefaultRenderingFullRendererName1() { + ObjectNode json = jsonMapper.createObjectNode(); json.put("query", "abc"); json.put("format", "XmlRenderer"); assertXmlResult(json, driver); } @Test - public void testNormalResultExplicitDefaultRenderingFullRendererName2() throws Exception { - JSONObject json = new JSONObject(); + public void testNormalResultExplicitDefaultRenderingFullRendererName2() { + ObjectNode json = jsonMapper.createObjectNode(); json.put("query", "abc"); json.put("format", "JsonRenderer"); assertJsonResult(json, driver); @@ -253,7 +264,7 @@ public class JSONSearchHandlerTestCase { " </hit>\n" + "</result>\n"; - private void assertXmlResult(JSONObject json, RequestHandlerTestDriver driver) { + private void assertXmlResult(JsonNode json, RequestHandlerTestDriver driver) { assertOkResult(driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE), xmlResult); } @@ -263,7 +274,7 @@ public class JSONSearchHandlerTestCase { + "{\"id\":\"testHit\",\"relevance\":1.0,\"fields\":{\"uri\":\"testHit\"}}" + "]}}"; - private void assertJsonResult(JSONObject json, RequestHandlerTestDriver driver) { + private void assertJsonResult(JsonNode json, RequestHandlerTestDriver driver) { assertOkResult(driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), JSON_CONTENT_TYPE), jsonResult); } @@ -288,7 +299,7 @@ public class JSONSearchHandlerTestCase { } - private RequestHandlerTestDriver driverWithConfig(String configDirectory) throws Exception { + private RequestHandlerTestDriver driverWithConfig(String configDirectory) throws IOException { IOUtils.copyDirectory(new File(testDir, configDirectory), new File(tempDir), 1); generateComponentsConfigForActive(); configurer.reloadConfig(); @@ -299,44 +310,44 @@ public class JSONSearchHandlerTestCase { } @Test - public void testSelectParameters() throws Exception { - JSONObject json = new JSONObject(); + public void testSelectParameters() throws IOException { + ObjectNode json = jsonMapper.createObjectNode(); - JSONObject select = new JSONObject(); + ObjectNode select = jsonMapper.createObjectNode(); - JSONObject where = new JSONObject(); + ObjectNode where = jsonMapper.createObjectNode(); where.put("where", "where"); - JSONObject grouping = new JSONObject(); + ObjectNode grouping = jsonMapper.createObjectNode(); grouping.put("grouping", "grouping"); - select.put("where", where); - select.put("grouping", grouping); + select.set("where", where); + select.set("grouping", grouping); - json.put("select", select); + json.set("select", select); - Inspector inspector = SlimeUtils.jsonToSlime(json.toString().getBytes("utf-8")).get(); + Inspector inspector = SlimeUtils.jsonToSlime(json.toString().getBytes(StandardCharsets.UTF_8)).get(); Map<String, String> map = new HashMap<>(); searchHandler.createRequestMapping(inspector, map, ""); - JSONObject processedWhere = new JSONObject(map.get("select.where")); - assertEquals(where.toString(), processedWhere.toString()); + JsonNode processedWhere = jsonMapper.readTree(map.get("select.where")); + JsonTestHelper.assertJsonEquals(where.toString(), processedWhere.toString()); - JSONObject processedGrouping = new JSONObject(map.get("select.grouping")); - assertEquals(grouping.toString(), processedGrouping.toString()); + JsonNode processedGrouping = jsonMapper.readTree(map.get("select.grouping")); + JsonTestHelper.assertJsonEquals(grouping.toString(), processedGrouping.toString()); } @Test - public void testJsonQueryWithSelectWhere() throws Exception { - JSONObject root = new JSONObject(); - JSONObject select = new JSONObject(); - JSONObject where = new JSONObject(); - JSONArray term = new JSONArray(); - term.put("default"); - term.put("bad"); - where.put("contains", term); - select.put("where", where); - root.put("select", select); + public void testJsonQueryWithSelectWhere() { + ObjectNode root = jsonMapper.createObjectNode(); + ObjectNode select = jsonMapper.createObjectNode(); + ObjectNode where = jsonMapper.createObjectNode(); + ArrayNode term = jsonMapper.createArrayNode(); + term.add("default"); + term.add("bad"); + where.set("contains", term); + select.set("where", where); + root.set("select", select); // Run query String result = driver.sendRequest(uri + "searchChain=echoingQuery", com.yahoo.jdisc.http.HttpRequest.Method.POST, root.toString(), JSON_CONTENT_TYPE).readAll(); @@ -393,8 +404,8 @@ public class JSONSearchHandlerTestCase { } @Test - public void testJsonQueryWithYQL() throws Exception { - JSONObject root = new JSONObject(); + public void testJsonQueryWithYQL() { + ObjectNode root = jsonMapper.createObjectNode(); root.put("yql", "select * from sources * where default contains 'bad';"); // Run query @@ -404,10 +415,10 @@ public class JSONSearchHandlerTestCase { } @Test - public void testRequestMapping() throws Exception { - JSONObject json = new JSONObject(); + public void testRequestMapping() { + ObjectNode json = jsonMapper.createObjectNode(); json.put("yql", "select * from sources * where sddocname contains \"blog_post\" limit 0 | all(group(date) max(3) order(-count())each(output(count())));"); - json.put("hits", 10.0); + json.put("hits", 10); json.put("offset", 5); json.put("queryProfile", "foo"); json.put("nocache", false); @@ -417,7 +428,7 @@ public class JSONSearchHandlerTestCase { json.put("select", "_all"); - JSONObject model = new JSONObject(); + ObjectNode model = jsonMapper.createObjectNode(); model.put("defaultIndex", 1); model.put("encoding", "json"); model.put("filter", "default"); @@ -427,9 +438,9 @@ public class JSONSearchHandlerTestCase { model.put("searchPath", "node1"); model.put("sources", "source1,source2"); model.put("type", "yql"); - json.put("model", model); + json.set("model", model); - JSONObject ranking = new JSONObject(); + ObjectNode ranking = jsonMapper.createObjectNode(); ranking.put("location", "123789.89123N;128123W"); ranking.put("features", "none"); ranking.put("listFeatures", false); @@ -439,61 +450,61 @@ public class JSONSearchHandlerTestCase { ranking.put("freshness", "0.05"); ranking.put("queryCache", false); - JSONObject matchPhase = new JSONObject(); + ObjectNode matchPhase = jsonMapper.createObjectNode(); matchPhase.put("maxHits", "100"); matchPhase.put("attribute", "title"); matchPhase.put("ascending", true); - JSONObject diversity = new JSONObject(); + ObjectNode diversity = jsonMapper.createObjectNode(); diversity.put("attribute", "title"); diversity.put("minGroups", 1); - matchPhase.put("diversity", diversity); - ranking.put("matchPhase", matchPhase); - json.put("ranking", ranking); + matchPhase.set("diversity", diversity); + ranking.set("matchPhase", matchPhase); + json.set("ranking", ranking); - JSONObject presentation = new JSONObject(); + ObjectNode presentation = jsonMapper.createObjectNode(); presentation.put("bolding", true); presentation.put("format", "json"); presentation.put("summary", "none"); presentation.put("template", "json"); presentation.put("timing", false); - json.put("presentation", presentation); + json.set("presentation", presentation); - JSONObject collapse = new JSONObject(); + ObjectNode collapse = jsonMapper.createObjectNode(); collapse.put("field", "none"); collapse.put("size", 2); collapse.put("summary", "default"); - json.put("collapse", collapse); + json.set("collapse", collapse); - JSONObject trace = new JSONObject(); + ObjectNode trace = jsonMapper.createObjectNode(); trace.put("level", 1); trace.put("timestamps", false); trace.put("rules", "none"); - json.put("trace", trace); + json.set("trace", trace); - JSONObject pos = new JSONObject(); + ObjectNode pos = jsonMapper.createObjectNode(); pos.put("ll", "1263123N;1231.9W"); pos.put("radius", "71234m"); pos.put("bb", "1237123W;123218N"); pos.put("attribute", "default"); - json.put("pos", pos); + json.set("pos", pos); - JSONObject streaming = new JSONObject(); + ObjectNode streaming = jsonMapper.createObjectNode(); streaming.put("userid", 123); streaming.put("groupname", "abc"); streaming.put("selection", "none"); streaming.put("priority", 10); streaming.put("maxbucketspervisitor", 5); - json.put("streaming", streaming); + json.set("streaming", streaming); - JSONObject rules = new JSONObject(); + ObjectNode rules = jsonMapper.createObjectNode(); rules.put("off", false); rules.put("rulebase", "default"); - json.put("rules", rules); + json.set("rules", rules); - JSONObject metrics = new JSONObject(); + ObjectNode metrics = jsonMapper.createObjectNode(); metrics.put("ignore", "_all"); - json.put("metrics", metrics); + json.set("metrics", metrics); json.put("recall", "none"); json.put("user", 123); @@ -501,7 +512,7 @@ public class JSONSearchHandlerTestCase { json.put("hitcountestimate", true); // Create mapping - Inspector inspector = SlimeUtils.jsonToSlime(json.toString().getBytes("utf-8")).get(); + Inspector inspector = SlimeUtils.jsonToSlime(json.toString().getBytes(StandardCharsets.UTF_8)).get(); Map<String, String> map = new HashMap<>(); searchHandler.createRequestMapping(inspector, map, ""); @@ -518,12 +529,12 @@ public class JSONSearchHandlerTestCase { // Get mapping Map<String, String> propertyMap = request.propertyMap(); - assertEquals("Should have same mapping for properties", map, propertyMap); + Assertions.assertThat(propertyMap).isEqualTo(map); } @Test - public void testContentTypeParsing() throws Exception { - JSONObject json = new JSONObject(); + public void testContentTypeParsing() { + ObjectNode json = jsonMapper.createObjectNode(); json.put("query", "abc"); assertOkResult(driver.sendRequest(uri, com.yahoo.jdisc.http.HttpRequest.Method.POST, json.toString(), "Application/JSON; charset=utf-8"), jsonResult); } diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java index c4d49c11f5e..48003f6586f 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.rendering; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.ListenableFuture; @@ -56,9 +55,6 @@ import com.yahoo.tensor.TensorType; import com.yahoo.tensor.serialization.TypedBinaryFormat; import com.yahoo.text.Utf8; import com.yahoo.yolean.trace.TraceNode; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; import org.junit.Before; import org.junit.Test; @@ -67,7 +63,6 @@ import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.nio.charset.StandardCharsets; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; @@ -83,6 +78,8 @@ import static org.junit.Assert.assertTrue; */ public class JsonRendererTestCase { + private static final ObjectMapper jsonMapper = new ObjectMapper(); + private JsonRenderer originalRenderer; private JsonRenderer renderer; @@ -959,7 +956,7 @@ public class JsonRendererTestCase { } @Test - public void testJsonObjects() throws InterruptedException, ExecutionException, IOException, JSONException { + public void testJsonObjects() throws InterruptedException, ExecutionException, IOException { String expected = "{" + " \"root\": {" + " \"children\": [" @@ -973,14 +970,6 @@ public class JsonRendererTestCase { + " }," + " \"json producer\": {" + " \"long in structured\": 7809531904" - + " }," - + " \"org.json array\": [" - + " true," - + " true," - + " false" - + " ]," - + " \"org.json object\": {" - + " \"forty-two\": 42" + " }" + " }," + " \"id\": \"json objects\"," @@ -996,26 +985,17 @@ public class JsonRendererTestCase { + "}"; Result r = newEmptyResult(); Hit h = new Hit("json objects"); - JSONObject o = new JSONObject(); - JSONArray a = new JSONArray(); - ObjectMapper mapper = new ObjectMapper(); - JsonNode j = mapper.createObjectNode(); + ObjectNode j = jsonMapper.createObjectNode(); JSONString s = new JSONString("{\"a\": \"b\"}"); Slime slime = new Slime(); Cursor c = slime.setObject(); c.setLong("long in structured", 7809531904L); SlimeAdapter slimeInit = new SlimeAdapter(slime.get()); StructuredData struct = new StructuredData(slimeInit); - ((ObjectNode) j).put("Nineteen-eighty-four", 1984); - o.put("forty-two", 42); - a.put(true); - a.put(true); - a.put(false); + j.put("Nineteen-eighty-four", 1984); h.setField("inspectable", s); h.setField("jackson", j); h.setField("json producer", struct); - h.setField("org.json array", a); - h.setField("org.json object", o); r.hits().add(h); String summary = render(r); assertEqualJson(expected, summary); @@ -1236,11 +1216,13 @@ public class JsonRendererTestCase { public void testThatTheJsonValidatorCanCatchErrors() { String json = "{" + " \"root\": {" - + " \"duplicate\": 1," - + " \"duplicate\": 2" + + " \"invalidvalue\": 1adsf," + " }" + "}"; - assertEquals("Duplicate key \"duplicate\"", validateJSON(json)); + assertEquals( + "Unexpected character ('a' (code 97)): was expecting comma to separate Object entries\n" + + " at [Source: { \"root\": { \"invalidvalue\": 1adsf, }}; line: 1, column: 41]", + validateJSON(json)); } @Test @@ -1316,9 +1298,9 @@ public class JsonRendererTestCase { private String validateJSON(String presumablyValidJson) { try { - new JSONObject(presumablyValidJson); + jsonMapper.readTree(presumablyValidJson); return ""; - } catch (JSONException e) { + } catch (IOException e) { return e.getMessage(); } } diff --git a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java index 3239a97a094..9bcd3addd92 100644 --- a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java +++ b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java @@ -1,6 +1,9 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.select; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.ExactStringItem; import com.yahoo.prelude.query.Item; @@ -17,19 +20,15 @@ import com.yahoo.prelude.query.WordItem; import com.yahoo.processing.IllegalInputException; import com.yahoo.search.Query; import com.yahoo.search.grouping.GroupingRequest; -import com.yahoo.search.grouping.request.AllOperation; import com.yahoo.search.query.QueryTree; import com.yahoo.search.query.Select; import com.yahoo.search.query.SelectParser; import com.yahoo.search.query.parser.Parsable; import com.yahoo.search.query.parser.ParserEnvironment; import com.yahoo.search.yql.VespaGroupingStep; -import org.json.JSONException; -import org.json.JSONObject; import org.junit.Test; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; @@ -48,15 +47,18 @@ import static org.junit.Assert.fail; */ public class SelectTestCase { + private static final ObjectMapper jsonMapper = new ObjectMapper(); + private final SelectParser parser = new SelectParser(new ParserEnvironment()); //------------------------------------------------------------------- "where" tests @Test - public void test_contains() throws Exception { - JSONObject json = new JSONObject(); - List<String> contains = Arrays.asList("default", "foo"); - json.put("contains", contains); + public void test_contains() { + ObjectNode json = jsonMapper.createObjectNode(); + ArrayNode arrayNode = jsonMapper.createArrayNode(); + arrayNode.add("default").add("foo"); + json.set("contains", arrayNode); assertParse(json.toString(), "default:foo"); } @@ -77,21 +79,21 @@ public class SelectTestCase { @Test public void testOr() throws Exception { - JSONObject json_two_or = new JSONObject(); - JSONObject json_three_or = new JSONObject(); - List<String> contains1 = Arrays.asList("title", "madonna"); - List<String> contains2 = Arrays.asList("title", "saint"); - List<String> contains3 = Arrays.asList("title", "angel"); - - JSONObject contains_json1 = new JSONObject(); - JSONObject contains_json2 = new JSONObject(); - JSONObject contains_json3 = new JSONObject(); - contains_json1.put("contains", contains1); - contains_json2.put("contains", contains2); - contains_json3.put("contains", contains3); - - json_two_or.put("or", Arrays.asList(contains_json1, contains_json2)); - json_three_or.put("or", Arrays.asList(contains_json1, contains_json2, contains_json3)); + ObjectNode json_two_or = jsonMapper.createObjectNode(); + ObjectNode json_three_or = jsonMapper.createObjectNode(); + ArrayNode contains1 = jsonMapper.createArrayNode().add("title").add("madonna"); + ArrayNode contains2 = jsonMapper.createArrayNode().add("title").add("saint"); + ArrayNode contains3 = jsonMapper.createArrayNode().add("title").add("angel"); + + ObjectNode contains_json1 = jsonMapper.createObjectNode(); + ObjectNode contains_json2 = jsonMapper.createObjectNode(); + ObjectNode contains_json3 = jsonMapper.createObjectNode(); + contains_json1.set("contains", contains1); + contains_json2.set("contains", contains2); + contains_json3.set("contains", contains3); + + json_two_or.set("or", jsonMapper.createArrayNode().add(contains_json1).add(contains_json2)); + json_three_or.set("or", jsonMapper.createArrayNode().add(contains_json1).add(contains_json2).add(contains_json3)); assertParse(json_two_or.toString(), "OR title:madonna title:saint"); assertParse(json_three_or.toString(), "OR title:madonna title:saint title:angel"); @@ -99,178 +101,178 @@ public class SelectTestCase { @Test public void testAnd() throws Exception{ - JSONObject json_two_and = new JSONObject(); - JSONObject json_three_and = new JSONObject(); - List<String> contains1 = Arrays.asList("title", "madonna"); - List<String> contains2 = Arrays.asList("title", "saint"); - List<String> contains3 = Arrays.asList("title", "angel"); - - JSONObject contains_json1 = new JSONObject(); - JSONObject contains_json2 = new JSONObject(); - JSONObject contains_json3 = new JSONObject(); - contains_json1.put("contains", contains1); - contains_json2.put("contains", contains2); - contains_json3.put("contains", contains3); - - json_two_and.put("and", Arrays.asList(contains_json1, contains_json2)); - json_three_and.put("and", Arrays.asList(contains_json1, contains_json2, contains_json3)); + ObjectNode json_two_and = jsonMapper.createObjectNode(); + ObjectNode json_three_and = jsonMapper.createObjectNode(); + ArrayNode contains1 = jsonMapper.createArrayNode().add("title").add("madonna"); + ArrayNode contains2 = jsonMapper.createArrayNode().add("title").add("saint"); + ArrayNode contains3 = jsonMapper.createArrayNode().add("title").add("angel"); + + ObjectNode contains_json1 = jsonMapper.createObjectNode(); + ObjectNode contains_json2 = jsonMapper.createObjectNode(); + ObjectNode contains_json3 = jsonMapper.createObjectNode(); + contains_json1.set("contains", contains1); + contains_json2.set("contains", contains2); + contains_json3.set("contains", contains3); + + json_two_and.set("and", jsonMapper.createArrayNode().add(contains_json1).add(contains_json2)); + json_three_and.set("and", jsonMapper.createArrayNode().add(contains_json1).add(contains_json2).add(contains_json3)); assertParse(json_two_and.toString(), "AND title:madonna title:saint"); assertParse(json_three_and.toString(), "AND title:madonna title:saint title:angel"); } @Test - public void testAndNot() throws JSONException { - JSONObject json_and_not = new JSONObject(); - List<String> contains1 = Arrays.asList("title", "madonna"); - List<String> contains2 = Arrays.asList("title", "saint"); + public void testAndNot() { + ObjectNode json_and_not = jsonMapper.createObjectNode(); + ArrayNode contains1 = jsonMapper.createArrayNode().add("title").add("madonna"); + ArrayNode contains2 = jsonMapper.createArrayNode().add("title").add("saint"); - JSONObject contains_json1 = new JSONObject(); - JSONObject contains_json2 = new JSONObject(); - contains_json1.put("contains", contains1); - contains_json2.put("contains", contains2); + ObjectNode contains_json1 = jsonMapper.createObjectNode(); + ObjectNode contains_json2 = jsonMapper.createObjectNode(); + contains_json1.set("contains", contains1); + contains_json2.set("contains", contains2); - json_and_not.put("and_not", Arrays.asList(contains_json1, contains_json2)); + json_and_not.set("and_not", jsonMapper.createArrayNode().add(contains_json1).add(contains_json2)); assertParse(json_and_not.toString(), "+title:madonna -title:saint"); } @Test - public void testLessThan() throws JSONException { - JSONObject range_json = new JSONObject(); - JSONObject operators = new JSONObject(); + public void testLessThan() { + ObjectNode range_json = jsonMapper.createObjectNode(); + ObjectNode operators = jsonMapper.createObjectNode(); operators.put("<", 500); - List<Object> range = Arrays.asList("price", operators); + ArrayNode range = jsonMapper.createArrayNode().add("price").add(operators); - range_json.put("range", range); + range_json.set("range", range); assertParse(range_json.toString(), "price:<500"); } @Test - public void testGreaterThan() throws JSONException { - JSONObject range_json = new JSONObject(); - JSONObject operators = new JSONObject(); + public void testGreaterThan() { + ObjectNode range_json = jsonMapper.createObjectNode(); + ObjectNode operators = jsonMapper.createObjectNode(); operators.put(">", 500); - List<Object> range = Arrays.asList("price", operators); + ArrayNode range = jsonMapper.createArrayNode().add("price").add(operators); - range_json.put("range", range); + range_json.set("range", range); assertParse(range_json.toString(), "price:>500"); } @Test - public void testLessThanOrEqual() throws JSONException { - JSONObject range_json = new JSONObject(); - JSONObject operators = new JSONObject(); + public void testLessThanOrEqual() { + ObjectNode range_json = jsonMapper.createObjectNode(); + ObjectNode operators = jsonMapper.createObjectNode(); operators.put("<=", 500); - List<Object> range = Arrays.asList("price", operators); + ArrayNode range = jsonMapper.createArrayNode().add("price").add(operators); - range_json.put("range", range); + range_json.set("range", range); assertParse(range_json.toString(), "price:[;500]"); } @Test - public void testGreaterThanOrEqual() throws JSONException { - JSONObject range_json = new JSONObject(); - JSONObject operators = new JSONObject(); + public void testGreaterThanOrEqual() { + ObjectNode range_json = jsonMapper.createObjectNode(); + ObjectNode operators = jsonMapper.createObjectNode(); operators.put(">=", 500); - List<Object> range = Arrays.asList("price", operators); + ArrayNode range = jsonMapper.createArrayNode().add("price").add(operators); - range_json.put("range", range); + range_json.set("range", range); assertParse(range_json.toString(), "price:[500;]"); } @Test - public void testEquality() throws JSONException { - JSONObject range_json = new JSONObject(); - JSONObject operators = new JSONObject(); + public void testEquality() { + ObjectNode range_json = jsonMapper.createObjectNode(); + ObjectNode operators = jsonMapper.createObjectNode(); operators.put("=", 500); - List<Object> range = Arrays.asList("price", operators); + ArrayNode range = jsonMapper.createArrayNode().add("price").add(operators); - range_json.put("range", range); + range_json.set("range", range); assertParse(range_json.toString(), "price:500"); } @Test - public void testNegativeLessThan() throws JSONException { - JSONObject range_json = new JSONObject(); - JSONObject operators = new JSONObject(); + public void testNegativeLessThan() { + ObjectNode range_json = jsonMapper.createObjectNode(); + ObjectNode operators = jsonMapper.createObjectNode(); operators.put("<", -500); - List<Object> range = Arrays.asList("price", operators); + ArrayNode range = jsonMapper.createArrayNode().add("price").add(operators); - range_json.put("range", range); + range_json.set("range", range); assertParse(range_json.toString(), "price:<-500"); } @Test - public void testNegativeGreaterThan() throws JSONException { - JSONObject range_json = new JSONObject(); - JSONObject operators = new JSONObject(); + public void testNegativeGreaterThan() { + ObjectNode range_json = jsonMapper.createObjectNode(); + ObjectNode operators = jsonMapper.createObjectNode(); operators.put(">", -500); - List<Object> range = Arrays.asList("price", operators); + ArrayNode range = jsonMapper.createArrayNode().add("price").add(operators); - range_json.put("range", range); + range_json.set("range", range); assertParse(range_json.toString(), "price:>-500"); } @Test - public void testNegativeLessThanOrEqual() throws JSONException { - JSONObject range_json = new JSONObject(); - JSONObject operators = new JSONObject(); + public void testNegativeLessThanOrEqual() { + ObjectNode range_json = jsonMapper.createObjectNode(); + ObjectNode operators = jsonMapper.createObjectNode(); operators.put("<=", -500); - List<Object> range = Arrays.asList("price", operators); + ArrayNode range = jsonMapper.createArrayNode().add("price").add(operators); - range_json.put("range", range); + range_json.set("range", range); assertParse(range_json.toString(), "price:[;-500]"); } @Test - public void testNegativeGreaterThanOrEqual() throws JSONException { - JSONObject range_json = new JSONObject(); - JSONObject operators = new JSONObject(); + public void testNegativeGreaterThanOrEqual() { + ObjectNode range_json = jsonMapper.createObjectNode(); + ObjectNode operators = jsonMapper.createObjectNode(); operators.put(">=", -500); - List<Object> range = Arrays.asList("price", operators); + ArrayNode range = jsonMapper.createArrayNode().add("price").add(operators); - range_json.put("range", range); + range_json.set("range", range); assertParse(range_json.toString(), "price:[-500;]"); } @Test - public void testNegativeEquality() throws JSONException { - JSONObject range_json = new JSONObject(); - JSONObject operators = new JSONObject(); + public void testNegativeEquality() { + ObjectNode range_json = jsonMapper.createObjectNode(); + ObjectNode operators = jsonMapper.createObjectNode(); operators.put("=", -500); - List<Object> range = Arrays.asList("price", operators); + ArrayNode range = jsonMapper.createArrayNode().add("price").add(operators); - range_json.put("range", range); + range_json.set("range", range); assertParse(range_json.toString(), "price:-500"); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ProtonMetrics.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ProtonMetrics.java index c6d907ec7fc..ed88902e094 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ProtonMetrics.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/ProtonMetrics.java @@ -1,14 +1,19 @@ package com.yahoo.vespa.hosted.controller.api.application.v4.model; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + import java.util.HashMap; import java.util.Map; +import java.util.logging.Level; import java.util.logging.LogManager; import java.util.logging.Logger; -import org.json.JSONException; -import org.json.JSONObject; public class ProtonMetrics { + private static final ObjectMapper jsonMapper = new ObjectMapper(); + private static final Logger logger = LogManager.getLogManager().getLogger(ProtonMetrics.class.getName()); public static final String DOCUMENTS_ACTIVE_COUNT = "documentsActiveCount"; @@ -45,19 +50,19 @@ public class ProtonMetrics { return this; } - public JSONObject toJson() { + public JsonNode toJson() { try { - JSONObject protonMetrics = new JSONObject(); + ObjectNode protonMetrics = jsonMapper.createObjectNode(); protonMetrics.put("clusterId", clusterId); - JSONObject jsonMetrics = new JSONObject(); + ObjectNode jsonMetrics = jsonMapper.createObjectNode(); for (Map.Entry<String, Double> entry : metrics.entrySet()) { jsonMetrics.put(entry.getKey(), entry.getValue()); } - protonMetrics.put("metrics", jsonMetrics); + protonMetrics.set("metrics", jsonMetrics); return protonMetrics; - } catch (JSONException e) { - logger.severe("Unable to convert Proton Metrics to JSON Object"); + } catch (Exception e) { + logger.log(Level.SEVERE, "Unable to convert Proton Metrics to JSON Object: " + e.getMessage(), e); } - return new JSONObject(); + return jsonMapper.createObjectNode(); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java index 76fa52c0706..87531240752 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java @@ -1,6 +1,5 @@ package com.yahoo.vespa.hosted.controller.certificate; -import com.google.common.collect.Sets; import com.google.common.hash.Hashing; import com.google.common.io.BaseEncoding; import com.yahoo.config.application.api.DeploymentInstanceSpec; @@ -14,19 +13,15 @@ import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.security.SubjectAlternativeName; import com.yahoo.security.X509CertificateUtils; -import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import org.jetbrains.annotations.NotNull; @@ -35,17 +30,12 @@ import java.security.cert.X509Certificate; import java.time.Clock; import java.time.Duration; import java.time.Instant; -import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; -import java.util.OptionalInt; import java.util.Set; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -54,6 +44,8 @@ import java.util.stream.Collectors; * Looks up stored endpoint certificate metadata, provisions new certificates if none is found, * re-provisions if zone is not covered, and uses refreshed certificates if a newer version is available. * + * See also EndpointCertificateMaintainer, which handles refreshes, deletions and triggers deployments + * * @author andreer */ public class EndpointCertificateManager { @@ -66,9 +58,6 @@ public class EndpointCertificateManager { private final EndpointCertificateProvider endpointCertificateProvider; private final Clock clock; private final BooleanFlag validateEndpointCertificates; - private final StringFlag deleteUnusedEndpointCertificates; - private final BooleanFlag endpointCertInSharedRouting; - private final BooleanFlag useEndpointCertificateMaintainer; public EndpointCertificateManager(ZoneRegistry zoneRegistry, CuratorDb curator, @@ -81,16 +70,6 @@ public class EndpointCertificateManager { this.endpointCertificateProvider = endpointCertificateProvider; this.clock = clock; this.validateEndpointCertificates = Flags.VALIDATE_ENDPOINT_CERTIFICATES.bindTo(flagSource); - this.deleteUnusedEndpointCertificates = Flags.DELETE_UNUSED_ENDPOINT_CERTIFICATES.bindTo(flagSource); - this.endpointCertInSharedRouting = Flags.ENDPOINT_CERT_IN_SHARED_ROUTING.bindTo(flagSource); - this.useEndpointCertificateMaintainer = Flags.USE_ENDPOINT_CERTIFICATE_MAINTAINER.bindTo(flagSource); - Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> { - try { - this.deleteUnusedCertificates(); - } catch (Throwable t) { - log.log(Level.INFO, "Unexpected Throwable caught while deleting unused endpoint certificates", t); - } - }, 1, 10, TimeUnit.MINUTES); } public Optional<EndpointCertificateMetadata> getEndpointCertificateMetadata(Instance instance, ZoneId zone, Optional<DeploymentInstanceSpec> instanceSpec) { @@ -105,10 +84,6 @@ public class EndpointCertificateManager { @NotNull private Optional<EndpointCertificateMetadata> getOrProvision(Instance instance, ZoneId zone, Optional<DeploymentInstanceSpec> instanceSpec) { - boolean endpointCertInSharedRouting = this.endpointCertInSharedRouting.with(FetchVector.Dimension.APPLICATION_ID, instance.id().serializedForm()).value(); - if (!zoneRegistry.zones().directlyRouted().ids().contains(zone) && !endpointCertInSharedRouting) - return Optional.empty(); - final var currentCertificateMetadata = curator.readEndpointCertificateMetadata(instance.id()); if (currentCertificateMetadata.isEmpty()) { @@ -130,74 +105,10 @@ public class EndpointCertificateManager { return Optional.of(reprovisionedCertificateMetadata); } - if (!useEndpointCertificateMaintainer.value()) { - // Look for and use refreshed certificate - var latestAvailableVersion = latestVersionInSecretStore(currentCertificateMetadata.get()); - if (latestAvailableVersion.isPresent() && latestAvailableVersion.getAsInt() > currentCertificateMetadata.get().version()) { - var refreshedCertificateMetadata = currentCertificateMetadata.get() - .withVersion(latestAvailableVersion.getAsInt()) - .withLastRefreshed(clock.instant().getEpochSecond()); - validateEndpointCertificate(refreshedCertificateMetadata, instance, zone); - curator.writeEndpointCertificateMetadata(instance.id(), refreshedCertificateMetadata); - return Optional.of(refreshedCertificateMetadata); - } - } - validateEndpointCertificate(currentCertificateMetadata.get(), instance, zone); return currentCertificateMetadata; } - enum CleanupMode { - DISABLE, - DRYRUN, - ENABLE - } - - private void deleteUnusedCertificates() { - CleanupMode mode = CleanupMode.valueOf(deleteUnusedEndpointCertificates.value().toUpperCase()); - if (mode == CleanupMode.DISABLE || useEndpointCertificateMaintainer.value()) return; - - var oneMonthAgo = clock.instant().minus(30, ChronoUnit.DAYS); - curator.readAllEndpointCertificateMetadata().forEach((applicationId, storedMetaData) -> { - var lastRequested = Instant.ofEpochSecond(storedMetaData.lastRequested()); - if (lastRequested.isBefore(oneMonthAgo) && hasNoDeployments(applicationId)) { - try (Lock lock = lock(applicationId)) { - if (Optional.of(storedMetaData).equals(curator.readEndpointCertificateMetadata(applicationId))) { - log.log(Level.INFO, "Cert for app " + applicationId.serializedForm() - + " has not been requested in a month and app has no deployments" - + (mode == CleanupMode.ENABLE ? ", deleting from provider and ZK" : "")); - if (mode == CleanupMode.ENABLE) { - endpointCertificateProvider.deleteCertificate(applicationId, storedMetaData); - curator.deleteEndpointCertificateMetadata(applicationId); - } - } - } - } - }); - } - - private Lock lock(ApplicationId applicationId) { - return curator.lock(TenantAndApplicationId.from(applicationId)); - } - - private boolean hasNoDeployments(ApplicationId applicationId) { - var deployments = curator.readApplication(TenantAndApplicationId.from(applicationId)) - .flatMap(app -> app.get(applicationId.instance())) - .map(Instance::deployments); - - return deployments.isEmpty() || deployments.get().size() == 0; - } - - private OptionalInt latestVersionInSecretStore(EndpointCertificateMetadata originalCertificateMetadata) { - try { - var certVersions = new HashSet<>(secretStore.listSecretVersions(originalCertificateMetadata.certName())); - var keyVersions = new HashSet<>(secretStore.listSecretVersions(originalCertificateMetadata.keyName())); - return Sets.intersection(certVersions, keyVersions).stream().mapToInt(Integer::intValue).max(); - } catch (SecretNotFoundException s) { - return OptionalInt.empty(); // Likely because the certificate is very recently provisioned - keep current version - } - } - private EndpointCertificateMetadata provisionEndpointCertificate(Instance instance, Optional<EndpointCertificateMetadata> currentMetadata, ZoneId deploymentZone, Optional<DeploymentInstanceSpec> instanceSpec) { List<String> currentlyPresentNames = currentMetadata.isPresent() ? diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java index 786547d4a67..8a48cbd281d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.dns; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; +import com.yahoo.yolean.Exceptions; import java.util.ArrayList; import java.util.Collection; @@ -82,7 +83,7 @@ public class NameServiceQueue { request.dispatchTo(nameService); queue.requests.poll(); } catch (Exception e) { - log.log(Level.WARNING, "Failed to execute " + request + ": " + e.getMessage() + + log.log(Level.WARNING, "Failed to execute " + request + ": " + Exceptions.toMessageString(e) + ", request will be retried"); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 56b7e5b2e46..979cd9060d9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -118,13 +118,13 @@ public class ControllerMaintenance extends AbstractComponent { this.outstandingChangeDeployer = duration(3, MINUTES); this.versionStatusUpdater = duration(3, MINUTES); this.readyJobsTrigger = duration(1, MINUTES); - this.deploymentMetricsMaintainer = duration(5, MINUTES); + this.deploymentMetricsMaintainer = duration(10, MINUTES); this.applicationOwnershipConfirmer = duration(12, HOURS); - this.systemUpgrader = duration(1, MINUTES); + this.systemUpgrader = duration(90, SECONDS); this.jobRunner = duration(90, SECONDS); this.osUpgrader = duration(1, MINUTES); this.contactInformationMaintainer = duration(12, HOURS); - this.nameServiceDispatcher = duration(10, SECONDS); + this.nameServiceDispatcher = duration(30, SECONDS); this.costReportMaintainer = duration(2, HOURS); this.resourceMeterMaintainer = duration(1, MINUTES); this.cloudEventReporter = duration(30, MINUTES); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index a1d7c3d16b4..e59875e9588 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -8,8 +8,6 @@ import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.log.LogLevel; import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; @@ -27,12 +25,13 @@ import java.time.temporal.ChronoUnit; import java.util.HashSet; import java.util.Optional; import java.util.OptionalInt; -import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; /** - * Updates refreshed endpoint certificates and triggers redeployment, and deletes unused certificates + * Updates refreshed endpoint certificates and triggers redeployment, and deletes unused certificates. + * + * See also EndpointCertificateManager, which provisions, reprovisions and validates certificates on deploy * * @author andreer */ @@ -45,7 +44,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { private final CuratorDb curator; private final SecretStore secretStore; private final EndpointCertificateProvider endpointCertificateProvider; - private final BooleanFlag useEndpointCertificateMaintainer; public EndpointCertificateMaintainer(Controller controller, Duration interval) { super(controller, interval, null, SystemName.all()); @@ -54,15 +52,10 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { this.secretStore = controller.secretStore(); this.curator = controller().curator(); this.endpointCertificateProvider = controller.serviceRegistry().endpointCertificateProvider(); - this.useEndpointCertificateMaintainer = Flags.USE_ENDPOINT_CERTIFICATE_MAINTAINER.bindTo(controller().flagSource()); } @Override protected boolean maintain() { - - if (!useEndpointCertificateMaintainer.value()) - return true; // handled by EndpointCertificateManager for now - try { // In order of importance deployRefreshedCertificates(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index d6739581c79..73528977166 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -80,10 +80,12 @@ public class JobRunner extends ControllerMaintainer { /** Advances each of the ready steps for the given run, or marks it as finished, and stashes it. Public for testing. */ public void advance(Run run) { if ( ! run.hasFailed() - && controller().clock().instant().isAfter(run.start().plus(jobTimeout))) { - jobs.abort(run.id()); - advance(jobs.run(run.id()).get()); - } + && controller().clock().instant().isAfter(run.start().plus(jobTimeout))) + executors.execute(() -> { + jobs.abort(run.id()); + advance(jobs.run(run.id()).get()); + }); + else if (run.readySteps().isEmpty()) executors.execute(() -> finish(run.id())); else diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 5a1496bf507..9331e5086cc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import ai.vespa.hosted.api.Signatures; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; @@ -100,9 +102,6 @@ import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import com.yahoo.yolean.Exceptions; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; import javax.ws.rs.ForbiddenException; import javax.ws.rs.InternalServerErrorException; @@ -151,6 +150,8 @@ import static java.util.stream.Collectors.toUnmodifiableList; @SuppressWarnings("unused") // created by injection public class ApplicationApiHandler extends LoggingRequestHandler { + private static final ObjectMapper jsonMapper = new ObjectMapper(); + private static final String OPTIONAL_PREFIX = "/api"; private final Controller controller; @@ -789,15 +790,15 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private JsonResponse buildResponseFromProtonMetrics(List<ProtonMetrics> protonMetrics) { try { - var jsonObject = new JSONObject(); - var jsonArray = new JSONArray(); + var jsonObject = jsonMapper.createObjectNode(); + var jsonArray = jsonMapper.createArrayNode(); for (ProtonMetrics metrics : protonMetrics) { - jsonArray.put(metrics.toJson()); + jsonArray.add(metrics.toJson()); } - jsonObject.put("metrics", jsonArray); - return new JsonResponse(200, jsonObject.toString()); - } catch (JSONException e) { - log.severe("Unable to build JsonResponse with Proton data"); + jsonObject.set("metrics", jsonArray); + return new JsonResponse(200, jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonObject)); + } catch (JsonProcessingException e) { + log.log(Level.SEVERE, "Unable to build JsonResponse with Proton data: " + e.getMessage(), e); return new JsonResponse(500, ""); } } @@ -1682,11 +1683,20 @@ public class ApplicationApiHandler extends LoggingRequestHandler { controller.jobController().deploy(id, type, version, applicationPackage); RunId runId = controller.jobController().last(id, type).get().id(); + DeploymentId deploymentId = new DeploymentId(id, type.zone(controller.system())); + Slime slime = new Slime(); Cursor rootObject = slime.setObject(); rootObject.setString("message", "Deployment started in " + runId + ". This may take about 15 minutes the first time."); rootObject.setLong("run", runId.number()); + var endpointArray = rootObject.setArray("endpoints"); + EndpointList zoneEndpoints = controller.routing().endpointsOf(deploymentId) + .scope(Endpoint.Scope.zone) + .not().legacy(); + for (var endpoint : controller.routing().directEndpoints(zoneEndpoints, deploymentId.applicationId())) { + toSlime(endpoint, endpointArray.addObject()); + } return new SlimeJsonResponse(slime); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 8fcbb365804..63452f40dbb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -736,8 +736,8 @@ public class ControllerTest { .map(prefix -> prefix + "app1.tenant1." + zone.region().value() + (zone.environment() == Environment.prod ? "" : "." + zone.environment().value()) + ".vespa.oath.cloud"))) - .collect(Collectors.toUnmodifiableList()), - tester.controllerTester().serviceRegistry().endpointCertificateMock().dnsNamesOf(context1.instanceId())); + .collect(Collectors.toUnmodifiableSet()), + Set.copyOf(tester.controllerTester().serviceRegistry().endpointCertificateMock().dnsNamesOf(context1.instanceId()))); // Next deployment reuses certificate context1.submit(applicationPackage).deploy(); @@ -751,7 +751,7 @@ public class ControllerTest { tester.controller().applications().deploy(context2.instanceId(), devZone, Optional.of(applicationPackage), DeployOptions.none()); assertTrue("Application deployed and activated", tester.configServer().application(context2.instanceId(), devZone).get().activated()); - assertFalse("Does not provision certificate in zones with routing layer", certificate.apply(context2.instance()).isPresent()); + assertTrue("Provisions certificate also in zone with routing layer", certificate.apply(context2.instance()).isPresent()); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java index 4f2000b1902..0d70fe48a77 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java @@ -146,27 +146,6 @@ public class EndpointCertificateManagerTest { } @Test - public void uses_refreshed_certificate_when_available_and_valid() { - secretStore.setSecret(testKeyName, "secret-key", 7); - secretStore.setSecret(testCertName, "cert", 7); - secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 8); - secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 9); - secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), 8); - mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7, 0, "request_id", - List.of("vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud", - "default.default.global.vespa.oath.cloud", - "*.default.default.global.vespa.oath.cloud", - "default.default.aws-us-east-1a.vespa.oath.cloud", - "*.default.default.aws-us-east-1a.vespa.oath.cloud"), - "issuer", Optional.empty(), Optional.empty())); - Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.empty()); - assertTrue(endpointCertificateMetadata.isPresent()); - assertEquals(testKeyName, endpointCertificateMetadata.get().keyName()); - assertEquals(testCertName, endpointCertificateMetadata.get().certName()); - assertEquals(8, endpointCertificateMetadata.get().version()); - } - - @Test public void reprovisions_certificate_when_necessary() { mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, 0, "uuid", List.of(), "issuer", Optional.empty(), Optional.empty())); secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), 0); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java index dbf102f23d7..66bda66bbf9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java @@ -2,15 +2,12 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.SecretStoreMock; -import org.junit.Before; import org.junit.Test; import java.time.Duration; @@ -33,11 +30,6 @@ public class EndpointCertificateMaintainerTest { private final EndpointCertificateMaintainer maintainer = new EndpointCertificateMaintainer(tester.controller(), Duration.ofHours(1)); private final EndpointCertificateMetadata exampleMetadata = new EndpointCertificateMetadata("keyName", "certName", 0, 0, "uuid", List.of(), "issuer", Optional.empty(), Optional.empty()); - @Before - public void setUp() throws Exception { - ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.USE_ENDPOINT_CERTIFICATE_MAINTAINER.id(), true); - } - @Test public void old_and_unused_cert_is_deleted() { tester.curator().writeEndpointCertificateMetadata(ApplicationId.defaultId(), exampleMetadata); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java index bda5a708a94..2bf6eb39089 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.container.http.filter.FilterChainRepository; import com.yahoo.jdisc.http.filter.SecurityRequestFilter; import com.yahoo.jdisc.http.filter.SecurityRequestFilterChain; +import com.yahoo.test.json.JsonTestHelper; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.hosted.controller.Controller; @@ -65,6 +66,10 @@ public class ContainerTester { .addRoleMember(action, identity); } + public void assertJsonResponse(Supplier<Request> request, File responseFile) { + assertResponse(request.get(), responseFile, 200, false, true); + } + public void assertResponse(Supplier<Request> request, File responseFile) { assertResponse(request.get(), responseFile); } @@ -82,6 +87,10 @@ public class ContainerTester { } public void assertResponse(Request request, File responseFile, int expectedStatusCode, boolean removeWhitespace) { + assertResponse(request, responseFile, expectedStatusCode, removeWhitespace, false); + } + + private void assertResponse(Request request, File responseFile, int expectedStatusCode, boolean removeWhitespace, boolean compareJson) { String expectedResponse = readTestFile(responseFile.toString()); expectedResponse = include(expectedResponse); if (removeWhitespace) expectedResponse = expectedResponse.replaceAll("(\"[^\"]*\")|\\s*", "$1"); // Remove whitespace @@ -106,7 +115,11 @@ public class ContainerTester { expectedResponsePattern, responseString); } } else { - assertEquals(responseFile.toString(), expectedResponse, responseString); + if (compareJson) { + JsonTestHelper.assertJsonEquals(expectedResponse, responseString); + } else { + assertEquals(responseFile.toString(), expectedResponse, responseString); + } } assertEquals("Status code", expectedStatusCode, response.getStatus()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 434c83898ee..6626134b69a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -242,7 +242,8 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/production-us-east-3/", POST) .data(entity) .userIdentity(HOSTED_VESPA_OPERATOR), - "{\"message\":\"Deployment started in run 1 of production-us-east-3 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1}"); + "{\"message\":\"Deployment started in run 1 of production-us-east-3 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1," + + "\"endpoints\":[{\"cluster\":\"default\",\"tls\":true,\"url\":\"https://instance1--application1--tenant1.us-east-3.vespa.oath.cloud:4443/\",\"scope\":\"zone\",\"routingMethod\":\"shared\"}]}"); app1.runJob(JobType.productionUsEast3); tester.controller().applications().deactivate(app1.instanceId(), ZoneId.from("prod", "us-east-3")); @@ -250,7 +251,8 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/dev-us-east-1/", POST) .data(entity) .userIdentity(USER_ID), - "{\"message\":\"Deployment started in run 1 of dev-us-east-1 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1}"); + "{\"message\":\"Deployment started in run 1 of dev-us-east-1 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1," + + "\"endpoints\":[{\"cluster\":\"default\",\"tls\":true,\"url\":\"https://instance1--application1--tenant1.us-east-1.dev.vespa.oath.cloud:4443/\",\"scope\":\"zone\",\"routingMethod\":\"shared\"}]}"); app1.runJob(JobType.devUsEast1); // GET dev application package @@ -514,7 +516,7 @@ public class ApplicationApiTest extends ControllerContainerTest { updateMetrics(); // GET metrics - tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/environment/dev/region/us-east-1/instance/default/metrics", GET) + tester.assertJsonResponse(request("/application/v4/tenant/tenant2/application/application1/environment/dev/region/us-east-1/instance/default/metrics", GET) .userIdentity(USER_ID), new File("proton-metrics.json")); @@ -1426,7 +1428,8 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/new-user/deploy/dev-us-east-1", POST) .data(entity) .userIdentity(userId), - "{\"message\":\"Deployment started in run 1 of dev-us-east-1 for tenant1.application1.new-user. This may take about 15 minutes the first time.\",\"run\":1}"); + "{\"message\":\"Deployment started in run 1 of dev-us-east-1 for tenant1.application1.new-user. This may take about 15 minutes the first time.\",\"run\":1," + + "\"endpoints\":[{\"cluster\":\"default\",\"tls\":true,\"url\":\"https://new-user--application1--tenant1.us-east-1.dev.vespa.oath.cloud:4443/\",\"scope\":\"zone\",\"routingMethod\":\"shared\"}]}"); } @Test @@ -1471,7 +1474,8 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/sandbox/application/myapp/instance/default/deploy/dev-us-east-1", POST) .data(entity) .userIdentity(developer), - "{\"message\":\"Deployment started in run 1 of dev-us-east-1 for sandbox.myapp. This may take about 15 minutes the first time.\",\"run\":1}", + "{\"message\":\"Deployment started in run 1 of dev-us-east-1 for sandbox.myapp. This may take about 15 minutes the first time.\",\"run\":1," + + "\"endpoints\":[{\"cluster\":\"default\",\"tls\":true,\"url\":\"https://myapp--sandbox.us-east-1.dev.vespa.oath.cloud:4443/\",\"scope\":\"zone\",\"routingMethod\":\"shared\"}]}", 200); // To add temporary support allowing tenant admins to launch services @@ -1482,7 +1486,8 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/sandbox/application/myapp/instance/default/deploy/dev-us-east-1", POST) .data(entity) .userIdentity(developer2), - "{\"message\":\"Deployment started in run 2 of dev-us-east-1 for sandbox.myapp. This may take about 15 minutes the first time.\",\"run\":2}", + "{\"message\":\"Deployment started in run 2 of dev-us-east-1 for sandbox.myapp. This may take about 15 minutes the first time.\",\"run\":2," + + "\"endpoints\":[{\"cluster\":\"default\",\"tls\":true,\"url\":\"https://myapp--sandbox.us-east-1.dev.vespa.oath.cloud:4443/\",\"scope\":\"zone\",\"routingMethod\":\"shared\"}]}", 200); @@ -1491,7 +1496,8 @@ public class ApplicationApiTest extends ControllerContainerTest { .data(applicationPackageInstance1.zippedContent()) .contentType("application/zip") .userIdentity(developer2), - "{\"message\":\"Deployment started in run 3 of dev-us-east-1 for sandbox.myapp. This may take about 15 minutes the first time.\",\"run\":3}"); + "{\"message\":\"Deployment started in run 3 of dev-us-east-1 for sandbox.myapp. This may take about 15 minutes the first time.\",\"run\":3," + + "\"endpoints\":[{\"cluster\":\"default\",\"tls\":true,\"url\":\"https://myapp--sandbox.us-east-1.dev.vespa.oath.cloud:4443/\",\"scope\":\"zone\",\"routingMethod\":\"shared\"}]}"); // POST (deploy) an application package not as content type application/zip — not multipart — is disallowed tester.assertResponse(request("/application/v4/tenant/sandbox/application/myapp/instance/default/deploy/dev-us-east-1", POST) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index 828e2856cae..c43abf276c5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.restapi.application; import com.yahoo.component.Version; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.test.json.JsonTestHelper; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; @@ -12,8 +13,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; -import org.json.JSONException; -import org.json.JSONObject; import org.junit.Test; import java.io.ByteArrayOutputStream; @@ -180,12 +179,10 @@ public class JobControllerApiHandlerHelperTest { "jobs-direct-deployment.json"); } - private void compare(HttpResponse response, String expected) throws JSONException, IOException { + private void compare(HttpResponse response, String expected) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); response.render(baos); - JSONObject actualJSON = new JSONObject(new String(baos.toByteArray())); - JSONObject expectedJSON = new JSONObject(expected); - assertEquals(expectedJSON.toString(), actualJSON.toString()); + JsonTestHelper.assertJsonEquals(expected, baos.toString()); } private void assertResponse(HttpResponse response, String fileName) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted-2.json index 8ea3f318d1d..c53cee8fd97 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted-2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted-2.json @@ -1,4 +1,13 @@ { "message": "Deployment started in run 1 of dev-us-east-1 for tenant1.application1.myuser. This may take about 15 minutes the first time.", - "run": 1 + "run": 1, + "endpoints": [ + { + "cluster": "default", + "tls": true, + "url": "https://myuser--application1--tenant1.us-east-1.dev.vespa.oath.cloud:4443/", + "scope": "zone", + "routingMethod": "shared" + } + ] }
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/proton-metrics.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/proton-metrics.json index a7e5b3918d8..3fba9b3c91c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/proton-metrics.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/proton-metrics.json @@ -1,23 +1,26 @@ { - "metrics": [{ - "clusterId": "content/doc/", - "metrics": { - "resourceMemoryUsageAverage": 0.103482, - "documentsReadyCount": 11430, - "documentDiskUsage": 44021, - "resourceDiskUsageAverage": 0.0168421, - "documentsTotalCount": 11430, - "documentsActiveCount": 11430 + "metrics": [ + { + "clusterId": "content/doc/", + "metrics": { + "resourceMemoryUsageAverage": 0.103482, + "documentsReadyCount": 11430.0, + "documentDiskUsage": 44021.0, + "resourceDiskUsageAverage": 0.0168421, + "documentsTotalCount": 11430.0, + "documentsActiveCount": 11430.0 + } + }, + { + "clusterId": "content/music/", + "metrics": { + "resourceMemoryUsageAverage": 0.00912, + "documentsReadyCount": 32000.0, + "documentDiskUsage": 90113.0, + "resourceDiskUsageAverage": 0.23912, + "documentsTotalCount": 32210.0, + "documentsActiveCount": 32210.0 + } } - }, { - "clusterId": "content/music/", - "metrics": { - "resourceMemoryUsageAverage": 0.00912, - "documentsReadyCount": 32000, - "documentDiskUsage": 90113, - "resourceDiskUsageAverage": 0.23912, - "documentsTotalCount": 32210, - "documentsActiveCount": 32210 - } - }] -}
\ No newline at end of file + ] +} diff --git a/dist/vespa.spec b/dist/vespa.spec index b19d1c9a201..42039fae1d8 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -158,6 +158,7 @@ Requires: gdb Requires: nc Requires: net-tools Requires: unzip +Requires: zstd %if 0%{?el7} Requires: llvm7.0 Requires: vespa-icu >= 65.1.0-1 diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerStats.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerStats.java index 797dffdef1f..dc2db50d3ab 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerStats.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/ContainerStats.java @@ -1,44 +1,31 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.dockerapi; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.github.dockerjava.api.model.CpuStatsConfig; -import com.github.dockerjava.api.model.MemoryStatsConfig; -import com.github.dockerjava.api.model.StatisticNetworksConfig; -import com.github.dockerjava.api.model.Statistics; - -import java.io.IOException; -import java.io.UncheckedIOException; +import java.util.Collections; +import java.util.LinkedHashMap; import java.util.Map; -import java.util.Optional; -import java.util.TreeMap; -import java.util.stream.Collectors; +import java.util.Objects; /** - * Wrapper class for {@link com.github.dockerjava.api.model.Statistics} to prevent leaking from docker-java library. + * CPU, memory and network statistics collected from a container. * * @author freva */ +// TODO: Move this to node-admin when docker-api module can be removed public class ContainerStats { + private final Map<String, NetworkStats> networkStatsByInterface; private final MemoryStats memoryStats; private final CpuStats cpuStats; - ContainerStats(Statistics statistics) { - // Network stats are null when container uses host network - this.networkStatsByInterface = Optional.ofNullable(statistics.getNetworks()).orElseGet(Map::of) - .entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - e -> new NetworkStats(e.getValue()), - (u, v) -> { throw new IllegalStateException(); }, - TreeMap::new)); - this.memoryStats = new MemoryStats(statistics.getMemoryStats()); - this.cpuStats = new CpuStats(statistics.getCpuStats()); + public ContainerStats(Map<String, NetworkStats> networkStatsByInterface, MemoryStats memoryStats, CpuStats cpuStats) { + this.networkStatsByInterface = new LinkedHashMap<>(Objects.requireNonNull(networkStatsByInterface)); + this.memoryStats = Objects.requireNonNull(memoryStats); + this.cpuStats = Objects.requireNonNull(cpuStats); } public Map<String, NetworkStats> getNetworks() { - return networkStatsByInterface; + return Collections.unmodifiableMap(networkStatsByInterface); } public MemoryStats getMemoryStats() { @@ -49,7 +36,22 @@ public class ContainerStats { return cpuStats; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ContainerStats that = (ContainerStats) o; + return networkStatsByInterface.equals(that.networkStatsByInterface) && memoryStats.equals(that.memoryStats) && cpuStats.equals(that.cpuStats); + } + + @Override + public int hashCode() { + return Objects.hash(networkStatsByInterface, memoryStats, cpuStats); + } + + /** Statistics for network usage */ public static class NetworkStats { + private final long rxBytes; private final long rxDropped; private final long rxErrors; @@ -57,40 +59,109 @@ public class ContainerStats { private final long txDropped; private final long txErrors; - private NetworkStats(StatisticNetworksConfig statisticNetworksConfig) { - this.rxBytes = statisticNetworksConfig.getRxBytes(); - this.rxDropped = statisticNetworksConfig.getRxDropped(); - this.rxErrors = statisticNetworksConfig.getRxErrors(); - this.txBytes = statisticNetworksConfig.getTxBytes(); - this.txDropped = statisticNetworksConfig.getTxDropped(); - this.txErrors = statisticNetworksConfig.getTxErrors(); + public NetworkStats(long rxBytes, long rxDropped, long rxErrors, long txBytes, long txDropped, long txErrors) { + this.rxBytes = rxBytes; + this.rxDropped = rxDropped; + this.rxErrors = rxErrors; + this.txBytes = txBytes; + this.txDropped = txDropped; + this.txErrors = txErrors; } + /** Returns received bytes */ public long getRxBytes() { return this.rxBytes; } + + /** Returns received bytes, which was dropped */ public long getRxDropped() { return this.rxDropped; } + + /** Returns received errors */ public long getRxErrors() { return this.rxErrors; } + + /** Returns transmitted bytes */ public long getTxBytes() { return this.txBytes; } + + /** Returns transmitted bytes, which was dropped */ public long getTxDropped() { return this.txDropped; } + + /** Returns transmission errors */ public long getTxErrors() { return this.txErrors; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NetworkStats that = (NetworkStats) o; + return rxBytes == that.rxBytes && rxDropped == that.rxDropped && rxErrors == that.rxErrors && txBytes == that.txBytes && txDropped == that.txDropped && txErrors == that.txErrors; + } + + @Override + public int hashCode() { + return Objects.hash(rxBytes, rxDropped, rxErrors, txBytes, txDropped, txErrors); + } + + @Override + public String toString() { + return "NetworkStats{" + + "rxBytes=" + rxBytes + + ", rxDropped=" + rxDropped + + ", rxErrors=" + rxErrors + + ", txBytes=" + txBytes + + ", txDropped=" + txDropped + + ", txErrors=" + txErrors + + '}'; + } + } - public class MemoryStats { + /** Statistics for memory usage */ + public static class MemoryStats { + private final long cache; private final long usage; private final long limit; - private MemoryStats(MemoryStatsConfig memoryStats) { - this.cache = memoryStats.getStats().getCache(); - this.usage = memoryStats.getUsage(); - this.limit = memoryStats.getLimit(); + public MemoryStats(long cache, long usage, long limit) { + this.cache = cache; + this.usage = usage; + this.limit = limit; } + /** Returns memory used by cache in bytes */ public long getCache() { return this.cache; } + + /** Returns memory usage in bytes */ public long getUsage() { return this.usage; } + + /** Returns memory limit in bytes */ public long getLimit() { return this.limit; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + MemoryStats that = (MemoryStats) o; + return cache == that.cache && usage == that.usage && limit == that.limit; + } + + @Override + public int hashCode() { + return Objects.hash(cache, usage, limit); + } + + @Override + public String toString() { + return "MemoryStats{" + + "cache=" + cache + + ", usage=" + usage + + ", limit=" + limit + + '}'; + } + } - public class CpuStats { + /** Statistics for CPU usage */ + public static class CpuStats { + private final int onlineCpus; private final long systemCpuUsage; private final long totalUsage; @@ -99,15 +170,15 @@ public class ContainerStats { private final long throttlingActivePeriods; private final long throttledPeriods; - public CpuStats(CpuStatsConfig cpuStats) { - // Added in 1.27 - this.onlineCpus = cpuStats.getCpuUsage().getPercpuUsage().size(); - this.systemCpuUsage = cpuStats.getSystemCpuUsage(); - this.totalUsage = cpuStats.getCpuUsage().getTotalUsage(); - this.usageInKernelMode = cpuStats.getCpuUsage().getUsageInKernelmode(); - this.throttledTime = cpuStats.getThrottlingData().getThrottledTime(); - this.throttlingActivePeriods = cpuStats.getThrottlingData().getPeriods(); - this.throttledPeriods = cpuStats.getThrottlingData().getThrottledPeriods(); + public CpuStats(int onlineCpus, long systemCpuUsage, long totalUsage, long usageInKernelMode, + long throttledTime, long throttlingActivePeriods, long throttledPeriods) { + this.onlineCpus = onlineCpus; + this.systemCpuUsage = systemCpuUsage; + this.totalUsage = totalUsage; + this.usageInKernelMode = usageInKernelMode; + this.throttledTime = throttledTime; + this.throttlingActivePeriods = throttlingActivePeriods; + this.throttledPeriods = throttledPeriods; } public int getOnlineCpus() { return this.onlineCpus; } @@ -129,15 +200,33 @@ public class ContainerStats { /** Number of periods this container hit the throttling limit */ public long getThrottledPeriods() { return throttledPeriods; } - } - // For testing only, create ContainerStats from JSON returned by docker daemon stats API - public static ContainerStats fromJson(String json) { - try { - Statistics statistics = new ObjectMapper().readValue(json, Statistics.class); - return new ContainerStats(statistics); - } catch (IOException e) { - throw new UncheckedIOException(e); + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + CpuStats cpuStats = (CpuStats) o; + return onlineCpus == cpuStats.onlineCpus && systemCpuUsage == cpuStats.systemCpuUsage && totalUsage == cpuStats.totalUsage && usageInKernelMode == cpuStats.usageInKernelMode && throttledTime == cpuStats.throttledTime && throttlingActivePeriods == cpuStats.throttlingActivePeriods && throttledPeriods == cpuStats.throttledPeriods; + } + + @Override + public int hashCode() { + return Objects.hash(onlineCpus, systemCpuUsage, totalUsage, usageInKernelMode, throttledTime, throttlingActivePeriods, throttledPeriods); } + + @Override + public String toString() { + return "CpuStats{" + + "onlineCpus=" + onlineCpus + + ", systemCpuUsage=" + systemCpuUsage + + ", totalUsage=" + totalUsage + + ", usageInKernelMode=" + usageInKernelMode + + ", throttledTime=" + throttledTime + + ", throttlingActivePeriods=" + throttlingActivePeriods + + ", throttledPeriods=" + throttledPeriods + + '}'; + } + } + } diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerEngine.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerEngine.java index a45855764ed..630efb7990f 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerEngine.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerEngine.java @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.dockerapi; +import com.fasterxml.jackson.databind.ObjectMapper; import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.command.ExecCreateCmdResponse; import com.github.dockerjava.api.command.InspectContainerResponse; @@ -33,6 +34,8 @@ import com.yahoo.vespa.hosted.dockerapi.metrics.Gauge; import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics; import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; import java.time.Clock; import java.time.Duration; import java.time.Instant; @@ -43,6 +46,7 @@ import java.util.Map; import java.util.Optional; import java.util.OptionalLong; import java.util.Set; +import java.util.TreeMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.logging.Level; @@ -198,8 +202,7 @@ public class DockerEngine implements ContainerEngine { try { DockerStatsCallback statsCallback = dockerClient.statsCmd(containerName.asString()).exec(new DockerStatsCallback()); statsCallback.awaitCompletion(5, TimeUnit.SECONDS); - - return statsCallback.stats.map(ContainerStats::new); + return statsCallback.stats.map(DockerEngine::containerStatsFrom); } catch (NotFoundException ignored) { return Optional.empty(); } catch (RuntimeException | InterruptedException e) { @@ -437,4 +440,39 @@ public class DockerEngine implements ContainerEngine { return DockerClientImpl.getInstance(dockerClientConfig) .withDockerCmdExecFactory(dockerFactory); } + + private static ContainerStats containerStatsFrom(Statistics statistics) { + return new ContainerStats(Optional.ofNullable(statistics.getNetworks()).orElseGet(Map::of) + .entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> new ContainerStats.NetworkStats(e.getValue().getRxBytes(), e.getValue().getRxDropped(), + e.getValue().getRxErrors(), e.getValue().getTxBytes(), + e.getValue().getTxDropped(), e.getValue().getTxErrors()), + (u, v) -> { + throw new IllegalStateException(); + }, + TreeMap::new)), + new ContainerStats.MemoryStats(statistics.getMemoryStats().getStats().getCache(), + statistics.getMemoryStats().getUsage(), + statistics.getMemoryStats().getLimit()), + new ContainerStats.CpuStats(statistics.getCpuStats().getCpuUsage().getPercpuUsage().size(), + statistics.getCpuStats().getSystemCpuUsage(), + statistics.getCpuStats().getCpuUsage().getTotalUsage(), + statistics.getCpuStats().getCpuUsage().getUsageInKernelmode(), + statistics.getCpuStats().getThrottlingData().getThrottledTime(), + statistics.getCpuStats().getThrottlingData().getPeriods(), + statistics.getCpuStats().getThrottlingData().getThrottledPeriods())); + } + + // For testing only, create ContainerStats from JSON returned by docker daemon stats API + public static ContainerStats statsFromJson(String json) { + try { + Statistics statistics = new ObjectMapper().readValue(json, Statistics.class); + return containerStatsFrom(statistics); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } diff --git a/document/abi-spec.json b/document/abi-spec.json index df1fb840cbc..4a94d7e55de 100644 --- a/document/abi-spec.json +++ b/document/abi-spec.json @@ -3164,6 +3164,8 @@ "public void printXml(com.yahoo.document.serialization.XmlStream)", "public void clear()", "public void assign(java.lang.Object)", + "public void assignSerializedTensor(byte[])", + "public java.util.Optional getSerializedTensor()", "public void assignTensor(java.util.Optional)", "public void serialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldWriter)", "public void deserialize(com.yahoo.document.Field, com.yahoo.document.serialization.FieldReader)", @@ -4471,4 +4473,4 @@ "protected com.yahoo.document.update.ValueUpdate$ValueUpdateClassID valueUpdateClassID" ] } -}
\ No newline at end of file +} diff --git a/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java index 8e7dbd3512a..3177ec54465 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/TensorFieldValue.java @@ -6,8 +6,10 @@ import com.yahoo.document.TensorDataType; import com.yahoo.document.serialization.FieldReader; import com.yahoo.document.serialization.FieldWriter; import com.yahoo.document.serialization.XmlStream; +import com.yahoo.io.GrowableByteBuffer; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.serialization.TypedBinaryFormat; import java.util.Optional; @@ -20,6 +22,8 @@ public class TensorFieldValue extends FieldValue { private Optional<Tensor> tensor; + private Optional<byte[]> serializedTensor; + private Optional<TensorDataType> dataType; /** @@ -28,27 +32,49 @@ public class TensorFieldValue extends FieldValue { * The tensor (and tensor type) can later be assigned with assignTensor(). */ public TensorFieldValue() { - this.dataType = Optional.empty(); - this.tensor = Optional.empty(); + this.dataType = Optional.empty(); + this.serializedTensor = Optional.empty(); + this.tensor = Optional.empty(); } /** Create an empty tensor field value for the given tensor type */ public TensorFieldValue(TensorType type) { this.dataType = Optional.of(new TensorDataType(type)); + this.serializedTensor = Optional.empty(); this.tensor = Optional.empty(); } /** Create a tensor field value containing the given tensor */ public TensorFieldValue(Tensor tensor) { this.dataType = Optional.of(new TensorDataType(tensor.type())); + this.serializedTensor = Optional.empty(); this.tensor = Optional.of(tensor); } + private void lazyDeserialize() { + if (tensor.isEmpty() && serializedTensor.isPresent()) { + Tensor t = TypedBinaryFormat.decode(Optional.empty(), GrowableByteBuffer.wrap(serializedTensor.get())); + if (dataType.isEmpty()) { + this.dataType = Optional.of(new TensorDataType(t.type())); + this.tensor = Optional.of(t); + } else { + if (t.type().isAssignableTo(dataType.get().getTensorType())) { + this.tensor = Optional.of(t); + } else { + throw new IllegalArgumentException("Type mismatch: Cannot assign tensor of type " + t.type() + + " to field of type " + dataType.get()); + } + } + } + } + public Optional<Tensor> getTensor() { + lazyDeserialize(); return tensor; } public Optional<TensorType> getTensorType() { + lazyDeserialize(); return dataType.isPresent() ? Optional.of(dataType.get().getTensorType()) : Optional.empty(); } @@ -59,8 +85,9 @@ public class TensorFieldValue extends FieldValue { @Override public String toString() { - if (tensor.isPresent()) { - return tensor.get().toString(); + var t = getTensor(); + if (t.isPresent()) { + return t.get().toString(); } else { return "null"; } @@ -74,6 +101,7 @@ public class TensorFieldValue extends FieldValue { @Override public void clear() { tensor = Optional.empty(); + serializedTensor = Optional.empty(); } @Override @@ -90,12 +118,28 @@ public class TensorFieldValue extends FieldValue { } } + public void assignSerializedTensor(byte[] data) { + serializedTensor = Optional.of(data); + tensor = Optional.empty(); + } + + public Optional<byte[]> getSerializedTensor() { + if (serializedTensor.isPresent()) { + return serializedTensor; + } else if (tensor.isPresent()) { + serializedTensor = Optional.of(TypedBinaryFormat.encode(tensor.get())); + assert(serializedTensor.isPresent()); + } + return serializedTensor; + } + /** * Assigns the given tensor to this field value. * * The tensor type is also set from the given tensor if it was not set before. */ public void assignTensor(Optional<Tensor> tensor) { + this.serializedTensor = Optional.empty(); if (tensor.isPresent()) { if (getTensorType().isPresent() && !tensor.get().type().isAssignableTo(getTensorType().get())) { @@ -126,7 +170,7 @@ public class TensorFieldValue extends FieldValue { TensorFieldValue other = (TensorFieldValue)o; if ( ! getTensorType().equals(other.getTensorType())) return false; - if ( ! tensor.equals(other.tensor)) return false; + if ( ! getTensor().equals(other.getTensor())) return false; return true; } @@ -136,4 +180,3 @@ public class TensorFieldValue extends FieldValue { } } - diff --git a/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java b/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java deleted file mode 100644 index 67508c61b23..00000000000 --- a/document/src/main/java/com/yahoo/document/json/SingleDocumentParser.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.document.json; - -import com.fasterxml.jackson.core.JsonFactory; -import com.yahoo.document.DocumentOperation; -import com.yahoo.document.DocumentPut; -import com.yahoo.document.DocumentTypeManager; -import com.yahoo.document.DocumentUpdate; -import com.yahoo.vespaxmlparser.DocumentFeedOperation; -import com.yahoo.vespaxmlparser.DocumentUpdateFeedOperation; -import com.yahoo.vespaxmlparser.FeedOperation; - -import java.io.IOException; -import java.io.InputStream; - -/** - * Parser that supports parsing PUT operation and UPDATE operation. - * - * @author dybis - */ -public class SingleDocumentParser { - - private static final JsonFactory jsonFactory = new JsonFactory().disable(JsonFactory.Feature.CANONICALIZE_FIELD_NAMES); - private DocumentTypeManager docMan; - - public SingleDocumentParser(DocumentTypeManager docMan) { - this.docMan = docMan; - } - - public FeedOperation parsePut(InputStream inputStream, String docId) { - return parse(inputStream, docId, DocumentOperationType.PUT); - } - - public FeedOperation parseUpdate(InputStream inputStream, String docId) { - return parse(inputStream, docId, DocumentOperationType.UPDATE); - } - - private FeedOperation parse(InputStream inputStream, String docId, DocumentOperationType documentOperationType) { - JsonReader reader = new JsonReader(docMan, inputStream, jsonFactory); - DocumentOperation documentOperation = reader.readSingleDocument(documentOperationType, docId); - try { - inputStream.close(); - } catch (IOException e) { - throw new IllegalStateException(e); - } - if (documentOperationType == DocumentOperationType.PUT) { - return new DocumentFeedOperation(((DocumentPut) documentOperation).getDocument(), documentOperation.getCondition()); - } else { - return new DocumentUpdateFeedOperation((DocumentUpdate) documentOperation, documentOperation.getCondition()); - } - } - -} diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java index 92b3b566b85..914f0ad8759 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentDeserializer6.java @@ -97,7 +97,7 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu final public DocumentTypeManager getDocumentTypeManager() { return manager; } public void read(Document document) { - read(null, document); + read(null, document); } @SuppressWarnings("deprecation") @@ -243,8 +243,7 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu int encodedTensorLength = buf.getInt1_4Bytes(); if (encodedTensorLength > 0) { byte[] encodedTensor = getBytes(null, encodedTensorLength); - value.assign(TypedBinaryFormat.decode(value.getTensorType(), - GrowableByteBuffer.wrap(encodedTensor))); + value.assignSerializedTensor(encodedTensor); } else { value.clear(); } @@ -374,9 +373,9 @@ public class VespaDocumentDeserializer6 extends BufferSerializer implements Docu Integer f_id = fieldIdsAndLengths.get(i).first; Field structField = priType.getField(f_id); if (structField != null) { - FieldValue value = structField.getDataType().createFieldValue(); - value.deserialize(structField, this); - primary.setFieldValue(structField, value); + FieldValue value = structField.getDataType().createFieldValue(); + value.deserialize(structField, this); + primary.setFieldValue(structField, value); } //jump to beginning of next field: position(posBefore + fieldIdsAndLengths.get(i).second.intValue()); diff --git a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java index 3fca853b4d1..1a5c5c8257b 100644 --- a/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java +++ b/document/src/main/java/com/yahoo/document/serialization/VespaDocumentSerializer6.java @@ -290,10 +290,10 @@ public class VespaDocumentSerializer6 extends BufferSerializer implements Docume @Override public void write(FieldBase field, TensorFieldValue value) { - if (value.getTensor().isPresent()) { - byte[] encodedTensor = TypedBinaryFormat.encode(value.getTensor().get()); - buf.putInt1_4Bytes(encodedTensor.length); - buf.put(encodedTensor); + var encodedTensor = value.getSerializedTensor(); + if (encodedTensor.isPresent()) { + buf.putInt1_4Bytes(encodedTensor.get().length); + buf.put(encodedTensor.get()); } else { buf.putInt1_4Bytes(0); } diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp index 0830c84fe39..7e61752cc1e 100644 --- a/document/src/tests/documentselectparsertest.cpp +++ b/document/src/tests/documentselectparsertest.cpp @@ -754,9 +754,9 @@ TEST_F(DocumentSelectParserTest, operators_5) PARSE("\"foo\".hash() == 123", *_doc[0], False); PARSEI("(234).hash() == 123", *_doc[0], False); PARSE("now() > 1311862500", *_doc[8], True); - PARSE("now() < 1611862500", *_doc[8], True); + PARSE("now() < 1911862500", *_doc[8], True); PARSE("now() < 1311862500", *_doc[8], False); - PARSE("now() > 1611862500", *_doc[8], False); + PARSE("now() > 1911862500", *_doc[8], False); // Arithmetics PARSEI("id.specific.hash() % 10 = 8", *_doc[0], True); diff --git a/document/src/vespa/document/bucket/bucket.h b/document/src/vespa/document/bucket/bucket.h index 44068e1c443..f189c2951c9 100644 --- a/document/src/vespa/document/bucket/bucket.h +++ b/document/src/vespa/document/bucket/bucket.h @@ -32,7 +32,7 @@ public: vespalib::string toString() const; struct hash { - size_t operator () (const Bucket& b) const { + size_t operator () (const Bucket& b) const noexcept { size_t hash1 = BucketId::hash()(b.getBucketId()); size_t hash2 = BucketSpace::hash()(b.getBucketSpace()); // Formula taken from std::hash_combine proposal diff --git a/document/src/vespa/document/bucket/bucketid.cpp b/document/src/vespa/document/bucket/bucketid.cpp index 668798d6c39..1b9cf1e5304 100644 --- a/document/src/vespa/document/bucket/bucketid.cpp +++ b/document/src/vespa/document/bucket/bucketid.cpp @@ -71,7 +71,7 @@ Initialize _initializeUsedMasks; } -void BucketId::initialize() { +void BucketId::initialize() noexcept { fillUsedMasks(BucketId::_usedMasks, BucketId::maxNumBits); fillStripMasks(BucketId::_stripMasks, BucketId::maxNumBits); } @@ -91,7 +91,7 @@ void BucketId::throwFailedSetUsedBits(uint32_t used, uint32_t availBits) { } BucketId::Type -BucketId::reverse(Type id) +BucketId::reverse(Type id) noexcept { id = ((id & 0x5555555555555555l) << 1) | ((id & 0xaaaaaaaaaaaaaaaal) >> 1); id = ((id & 0x3333333333333333l) << 2) | ((id & 0xccccccccccccccccl) >> 2); @@ -100,7 +100,7 @@ BucketId::reverse(Type id) } BucketId::Type -BucketId::keyToBucketId(Type key) +BucketId::keyToBucketId(Type key) noexcept { Type retVal = reverse(key); @@ -113,7 +113,7 @@ BucketId::keyToBucketId(Type key) } bool -BucketId::contains(const BucketId& id) const +BucketId::contains(const BucketId& id) const noexcept { if (id.getUsedBits() < getUsedBits()) { return false; diff --git a/document/src/vespa/document/bucket/bucketid.h b/document/src/vespa/document/bucket/bucketid.h index b31f9080acc..675a0d23ebd 100644 --- a/document/src/vespa/document/bucket/bucketid.h +++ b/document/src/vespa/document/bucket/bucketid.h @@ -37,7 +37,7 @@ class BucketId { public: struct hash { - size_t operator () (const BucketId& g) const { + size_t operator () (const BucketId& g) const noexcept { return g.getId(); } }; @@ -55,23 +55,23 @@ public: /** Create a bucket id using a set of bits from a raw unchecked value. */ BucketId(uint32_t useBits, Type id) noexcept : _id(createUsedBits(useBits, id)) { } - bool operator<(const BucketId& id) const { + bool operator<(const BucketId& id) const noexcept { return getId() < id.getId(); } - bool operator==(const BucketId& id) const { return getId() == id.getId(); } - bool operator!=(const BucketId& id) const { return getId() != id.getId(); } + bool operator==(const BucketId& id) const noexcept { return getId() == id.getId(); } + bool operator!=(const BucketId& id) const noexcept { return getId() != id.getId(); } vespalib::string toString() const; - bool valid() const { + bool valid() const noexcept { return validUsedBits(getUsedBits()); } - static bool validUsedBits(uint32_t usedBits) { + static bool validUsedBits(uint32_t usedBits) noexcept { return (usedBits >= minNumBits) && (usedBits <= maxNumBits); } - bool isSet() const { + bool isSet() const noexcept { return _id != 0u; } /** @@ -79,14 +79,14 @@ public: * verify that two different documents belong to the same bucket given some * level of bucket splitting, use this to ignore the unused bits. */ - BucketId stripUnused() const { return BucketId(getUsedBits(), getId()); } + BucketId stripUnused() const noexcept { return BucketId(getUsedBits(), getId()); } /** * Checks whether the given bucket is contained within this bucket. That is, * if it is the same bucket, or if it is a bucket using more bits, which is * identical to this one if set to use as many bits as this one. */ - bool contains(const BucketId& id) const; + bool contains(const BucketId& id) const noexcept; // Functions exposing internals we want to make users independent of @@ -97,7 +97,7 @@ public: static constexpr uint32_t maxNumBits = (8 * sizeof(Type) - CountBits); static constexpr uint32_t minNumBits = 1u; // See comment above. - uint32_t getUsedBits() const { return _id >> maxNumBits; } + uint32_t getUsedBits() const noexcept { return _id >> maxNumBits; } void setUsedBits(uint32_t used) { uint32_t availBits = maxNumBits; @@ -113,22 +113,22 @@ public: } /** Get the bucket id value stripped of the bits that are not in use. */ - Type getId() const { return (_id & getStripMask()); } + Type getId() const noexcept { return (_id & getStripMask()); } /** * Get the bucket id value stripped of the count bits plus the bits that * are not in use. */ - Type withoutCountBits() const { return (_id & getUsedMask()); } + Type withoutCountBits() const noexcept { return (_id & getUsedMask()); } - Type getRawId() const { return _id; } + Type getRawId() const noexcept { return _id; } /** * Reverses the bits in the given number, except the countbits part. * Used for sorting in the bucket database as we want related buckets * to be sorted next to each other. */ - static Type bucketIdToKey(Type id) { + static Type bucketIdToKey(Type id) noexcept { Type retVal = reverse(id); Type usedCountLSB = id >> maxNumBits; @@ -139,39 +139,39 @@ public: return retVal; } - static Type keyToBucketId(Type key); + static Type keyToBucketId(Type key) noexcept ; /** * Reverses the bucket id bitwise, except the countbits part, * and returns the value, */ - Type toKey() const { return bucketIdToKey(getId()); }; + Type toKey() const noexcept { return bucketIdToKey(getId()); }; /** * Reverses the order of the bits in the bucket id. */ - static Type reverse(Type id); + static Type reverse(Type id) noexcept; /** * Returns the value of the Nth bit, counted in the reverse order of the * bucket id. */ - uint8_t getBit(uint32_t n) const { + uint8_t getBit(uint32_t n) const noexcept { return (_id & ((Type)1 << n)) == 0 ? 0 : 1; } - static void initialize(); + static void initialize() noexcept; private: static Type _usedMasks[maxNumBits+1]; static Type _stripMasks[maxNumBits+1]; Type _id; - Type getUsedMask() const { + Type getUsedMask() const noexcept { return _usedMasks[getUsedBits()]; } - Type getStripMask() const { + Type getStripMask() const noexcept { return _stripMasks[getUsedBits()]; } diff --git a/documentapi/src/tests/policies/policies_test.cpp b/documentapi/src/tests/policies/policies_test.cpp index 02bd6b297d0..c80e1221eba 100644 --- a/documentapi/src/tests/policies/policies_test.cpp +++ b/documentapi/src/tests/policies/policies_test.cpp @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "testframe.h" - #include <vespa/documentapi/documentapi.h> #include <vespa/documentapi/messagebus/policies/andpolicy.h> #include <vespa/documentapi/messagebus/policies/contentpolicy.h> @@ -25,6 +24,7 @@ #include <vespa/document/update/documentupdate.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/stringfmt.h> +#include <thread> #include <vespa/log/log.h> LOG_SETUP("policies_test"); diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/CMakeLists.txt b/documentapi/src/vespa/documentapi/messagebus/policies/CMakeLists.txt index 83e1df02a24..82865ea5996 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/CMakeLists.txt +++ b/documentapi/src/vespa/documentapi/messagebus/policies/CMakeLists.txt @@ -16,5 +16,7 @@ vespa_add_library(documentapi_documentapipolicies OBJECT asyncinitializationpolicy.cpp DEPENDS ) +vespa_generate_config(documentapi_documentapipolicies ../../../../main/resources/configdefinitions/document-protocol-policies.def) +install_config_definition(../../../../main/resources/configdefinitions/document-protocol-policies.def documentapi.messagebus.protocol.document-protocol-policies.def) vespa_generate_config(documentapi_documentapipolicies ../../../../main/resources/configdefinitions/documentrouteselectorpolicy.def) install_config_definition(../../../../main/resources/configdefinitions/documentrouteselectorpolicy.def documentapi.messagebus.protocol.documentrouteselectorpolicy.def) diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index 239cf8f0f23..23127cc12b5 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -19,6 +19,7 @@ vespa_define_module( src/tests/eval/function src/tests/eval/function_speed src/tests/eval/gbdt + src/tests/eval/gen_spec src/tests/eval/inline_operation src/tests/eval/interpreted_function src/tests/eval/multiply_add @@ -64,6 +65,7 @@ vespa_define_module( src/tests/instruction/mixed_simple_join_function src/tests/instruction/pow_as_map_optimizer src/tests/instruction/remove_trivial_dimension_optimizer + src/tests/instruction/sparse_dot_product_function src/tests/instruction/sum_max_dot_product_function src/tests/instruction/vector_from_doubles_function src/tests/streamed/value diff --git a/eval/src/tests/eval/compile_cache/compile_cache_test.cpp b/eval/src/tests/eval/compile_cache/compile_cache_test.cpp index 5dea89b3a63..a0c71e2f756 100644 --- a/eval/src/tests/eval/compile_cache/compile_cache_test.cpp +++ b/eval/src/tests/eval/compile_cache/compile_cache_test.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/time_bomb.h> #include <vespa/eval/eval/llvm/compile_cache.h> #include <vespa/eval/eval/key_gen.h> #include <vespa/eval/eval/test/eval_spec.h> @@ -7,7 +8,6 @@ #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/blockingthreadstackexecutor.h> #include <vespa/vespalib/util/stringfmt.h> -#include <thread> #include <set> using namespace vespalib; diff --git a/eval/src/tests/eval/fast_value/fast_value_test.cpp b/eval/src/tests/eval/fast_value/fast_value_test.cpp index 279f17a1ead..9d29d8de660 100644 --- a/eval/src/tests/eval/fast_value/fast_value_test.cpp +++ b/eval/src/tests/eval/fast_value/fast_value_test.cpp @@ -3,7 +3,7 @@ #include <vespa/eval/eval/fast_value.hpp> #include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/value_codec.h> -#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/gen_spec.h> #include <vespa/vespalib/gtest/gtest.h> using namespace vespalib; @@ -142,29 +142,31 @@ TEST(FastValueBuilderTest, mixed_add_subspace_robustness) { } } -std::vector<Layout> layouts = { - {}, - {x(3)}, - {x(3),y(5)}, - {x(3),y(5),z(7)}, - float_cells({x(3),y(5),z(7)}), - {x({"a","b","c"})}, - {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), - {x(3),y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) +GenSpec G() { return GenSpec(); } + +const std::vector<GenSpec> layouts = { + G(), + G().idx("x", 3), + G().idx("x", 3).idx("y", 5), + G().idx("x", 3).idx("y", 5).idx("z", 7), + G().map("x", {"a","b","c"}), + G().map("x", {"a","b","c"}).map("y", {"foo","bar"}), + G().map("x", {"a","b","c"}).map("y", {"foo","bar"}).map("z", {"i","j","k","l"}), + G().idx("x", 3).map("y", {"foo", "bar"}).idx("z", 7), + G().map("x", {"a","b","c"}).idx("y", 5).map("z", {"i","j","k","l"}) }; TEST(FastValueBuilderFactoryTest, fast_values_can_be_copied) { auto factory = FastValueBuilderFactory::get(); for (const auto &layout: layouts) { - TensorSpec expect = spec(layout, N()); - std::unique_ptr<Value> value = value_from_spec(expect, factory); - std::unique_ptr<Value> copy = factory.copy(*value); - TensorSpec actual = spec_from_value(*copy); - EXPECT_EQ(actual, expect); + for (TensorSpec expect : { layout.cpy().cells_float().gen(), + layout.cpy().cells_double().gen() }) + { + std::unique_ptr<Value> value = value_from_spec(expect, factory); + std::unique_ptr<Value> copy = factory.copy(*value); + TensorSpec actual = spec_from_value(*copy); + EXPECT_EQ(actual, expect); + } } } diff --git a/eval/src/tests/eval/gen_spec/CMakeLists.txt b/eval/src/tests/eval/gen_spec/CMakeLists.txt new file mode 100644 index 00000000000..3613554f0a0 --- /dev/null +++ b/eval/src/tests/eval/gen_spec/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_gen_spec_test_app TEST + SOURCES + gen_spec_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_gen_spec_test_app COMMAND eval_gen_spec_test_app) diff --git a/eval/src/tests/eval/gen_spec/gen_spec_test.cpp b/eval/src/tests/eval/gen_spec/gen_spec_test.cpp new file mode 100644 index 00000000000..0d1a4744e42 --- /dev/null +++ b/eval/src/tests/eval/gen_spec/gen_spec_test.cpp @@ -0,0 +1,196 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/test/gen_spec.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib::eval; +using namespace vespalib::eval::test; + +//----------------------------------------------------------------------------- + +TEST(DimSpec, indexed_dimension) { + ValueType::Dimension ref("foo", 10); + DimSpec idx("foo", 10); + EXPECT_EQ(idx.type(), ref); + EXPECT_TRUE(ref.is_indexed()); + EXPECT_EQ(idx.name(), "foo"); + EXPECT_EQ(idx.size(), 10); + EXPECT_EQ(idx.label(3), TensorSpec::Label(size_t(3))); +} + +TEST(DimSpec, mapped_dimension) { + ValueType::Dimension ref("foo"); + DimSpec map("foo", {"a", "b", "c", "d"}); + EXPECT_EQ(map.type(), ref); + EXPECT_TRUE(ref.is_mapped()); + EXPECT_EQ(map.name(), "foo"); + EXPECT_EQ(map.size(), 4); + EXPECT_EQ(map.label(2), TensorSpec::Label("c")); +} + +TEST(DimSpec, simple_dictionary_creation) { + auto dict = DimSpec::make_dict(5, 1, ""); + std::vector<vespalib::string> expect = {"0", "1", "2", "3", "4"}; +} + +TEST(DimSpec, advanced_dictionary_creation) { + auto dict = DimSpec::make_dict(5, 3, "str_"); + std::vector<vespalib::string> expect = {"str_0", "str_3", "str_6", "str_9", "str_12"}; +} + +//----------------------------------------------------------------------------- + +TEST(GenSpec, default_spec) { + GenSpec spec; + EXPECT_TRUE(spec.dims().empty()); + EXPECT_EQ(spec.cells(), CellType::DOUBLE); + auto seq = spec.seq(); + for (size_t i = 0; i < 4096; ++i) { + EXPECT_EQ(seq(i), (i + 1.0)); + } +} + +//----------------------------------------------------------------------------- + +TensorSpec scalar_1 = TensorSpec("double").add({}, 1.0); +TensorSpec scalar_5 = TensorSpec("double").add({}, 5.0); + +TEST(GenSpec, scalar_double) { + EXPECT_EQ(GenSpec().gen(), scalar_1); + EXPECT_EQ(GenSpec().seq_bias(5.0).gen(), scalar_5); +} + +TEST(GenSpec, not_scalar_float_just_yet) { + EXPECT_EQ(GenSpec().cells_float().gen(), scalar_1); + EXPECT_EQ(GenSpec().cells_float().seq_bias(5.0).gen(), scalar_5); +} + +//----------------------------------------------------------------------------- + +TEST(Seq, seq_n) { + GenSpec::seq_t seq = GenSpec().seq_n().seq(); + for (size_t i = 0; i < 4096; ++i) { + EXPECT_EQ(seq(i), (i + 1.0)); + } +} + +TEST(Seq, seq_bias) { + GenSpec::seq_t seq = GenSpec().seq_bias(13.0).seq(); + for (size_t i = 0; i < 4096; ++i) { + EXPECT_EQ(seq(i), (i + 13.0)); + } +} + +//----------------------------------------------------------------------------- + +GenSpec flt() { return GenSpec().cells_float(); } +GenSpec dbl() { return GenSpec().cells_double(); } + +TEST(GenSpec, value_type) { + EXPECT_EQ(dbl().type().to_spec(), "double"); + EXPECT_EQ(flt().type().to_spec(), "double"); // NB + EXPECT_EQ(dbl().idx("x", 10).type().to_spec(), "tensor(x[10])"); + EXPECT_EQ(flt().idx("x", 10).type().to_spec(), "tensor<float>(x[10])"); + EXPECT_EQ(dbl().map("y", {}).type().to_spec(), "tensor(y{})"); + EXPECT_EQ(flt().map("y", {}).type().to_spec(), "tensor<float>(y{})"); + EXPECT_EQ(dbl().idx("x", 10).map("y", {}).type().to_spec(), "tensor(x[10],y{})"); + EXPECT_EQ(flt().idx("x", 10).map("y", {}).type().to_spec(), "tensor<float>(x[10],y{})"); + EXPECT_EQ(dbl().map("y", 3, 1).idx("x", 10).type().to_spec(), "tensor(x[10],y{})"); + EXPECT_EQ(flt().map("y", 3, 1, "str").idx("x", 10).type().to_spec(), "tensor<float>(x[10],y{})"); +} + +//----------------------------------------------------------------------------- + +TensorSpec basic_vector = TensorSpec("tensor(a[5])") + .add({{"a", 0}}, 1.0) + .add({{"a", 1}}, 2.0) + .add({{"a", 2}}, 3.0) + .add({{"a", 3}}, 4.0) + .add({{"a", 4}}, 5.0); + +TensorSpec float_vector = TensorSpec("tensor<float>(a[5])") + .add({{"a", 0}}, 1.0) + .add({{"a", 1}}, 2.0) + .add({{"a", 2}}, 3.0) + .add({{"a", 3}}, 4.0) + .add({{"a", 4}}, 5.0); + +TensorSpec custom_vector = TensorSpec("tensor(a[5])") + .add({{"a", 0}}, 5.0) + .add({{"a", 1}}, 4.0) + .add({{"a", 2}}, 3.0) + .add({{"a", 3}}, 2.0) + .add({{"a", 4}}, 1.0); + +TEST(GenSpec, generating_basic_vector) { + EXPECT_EQ(GenSpec().idx("a", 5).gen(), basic_vector); +} + +TEST(GenSpec, generating_float_vector) { + EXPECT_EQ(GenSpec().idx("a", 5).cells_float().gen(), float_vector); +} + +TEST(GenSpec, generating_custom_vector) { + GenSpec::seq_t my_seq = [](size_t idx) noexcept { return (5.0 - idx); }; + EXPECT_EQ(GenSpec().idx("a", 5).seq(my_seq).gen(), custom_vector); +} + +//----------------------------------------------------------------------------- + +TensorSpec basic_map = TensorSpec("tensor(a{})") + .add({{"a", "0"}}, 1.0) + .add({{"a", "1"}}, 2.0) + .add({{"a", "2"}}, 3.0); + +TensorSpec custom_map = TensorSpec("tensor(a{})") + .add({{"a", "s0"}}, 1.0) + .add({{"a", "s5"}}, 2.0) + .add({{"a", "s10"}}, 3.0); + +TEST(GenSpec, generating_basic_map) { + EXPECT_EQ(GenSpec().map("a", 3).gen(), basic_map); + EXPECT_EQ(GenSpec().map("a", 3, 1).gen(), basic_map); + EXPECT_EQ(GenSpec().map("a", 3, 1, "").gen(), basic_map); + EXPECT_EQ(GenSpec().map("a", {"0", "1", "2"}).gen(), basic_map); +} + +TEST(GenSpec, generating_custom_map) { + EXPECT_EQ(GenSpec().map("a", 3, 5, "s").gen(), custom_map); + EXPECT_EQ(GenSpec().map("a", {"s0", "s5", "s10"}).gen(), custom_map); +} + +//----------------------------------------------------------------------------- + +TensorSpec basic_mixed = TensorSpec("tensor(a{},b[1],c{},d[3])") + .add({{"a", "0"},{"b", 0},{"c", "0"},{"d", 0}}, 1.0) + .add({{"a", "0"},{"b", 0},{"c", "0"},{"d", 1}}, 2.0) + .add({{"a", "0"},{"b", 0},{"c", "0"},{"d", 2}}, 3.0) + .add({{"a", "1"},{"b", 0},{"c", "0"},{"d", 0}}, 4.0) + .add({{"a", "1"},{"b", 0},{"c", "0"},{"d", 1}}, 5.0) + .add({{"a", "1"},{"b", 0},{"c", "0"},{"d", 2}}, 6.0) + .add({{"a", "2"},{"b", 0},{"c", "0"},{"d", 0}}, 7.0) + .add({{"a", "2"},{"b", 0},{"c", "0"},{"d", 1}}, 8.0) + .add({{"a", "2"},{"b", 0},{"c", "0"},{"d", 2}}, 9.0); + +TensorSpec inverted_mixed = TensorSpec("tensor(a{},b[1],c{},d[3])") + .add({{"a", "0"},{"b", 0},{"c", "0"},{"d", 0}}, 1.0) + .add({{"a", "1"},{"b", 0},{"c", "0"},{"d", 0}}, 2.0) + .add({{"a", "2"},{"b", 0},{"c", "0"},{"d", 0}}, 3.0) + .add({{"a", "0"},{"b", 0},{"c", "0"},{"d", 1}}, 4.0) + .add({{"a", "1"},{"b", 0},{"c", "0"},{"d", 1}}, 5.0) + .add({{"a", "2"},{"b", 0},{"c", "0"},{"d", 1}}, 6.0) + .add({{"a", "0"},{"b", 0},{"c", "0"},{"d", 2}}, 7.0) + .add({{"a", "1"},{"b", 0},{"c", "0"},{"d", 2}}, 8.0) + .add({{"a", "2"},{"b", 0},{"c", "0"},{"d", 2}}, 9.0); + +TEST(GenSpec, generating_basic_mixed) { + EXPECT_EQ(GenSpec().map("a", 3).idx("b", 1).map("c", 1).idx("d", 3).gen(), basic_mixed); +} + +TEST(GenSpec, generating_inverted_mixed) { + EXPECT_EQ(GenSpec().idx("d", 3).map("c", 1).idx("b", 1).map("a", 3).gen(), inverted_mixed); +} + +//----------------------------------------------------------------------------- + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/eval/reference_operations/reference_operations_test.cpp b/eval/src/tests/eval/reference_operations/reference_operations_test.cpp index 0495923018e..3fcca5e34d8 100644 --- a/eval/src/tests/eval/reference_operations/reference_operations_test.cpp +++ b/eval/src/tests/eval/reference_operations/reference_operations_test.cpp @@ -1,13 +1,11 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/eval/eval/test/reference_operations.h> -#include <vespa/eval/eval/test/tensor_model.hpp> #include <vespa/vespalib/gtest/gtest.h> #include <iostream> using namespace vespalib; using namespace vespalib::eval; -using namespace vespalib::eval::test; TensorSpec dense_2d_some_cells(bool square) { return TensorSpec("tensor(a[3],d[5])") diff --git a/eval/src/tests/eval/simple_value/simple_value_test.cpp b/eval/src/tests/eval/simple_value/simple_value_test.cpp index 3a653b75172..c1301bf6b1a 100644 --- a/eval/src/tests/eval/simple_value/simple_value_test.cpp +++ b/eval/src/tests/eval/simple_value/simple_value_test.cpp @@ -5,7 +5,7 @@ #include <vespa/eval/instruction/generic_join.h> #include <vespa/eval/eval/interpreted_function.h> #include <vespa/eval/eval/test/reference_operations.h> -#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/gen_spec.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/gtest/gtest.h> @@ -23,45 +23,36 @@ using Handle = SharedStringRepo::Handle; vespalib::string as_str(string_id label) { return Handle::string_from_id(label); } -std::vector<Layout> layouts = { - {}, - {x(3)}, - {x(3),y(5)}, - {x(3),y(5),z(7)}, - float_cells({x(3),y(5),z(7)}), - {x({"a","b","c"})}, - {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), - {x(3),y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) +GenSpec G() { return GenSpec(); } + +const std::vector<GenSpec> layouts = { + G(), + G().idx("x", 3), + G().idx("x", 3).idx("y", 5), + G().idx("x", 3).idx("y", 5).idx("z", 7), + G().map("x", {"a","b","c"}), + G().map("x", {"a","b","c"}).map("y", {"foo","bar"}), + G().map("x", {"a","b","c"}).map("y", {"foo","bar"}).map("z", {"i","j","k","l"}), + G().idx("x", 3).map("y", {"foo", "bar"}).idx("z", 7), + G().map("x", {"a","b","c"}).idx("y", 5).map("z", {"i","j","k","l"}) }; -std::vector<Layout> join_layouts = { - {}, {}, - {x(5)}, {x(5)}, - {x(5)}, {y(5)}, - {x(5)}, {x(5),y(5)}, - {y(3)}, {x(2),z(3)}, - {x(3),y(5)}, {y(5),z(7)}, - float_cells({x(3),y(5)}), {y(5),z(7)}, - {x(3),y(5)}, float_cells({y(5),z(7)}), - float_cells({x(3),y(5)}), float_cells({y(5),z(7)}), - {x({"a","b","c"})}, {x({"a","b","c"})}, - {x({"a","b","c"})}, {x({"a","b"})}, - {x({"a","b","c"})}, {y({"foo","bar","baz"})}, - {x({"a","b","c"})}, {x({"a","b","c"}),y({"foo","bar","baz"})}, - {x({"a","b"}),y({"foo","bar","baz"})}, {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b"}),y({"foo","bar","baz"})}, {y({"foo","bar"}),z({"i","j","k","l"})}, - float_cells({x({"a","b"}),y({"foo","bar","baz"})}), {y({"foo","bar"}),z({"i","j","k","l"})}, - {x({"a","b"}),y({"foo","bar","baz"})}, float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), - float_cells({x({"a","b"}),y({"foo","bar","baz"})}), float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), - {x(3),y({"foo", "bar"})}, {y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5)}, {y(5),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y(5)}), {y(5),z({"i","j","k","l"})}, - {x({"a","b","c"}),y(5)}, float_cells({y(5),z({"i","j","k","l"})}), - float_cells({x({"a","b","c"}),y(5)}), float_cells({y(5),z({"i","j","k","l"})}) +const std::vector<GenSpec> join_layouts = { + G(), G(), + G().idx("x", 5), G().idx("x", 5), + G().idx("x", 5), G().idx("y", 5), + G().idx("x", 5), G().idx("x", 5).idx("y", 5), + G().idx("y", 3), G().idx("x", 2).idx("z", 3), + G().idx("x", 3).idx("y", 5), G().idx("y", 5).idx("z", 7), + G().map("x", {"a","b","c"}), G().map("x", {"a","b","c"}), + G().map("x", {"a","b","c"}), G().map("x", {"a","b"}), + G().map("x", {"a","b","c"}), G().map("y", {"foo","bar","baz"}), + G().map("x", {"a","b","c"}), G().map("x", {"a","b","c"}).map("y", {"foo","bar","baz"}), + G().map("x", {"a","b"}).map("y", {"foo","bar","baz"}), G().map("x", {"a","b","c"}).map("y", {"foo","bar"}), + G().map("x", {"a","b"}).map("y", {"foo","bar","baz"}), G().map("y", {"foo","bar"}).map("z", {"i","j","k","l"}), + G().idx("x", 3).map("y", {"foo", "bar"}), G().map("y", {"foo","bar"}).idx("z", 7), + G().map("x", {"a","b","c"}).idx("y", 5), G().idx("y", 5).map("z", {"i","j","k","l"}) + }; TensorSpec simple_value_join(const TensorSpec &a, const TensorSpec &b, join_fun_t function) { @@ -76,20 +67,26 @@ TensorSpec simple_value_join(const TensorSpec &a, const TensorSpec &b, join_fun_ TEST(SimpleValueTest, simple_values_can_be_converted_from_and_to_tensor_spec) { for (const auto &layout: layouts) { - TensorSpec expect = spec(layout, N()); - std::unique_ptr<Value> value = value_from_spec(expect, SimpleValueBuilderFactory::get()); - TensorSpec actual = spec_from_value(*value); - EXPECT_EQ(actual, expect); + for (TensorSpec expect : { layout.cpy().cells_float().gen(), + layout.cpy().cells_double().gen() }) + { + std::unique_ptr<Value> value = value_from_spec(expect, SimpleValueBuilderFactory::get()); + TensorSpec actual = spec_from_value(*value); + EXPECT_EQ(actual, expect); + } } } TEST(SimpleValueTest, simple_values_can_be_copied) { for (const auto &layout: layouts) { - TensorSpec expect = spec(layout, N()); - std::unique_ptr<Value> value = value_from_spec(expect, SimpleValueBuilderFactory::get()); - std::unique_ptr<Value> copy = SimpleValueBuilderFactory::get().copy(*value); - TensorSpec actual = spec_from_value(*copy); - EXPECT_EQ(actual, expect); + for (TensorSpec expect : { layout.cpy().cells_float().gen(), + layout.cpy().cells_double().gen() }) + { + std::unique_ptr<Value> value = value_from_spec(expect, SimpleValueBuilderFactory::get()); + std::unique_ptr<Value> copy = SimpleValueBuilderFactory::get().copy(*value); + TensorSpec actual = spec_from_value(*copy); + EXPECT_EQ(actual, expect); + } } } @@ -126,16 +123,26 @@ TEST(SimpleValueTest, simple_value_can_be_built_and_inspected) { EXPECT_EQ(result["bb"], 3); } +GenSpec::seq_t N_16ths = [] (size_t i) noexcept { return (i + 1.0) / 16.0; }; + TEST(SimpleValueTest, new_generic_join_works_for_simple_values) { ASSERT_TRUE((join_layouts.size() % 2) == 0); for (size_t i = 0; i < join_layouts.size(); i += 2) { - TensorSpec lhs = spec(join_layouts[i], Div16(N())); - TensorSpec rhs = spec(join_layouts[i + 1], Div16(N())); - for (auto fun: {operation::Add::f, operation::Sub::f, operation::Mul::f, operation::Div::f}) { - SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); - auto expect = ReferenceOperations::join(lhs, rhs, fun); - auto actual = simple_value_join(lhs, rhs, fun); - EXPECT_EQ(actual, expect); + const auto l = join_layouts[i].cpy().seq(N_16ths); + const auto r = join_layouts[i + 1].cpy().seq(N_16ths); + for (TensorSpec lhs : { l.cpy().cells_float().gen(), + l.cpy().cells_double().gen() }) + { + for (TensorSpec rhs : { r.cpy().cells_float().gen(), + r.cpy().cells_double().gen() }) + { + for (auto fun: {operation::Add::f, operation::Sub::f, operation::Mul::f, operation::Div::f}) { + SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); + auto expect = ReferenceOperations::join(lhs, rhs, fun); + auto actual = simple_value_join(lhs, rhs, fun); + EXPECT_EQ(actual, expect); + } + } } } } diff --git a/eval/src/tests/eval/tensor_lambda/tensor_lambda_test.cpp b/eval/src/tests/eval/tensor_lambda/tensor_lambda_test.cpp index 23bd16cb721..dd21b663fa9 100644 --- a/eval/src/tests/eval/tensor_lambda/tensor_lambda_test.cpp +++ b/eval/src/tests/eval/tensor_lambda/tensor_lambda_test.cpp @@ -8,7 +8,7 @@ #include <vespa/eval/instruction/dense_cell_range_function.h> #include <vespa/eval/instruction/dense_lambda_peek_function.h> #include <vespa/eval/instruction/fast_rename_optimizer.h> -#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/gen_spec.h> #include <vespa/eval/eval/test/eval_fixture.h> #include <vespa/eval/eval/tensor_nodes.h> @@ -25,15 +25,15 @@ const ValueBuilderFactory &prod_factory = FastValueBuilderFactory::get(); EvalFixture::ParamRepo make_params() { return EvalFixture::ParamRepo() - .add("a", spec(1)) - .add("b", spec(2)) - .add("x3", spec({x(3)}, N())) - .add("x3f", spec(float_cells({x(3)}), N())) - .add("x3m", spec({x({"0", "1", "2"})}, N())) - .add("x3y5", spec({x(3), y(5)}, N())) - .add("x3y5f", spec(float_cells({x(3), y(5)}), N())) - .add("x15", spec({x(15)}, N())) - .add("x15f", spec(float_cells({x(15)}), N())); + .add("a", GenSpec().seq_bias(1).gen()) + .add("b", GenSpec().seq_bias(2).gen()) + .add("x3", GenSpec().idx("x", 3).gen()) + .add("x3f", GenSpec().idx("x", 3).cells_float().gen()) + .add("x3m", GenSpec().map("x", 3).gen()) + .add("x3y5", GenSpec().idx("x", 3).idx("y", 5).gen()) + .add("x3y5f", GenSpec().idx("x", 3).idx("y", 5).cells_float().gen()) + .add("x15", GenSpec().idx("x", 15).gen()) + .add("x15f", GenSpec().idx("x", 15).cells_float().gen()); } EvalFixture::ParamRepo param_repo = make_params(); diff --git a/eval/src/tests/eval/value_codec/value_codec_test.cpp b/eval/src/tests/eval/value_codec/value_codec_test.cpp index 2b03cffe730..110b58c27de 100644 --- a/eval/src/tests/eval/value_codec/value_codec_test.cpp +++ b/eval/src/tests/eval/value_codec/value_codec_test.cpp @@ -2,7 +2,7 @@ #include <iostream> #include <vespa/eval/eval/simple_value.h> -#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/gen_spec.h> #include <vespa/eval/eval/value_codec.h> #include <vespa/vespalib/data/memory.h> #include <vespa/vespalib/gtest/gtest.h> @@ -15,28 +15,30 @@ using namespace vespalib::eval::test; const ValueBuilderFactory &factory = SimpleValueBuilderFactory::get(); -std::vector<Layout> layouts = { - {}, - {x(3)}, - {x(3),y(5)}, - {x(3),y(5),z(7)}, - float_cells({x(3),y(5),z(7)}), - {x({"a","b","c"})}, - {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), - {x(3),y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) +GenSpec G() { return GenSpec(); } + +const std::vector<GenSpec> layouts = { + G(), + G().idx("x", 3), + G().idx("x", 3).idx("y", 5), + G().idx("x", 3).idx("y", 5).idx("z", 7), + G().map("x", {"a","b","c"}), + G().map("x", {"a","b","c"}).map("y", {"foo","bar"}), + G().map("x", {"a","b","c"}).map("y", {"foo","bar"}).map("z", {"i","j","k","l"}), + G().idx("x", 3).map("y", {"foo", "bar"}).idx("z", 7), + G().map("x", {"a","b","c"}).idx("y", 5).map("z", {"i","j","k","l"}) }; TEST(ValueCodecTest, simple_values_can_be_converted_from_and_to_tensor_spec) { for (const auto &layout: layouts) { - TensorSpec expect = spec(layout, N()); - std::unique_ptr<Value> value = value_from_spec(expect, factory); - TensorSpec actual = spec_from_value(*value); - EXPECT_EQ(actual, expect); + for (TensorSpec expect : { layout.cpy().cells_float().gen(), + layout.cpy().cells_double().gen() }) + { + std::unique_ptr<Value> value = value_from_spec(expect, factory); + TensorSpec actual = spec_from_value(*value); + EXPECT_EQ(actual, expect); + } } } @@ -66,8 +68,8 @@ TEST(ValueCodecTest, simple_values_can_be_built_using_tensor_spec) { .add({{"w", "yyy"}, {"x", 1}, {"y", "yyy"}, {"z", 0}}, 0.0) .add({{"w", "yyy"}, {"x", 1}, {"y", "yyy"}, {"z", 1}}, 4.0); Value::UP full_tensor = value_from_spec(full_spec, factory); - EXPECT_EQUAL(full_spec, spec_from_value(*tensor)); - EXPECT_EQUAL(full_spec, spec_from_value(*full_tensor)); + EXPECT_EQ(full_spec, spec_from_value(*tensor)); + EXPECT_EQ(full_spec, spec_from_value(*full_tensor)); }; //----------------------------------------------------------------------------- @@ -333,11 +335,11 @@ TEST(ValueCodecTest, bad_sparse_tensors_are_caught) { bad.encode_default(data_default); bad.encode_with_double(data_double); bad.encode_with_float(data_float); - EXPECT_EXCEPTION(decode_value(data_default, factory), vespalib::IllegalStateException, + VESPA_EXPECT_EXCEPTION(decode_value(data_default, factory), vespalib::IllegalStateException, "serialized input claims 12345678 blocks of size 1*8, but only"); - EXPECT_EXCEPTION(decode_value(data_double, factory), vespalib::IllegalStateException, + VESPA_EXPECT_EXCEPTION(decode_value(data_double, factory), vespalib::IllegalStateException, "serialized input claims 12345678 blocks of size 1*8, but only"); - EXPECT_EXCEPTION(decode_value(data_float, factory), vespalib::IllegalStateException, + VESPA_EXPECT_EXCEPTION(decode_value(data_float, factory), vespalib::IllegalStateException, "serialized input claims 12345678 blocks of size 1*4, but only"); } @@ -386,11 +388,11 @@ TEST(ValueCodecTest, bad_dense_tensors_are_caught) { bad.encode_default(data_default); bad.encode_with_double(data_double); bad.encode_with_float(data_float); - EXPECT_EXCEPTION(decode_value(data_default, factory), vespalib::IllegalStateException, + VESPA_EXPECT_EXCEPTION(decode_value(data_default, factory), vespalib::IllegalStateException, "serialized input claims 1 blocks of size 60000*8, but only"); - EXPECT_EXCEPTION(decode_value(data_double, factory), vespalib::IllegalStateException, + VESPA_EXPECT_EXCEPTION(decode_value(data_double, factory), vespalib::IllegalStateException, "serialized input claims 1 blocks of size 60000*8, but only"); - EXPECT_EXCEPTION(decode_value(data_float, factory), vespalib::IllegalStateException, + VESPA_EXPECT_EXCEPTION(decode_value(data_float, factory), vespalib::IllegalStateException, "serialized input claims 1 blocks of size 60000*4, but only"); } diff --git a/eval/src/tests/instruction/add_trivial_dimension_optimizer/add_trivial_dimension_optimizer_test.cpp b/eval/src/tests/instruction/add_trivial_dimension_optimizer/add_trivial_dimension_optimizer_test.cpp index 35195522adc..d2dccfde2fd 100644 --- a/eval/src/tests/instruction/add_trivial_dimension_optimizer/add_trivial_dimension_optimizer_test.cpp +++ b/eval/src/tests/instruction/add_trivial_dimension_optimizer/add_trivial_dimension_optimizer_test.cpp @@ -5,7 +5,7 @@ #include <vespa/eval/eval/tensor_function.h> #include <vespa/eval/instruction/replace_type_function.h> #include <vespa/eval/instruction/fast_rename_optimizer.h> -#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/gen_spec.h> #include <vespa/eval/eval/test/eval_fixture.h> #include <vespa/vespalib/util/stringfmt.h> @@ -20,11 +20,11 @@ const ValueBuilderFactory &prod_factory = FastValueBuilderFactory::get(); EvalFixture::ParamRepo make_params() { return EvalFixture::ParamRepo() - .add("x5", spec({x(5)}, N())) - .add("x5f", spec(float_cells({x(5)}), N())) - .add("x5y1", spec({x(5),y(1)}, N())) - .add("y1z1", spec({y(1),z(1)}, N())) - .add("x_m", spec({x({"a"})}, N())); + .add("x5", GenSpec().idx("x", 5).gen()) + .add("x5f", GenSpec().idx("x", 5).cells_float().gen()) + .add("x5y1", GenSpec().idx("x", 5).idx("y", 1).gen()) + .add("y1z1", GenSpec().idx("y", 5).idx("z", 1).gen()) + .add("x_m", GenSpec().map("x", {"a"}).gen()); } EvalFixture::ParamRepo param_repo = make_params(); diff --git a/eval/src/tests/instruction/dense_replace_type_function/dense_replace_type_function_test.cpp b/eval/src/tests/instruction/dense_replace_type_function/dense_replace_type_function_test.cpp index 988ca79a04a..5dcdbc5bab8 100644 --- a/eval/src/tests/instruction/dense_replace_type_function/dense_replace_type_function_test.cpp +++ b/eval/src/tests/instruction/dense_replace_type_function/dense_replace_type_function_test.cpp @@ -5,7 +5,7 @@ #include <vespa/eval/eval/value_codec.h> #include <vespa/eval/eval/interpreted_function.h> #include <vespa/eval/instruction/replace_type_function.h> -#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/gen_spec.h> using namespace vespalib::eval::tensor_function; using namespace vespalib::eval::test; @@ -33,7 +33,7 @@ struct Fixture { std::vector<TensorFunction::Child::CREF> children; InterpretedFunction::State state; Fixture() - : my_value(value_from_spec(spec({x(10)}, N()), prod_factory)), + : my_value(value_from_spec(GenSpec().idx("x", 10).gen(), prod_factory)), new_type(ValueType::from_spec("tensor(x[5],y[2])")), mock_child(my_value->type()), my_fun(new_type, mock_child), diff --git a/eval/src/tests/instruction/fast_rename_optimizer/fast_rename_optimizer_test.cpp b/eval/src/tests/instruction/fast_rename_optimizer/fast_rename_optimizer_test.cpp index dc90a5e54a1..e915a396ae7 100644 --- a/eval/src/tests/instruction/fast_rename_optimizer/fast_rename_optimizer_test.cpp +++ b/eval/src/tests/instruction/fast_rename_optimizer/fast_rename_optimizer_test.cpp @@ -4,7 +4,7 @@ #include <vespa/eval/eval/tensor_function.h> #include <vespa/eval/instruction/replace_type_function.h> #include <vespa/eval/instruction/fast_rename_optimizer.h> -#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/gen_spec.h> #include <vespa/eval/eval/test/eval_fixture.h> #include <vespa/vespalib/util/stringfmt.h> @@ -19,13 +19,13 @@ const ValueBuilderFactory &prod_factory = FastValueBuilderFactory::get(); EvalFixture::ParamRepo make_params() { return EvalFixture::ParamRepo() - .add("x5", spec({x(5)}, N())) - .add("x5f", spec(float_cells({x(5)}), N())) - .add("x_m", spec({x({"a", "b", "c"})}, N())) - .add("xy_mm", spec({x({"a", "b", "c"}),y({"d","e"})}, N())) - .add("x5y3z_m", spec({x(5),y(3),z({"a","b"})}, N())) - .add("x5yz_m", spec({x(5),y({"a","b"}),z({"d","e"})}, N())) - .add("x5y3", spec({x(5),y(3)}, N())); + .add("x5", GenSpec().idx("x", 5).gen()) + .add("x5f", GenSpec().idx("x", 5).cells_float().gen()) + .add("x_m", GenSpec().map("x", {"a", "b", "c"}).gen()) + .add("xy_mm", GenSpec().map("x", {"a", "b", "c"}).map("y", {"d","e"}).gen()) + .add("x5y3z_m", GenSpec().idx("x", 5).idx("y", 3).map("z", {"a","b"}).gen()) + .add("x5yz_m", GenSpec().idx("x", 5).map("y", {"a","b"}).map("z", {"d","e"}).gen()) + .add("x5y3", GenSpec().idx("x", 5).idx("y", 3).gen()); } EvalFixture::ParamRepo param_repo = make_params(); diff --git a/eval/src/tests/instruction/generic_concat/generic_concat_test.cpp b/eval/src/tests/instruction/generic_concat/generic_concat_test.cpp index bc8ea84744f..17e012b8e33 100644 --- a/eval/src/tests/instruction/generic_concat/generic_concat_test.cpp +++ b/eval/src/tests/instruction/generic_concat/generic_concat_test.cpp @@ -7,7 +7,7 @@ #include <vespa/eval/instruction/generic_concat.h> #include <vespa/eval/eval/interpreted_function.h> #include <vespa/eval/eval/test/reference_operations.h> -#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/gen_spec.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/gtest/gtest.h> @@ -18,53 +18,48 @@ using namespace vespalib::eval::test; using vespalib::make_string_short::fmt; -std::vector<Layout> concat_layouts = { - {}, {}, - {}, {y(5)}, - float_cells({y(5)}), {}, - {}, float_cells({y(5)}), - {y(5)}, {}, - {y(2)}, {y(3)}, - {y(2)}, {x(3)}, - {x(2)}, {z(3)}, - {x(2),y(3)}, {x(2),y(3)}, - {x(2),y(3)}, {x(2),y(4)}, - {y(3),z(5)}, {y(3),z(5)}, - {y(3),z(5)}, {y(4),z(5)}, - {x(2),y(3),z(5)}, {x(2),y(3),z(5)}, - {x(2),y(3),z(5)}, {x(2),y(4),z(5)}, - {x(2),y(3),z({"a","b"})}, {x(2),y(3),z({"b","c"})}, - {x(2),y(3),z({"a","b"})}, {x(2),y(4),z({"b","c"})}, - {y(5)}, {y(2),x(5)}, - {x(3)}, {y(2),z(3)}, - {y(2)}, {y(3),x(5),z(2)}, - {y(2),x(5),z(2)}, {y(3),x(5),z(2)}, - {y(3),x(5)}, {x(5),z(7)}, - float_cells({y(3),x(5)}), {x(5),z(7)}, - float_cells({y(3),x(5)}), {}, - {y(3),x(5)}, float_cells({x(5),z(7)}), - float_cells({y(3),x(5)}), float_cells({x(5),z(7)}), - {x({"a","b","c"})}, {x({"a","b","c"})}, - {x({"a","b","c"})}, {x({"a","b"})}, - {x({"a","b","c"})}, {x({"b","c","d"})}, - float_cells({x({"a","b","c"})}), {x({"b","c","d"})}, - {x({"a","b","c"})}, float_cells({x({"b","c","d"})}), - float_cells({x({"a","b","c"})}), float_cells({z({"foo","bar","baz"})}), - {x({"a","b","c"})}, {x({"a","b","c"}),z({"foo","bar","baz"})}, - {x({"a","b"}),z({"foo","bar","baz"})}, {x({"a","b","c"}),z({"foo","bar"})}, - {x({"a","b","c"}),y(3)}, {y(2)}, - {x({"a","b","c"}),y(3)}, {z(5)}, - {x({"a","b","c"}),y(3)}, {y(2),z(5)}, - {x({"a","b","c"}),y(3)}, {y(2)}, - {x({"a","b","c"}),y(3),z(5)}, {z(5)}, - {y(2)}, {x({"a","b","c"}),y(3)}, - {z(5)}, {x({"a","b","c"}),y(3)}, - {y(2),z(5)}, {x({"a","b","c"}),y(3)}, - {y(2)}, {x({"a","b","c"}),y(3)}, - {z(5)}, {x({"a","b","c"}),y(3),z(5)}, - {y(2),z(5)}, {x({"a","b","c"}),y(3),z(5)}, - {y(2),x({"a","b","c"})}, {y(3),x({"b","c","d"})}, - {y(2),x({"a","b"})}, {y(3),z({"c","d"})} +GenSpec G() { return GenSpec(); } + +GenSpec::seq_t N_16ths = [] (size_t i) noexcept { return (i + 1.0) / 16.0; }; + +const std::vector<GenSpec> concat_layouts = { + G(), G(), + G(), G().idx("y", 5), + G().idx("y", 5), G(), + G().idx("y", 2), G().idx("y", 3), + G().idx("y", 2), G().idx("x", 3), + G().idx("x", 2), G().idx("z", 3), + G().idx("x", 2).idx("y", 3), G().idx("x", 2).idx("y", 3), + G().idx("x", 2).idx("y", 3), G().idx("x", 2).idx("y", 4), + G().idx("y", 3).idx("z", 5), G().idx("y", 3).idx("z", 5), + G().idx("y", 3).idx("z", 5), G().idx("y", 4).idx("z", 5), + G().idx("x", 2).idx("y", 3).idx("z", 5), G().idx("x", 2).idx("y", 3).idx("z", 5), + G().idx("x", 2).idx("y", 3).idx("z", 5), G().idx("x", 2).idx("y", 4).idx("z", 5), + G().idx("x", 2).idx("y", 3).map("z", {"a","b"}), G().idx("x", 2).idx("y", 3).map("z", {"b","c"}), + G().idx("x", 2).idx("y", 3).map("z", {"a","b"}), G().idx("x", 2).idx("y", 4).map("z", {"b","c"}), + G().idx("y", 5), G().idx("x", 5).idx("y", 2), + G().idx("x", 3), G().idx("y", 2).idx("z", 3), + G().idx("y", 2), G().idx("x", 5).idx("y", 3).idx("z", 2), + G().idx("x", 5).idx("y", 2).idx("z", 2), G().idx("x", 5).idx("y", 3).idx("z", 2), + G().idx("x", 5).idx("y", 3), G().idx("x", 5).idx("z", 7), + G().map("x", {"a","b","c"}), G().map("x", {"a","b","c"}), + G().map("x", {"a","b","c"}), G().map("x", {"a","b"}), + G().map("x", {"a","b","c"}), G().map("x", {"b","c","d"}), + G().map("x", {"a","b","c"}), G().map("x", {"a","b","c"}).map("z", {"foo","bar","baz"}), + G().map("x", {"a","b"}).map("z", {"foo","bar","baz"}), G().map("x", {"a","b","c"}).map("z", {"foo","bar"}), + G().map("x", {"a","b","c"}).idx("y", 3), G().idx("y", 2), + G().map("x", {"a","b","c"}).idx("y", 3), G().idx("z", 5), + G().map("x", {"a","b","c"}).idx("y", 3), G().idx("y", 2).idx("z", 5), + G().map("x", {"a","b","c"}).idx("y", 3), G().idx("y", 2), + G().map("x", {"a","b","c"}).idx("y", 3).idx("z", 5), G().idx("z", 5), + G().idx("y", 2), G().map("x", {"a","b","c"}).idx("y", 3), + G().idx("z", 5), G().map("x", {"a","b","c"}).idx("y", 3), + G().idx("y", 2).idx("z", 5), G().map("x", {"a","b","c"}).idx("y", 3), + G().idx("y", 2), G().map("x", {"a","b","c"}).idx("y", 3), + G().idx("z", 5), G().map("x", {"a","b","c"}).idx("y", 3).idx("z", 5), + G().idx("y", 2).idx("z", 5), G().map("x", {"a","b","c"}).idx("y", 3).idx("z", 5), + G().map("x", {"a","b","c"}).idx("y", 2), G().map("x", {"b","c","d"}).idx("y", 3), + G().map("x", {"a","b"}).idx("y", 2), G().idx("y", 3).map("z", {"c","d"}) }; TensorSpec perform_generic_concat(const TensorSpec &a, const TensorSpec &b, @@ -81,12 +76,20 @@ TensorSpec perform_generic_concat(const TensorSpec &a, const TensorSpec &b, void test_generic_concat_with(const ValueBuilderFactory &factory) { ASSERT_TRUE((concat_layouts.size() % 2) == 0); for (size_t i = 0; i < concat_layouts.size(); i += 2) { - const TensorSpec lhs = spec(concat_layouts[i], N()); - const TensorSpec rhs = spec(concat_layouts[i + 1], Div16(N())); - SCOPED_TRACE(fmt("\n===\nin LHS: %s\nin RHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); - auto actual = perform_generic_concat(lhs, rhs, "y", factory); - auto expect = ReferenceOperations::concat(lhs, rhs, "y"); - EXPECT_EQ(actual, expect); + const auto l = concat_layouts[i]; + const auto r = concat_layouts[i+1].cpy().seq(N_16ths); + for (TensorSpec lhs : { l.cpy().cells_float().gen(), + l.cpy().cells_double().gen() }) + { + for (TensorSpec rhs : { r.cpy().cells_float().gen(), + r.cpy().cells_double().gen() }) + { + SCOPED_TRACE(fmt("\n===\nin LHS: %s\nin RHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); + auto actual = perform_generic_concat(lhs, rhs, "y", factory); + auto expect = ReferenceOperations::concat(lhs, rhs, "y"); + EXPECT_EQ(actual, expect); + } + } } } diff --git a/eval/src/tests/instruction/generic_create/generic_create_test.cpp b/eval/src/tests/instruction/generic_create/generic_create_test.cpp index 00af75e4d83..fcf4618d592 100644 --- a/eval/src/tests/instruction/generic_create/generic_create_test.cpp +++ b/eval/src/tests/instruction/generic_create/generic_create_test.cpp @@ -6,7 +6,7 @@ #include <vespa/eval/instruction/generic_create.h> #include <vespa/eval/eval/interpreted_function.h> #include <vespa/eval/eval/test/reference_operations.h> -#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/gen_spec.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/gtest/gtest.h> #include <stdlib.h> @@ -19,18 +19,17 @@ using namespace vespalib::eval::test; using vespalib::make_string_short::fmt; -std::vector<Layout> create_layouts = { - {x(3)}, - {x(3),y(5)}, - {x(3),y(5),z(7)}, - float_cells({x(3),y(5),z(7)}), - {x({"a","b","c"})}, - {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), - {x(3),y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) +GenSpec G() { return GenSpec(); } + +const std::vector<GenSpec> create_layouts = { + G().idx("x", 3), + G().idx("x", 3).idx("y", 5), + G().idx("x", 3).idx("y", 5).idx("z", 7), + G().map("x", {"a","b","c"}), + G().map("x", {"a","b","c"}).map("y", {"foo","bar"}), + G().map("x", {"a","b","c"}).map("y", {"foo","bar"}).map("z", {"i","j","k","l"}), + G().idx("x", 3).map("y", {"foo", "bar"}).idx("z", 7), + G().map("x", {"a","b","c"}).idx("y", 5).map("z", {"i","j","k","l"}) }; TensorSpec remove_each(const TensorSpec &a, size_t n) { @@ -91,16 +90,19 @@ TensorSpec perform_generic_create(const TensorSpec &a, const ValueBuilderFactory } void test_generic_create_with(const ValueBuilderFactory &factory) { - for (const auto & layout : create_layouts) { - TensorSpec full = spec(layout, N()); - auto actual = perform_generic_create(full, factory); - auto expect = reference_create(full).normalize(); - EXPECT_EQ(actual, expect); - for (size_t n : {2, 3, 4, 5}) { - TensorSpec partial = remove_each(full, n); - actual = perform_generic_create(partial, factory); - expect = reference_create(partial).normalize(); + for (const auto &layout : create_layouts) { + for (TensorSpec full : { layout.cpy().cells_float().gen(), + layout.cpy().cells_double().gen() }) + { + auto actual = perform_generic_create(full, factory); + auto expect = reference_create(full).normalize(); EXPECT_EQ(actual, expect); + for (size_t n : {2, 3, 4, 5}) { + TensorSpec partial = remove_each(full, n); + actual = perform_generic_create(partial, factory); + expect = reference_create(partial).normalize(); + EXPECT_EQ(actual, expect); + } } } } diff --git a/eval/src/tests/instruction/generic_join/generic_join_test.cpp b/eval/src/tests/instruction/generic_join/generic_join_test.cpp index 8eca3cad763..f724cdf1024 100644 --- a/eval/src/tests/instruction/generic_join/generic_join_test.cpp +++ b/eval/src/tests/instruction/generic_join/generic_join_test.cpp @@ -6,7 +6,7 @@ #include <vespa/eval/instruction/generic_join.h> #include <vespa/eval/eval/interpreted_function.h> #include <vespa/eval/eval/test/reference_operations.h> -#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/gen_spec.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/gtest/gtest.h> @@ -17,33 +17,25 @@ using namespace vespalib::eval::test; using vespalib::make_string_short::fmt; -std::vector<Layout> join_layouts = { - {}, {}, - {x(5)}, {x(5)}, - {x(5)}, {y(5)}, - {x(5)}, {x(5),y(5)}, - {y(3)}, {x(2),z(3)}, - {x(3),y(5)}, {y(5),z(7)}, - float_cells({x(3),y(5)}), {y(5),z(7)}, - {x(3),y(5)}, float_cells({y(5),z(7)}), - float_cells({x(3),y(5)}), float_cells({y(5),z(7)}), - {x({"a","b","c"})}, {x({"a","b","c"})}, - {x({"a","b","c"})}, {x({"a","b"})}, - {x({"a","b","c"})}, {y({"foo","bar","baz"})}, - {x({"a","b","c"})}, {x({"a","b","c"}),y({"foo","bar","baz"})}, - {x({"a","b"}),y({"foo","bar","baz"})}, {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b"}),y({"foo","bar","baz"})}, {y({"foo","bar"}),z({"i","j","k","l"})}, - float_cells({x({"a","b"}),y({"foo","bar","baz"})}), {y({"foo","bar"}),z({"i","j","k","l"})}, - {x({"a","b"}),y({"foo","bar","baz"})}, float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), - float_cells({x({"a","b"}),y({"foo","bar","baz"})}), float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), - {x(3),y({"foo", "bar"})}, {y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5)}, {y(5),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y(5)}), {y(5),z({"i","j","k","l"})}, - {x({"a","b","c"}),y(5)}, float_cells({y(5),z({"i","j","k","l"})}), - float_cells({x({"a","b","c"}),y(5)}), float_cells({y(5),z({"i","j","k","l"})}), - {x({"a","b","c"}),y(5)}, float_cells({y(5)}), - {y(5)}, float_cells({x({"a","b","c"}),y(5)}), - {x({}),y(5)}, float_cells({y(5)}) +GenSpec::seq_t N_16ths = [] (size_t i) noexcept { return (i + 1.0) / 16.0; }; + +GenSpec G() { return GenSpec().seq(N_16ths); } + +const std::vector<GenSpec> join_layouts = { + G(), G(), + G().idx("x", 5), G().idx("x", 5), + G().idx("x", 5), G().idx("y", 5), + G().idx("x", 5), G().idx("x", 5).idx("y", 5), + G().idx("y", 3), G().idx("x", 2).idx("z", 3), + G().idx("x", 3).idx("y", 5), G().idx("y", 5).idx("z", 7), + G().map("x", {"a","b","c"}), G().map("x", {"a","b","c"}), + G().map("x", {"a","b","c"}), G().map("x", {"a","b"}), + G().map("x", {"a","b","c"}), G().map("y", {"foo","bar","baz"}), + G().map("x", {"a","b","c"}), G().map("x", {"a","b","c"}).map("y", {"foo","bar","baz"}), + G().map("x", {"a","b"}).map("y", {"foo","bar","baz"}), G().map("x", {"a","b","c"}).map("y", {"foo","bar"}), + G().map("x", {"a","b"}).map("y", {"foo","bar","baz"}), G().map("y", {"foo","bar"}).map("z", {"i","j","k","l"}), + G().idx("x", 3).map("y", {"foo", "bar"}), G().map("y", {"foo", "bar"}).idx("z", 7), + G().map("x", {"a","b","c"}).idx("y", 5), G().idx("y", 5).map("z", {"i","j","k","l"}) }; bool join_address(const TensorSpec::Address &a, const TensorSpec::Address &b, TensorSpec::Address &addr) { @@ -113,15 +105,23 @@ TEST(GenericJoinTest, dense_join_plan_can_be_executed) { TEST(GenericJoinTest, generic_join_works_for_simple_and_fast_values) { ASSERT_TRUE((join_layouts.size() % 2) == 0); for (size_t i = 0; i < join_layouts.size(); i += 2) { - TensorSpec lhs = spec(join_layouts[i], Div16(N())); - TensorSpec rhs = spec(join_layouts[i + 1], Div16(N())); - for (auto fun: {operation::Add::f, operation::Sub::f, operation::Mul::f, operation::Div::f}) { - SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); - auto expect = ReferenceOperations::join(lhs, rhs, fun); - auto simple = perform_generic_join(lhs, rhs, fun, SimpleValueBuilderFactory::get()); - auto fast = perform_generic_join(lhs, rhs, fun, FastValueBuilderFactory::get()); - EXPECT_EQ(simple, expect); - EXPECT_EQ(fast, expect); + const auto &l = join_layouts[i]; + const auto &r = join_layouts[i+1]; + for (TensorSpec lhs : { l.cpy().cells_float().gen(), + l.cpy().cells_double().gen() }) + { + for (TensorSpec rhs : { r.cpy().cells_float().gen(), + r.cpy().cells_double().gen() }) + { + for (auto fun: {operation::Add::f, operation::Sub::f, operation::Mul::f, operation::Div::f}) { + SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); + auto expect = ReferenceOperations::join(lhs, rhs, fun); + auto simple = perform_generic_join(lhs, rhs, fun, SimpleValueBuilderFactory::get()); + auto fast = perform_generic_join(lhs, rhs, fun, FastValueBuilderFactory::get()); + EXPECT_EQ(simple, expect); + EXPECT_EQ(fast, expect); + } + } } } } diff --git a/eval/src/tests/instruction/generic_map/generic_map_test.cpp b/eval/src/tests/instruction/generic_map/generic_map_test.cpp index 687b6aa60ac..8e39fa68072 100644 --- a/eval/src/tests/instruction/generic_map/generic_map_test.cpp +++ b/eval/src/tests/instruction/generic_map/generic_map_test.cpp @@ -6,7 +6,7 @@ #include <vespa/eval/instruction/generic_map.h> #include <vespa/eval/eval/interpreted_function.h> #include <vespa/eval/eval/test/reference_operations.h> -#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/gen_spec.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/gtest/gtest.h> @@ -17,18 +17,20 @@ using namespace vespalib::eval::test; using vespalib::make_string_short::fmt; -std::vector<Layout> map_layouts = { - {x(3)}, - {x(3),y(5)}, - {x(3),y(5),z(7)}, - float_cells({x(3),y(5),z(7)}), - {x({"a","b","c"})}, - {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), - {x(3),y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) +GenSpec::seq_t N_16ths = [] (size_t i) noexcept { return (i + 1.0) / 16.0; }; + +GenSpec G() { return GenSpec().seq(N_16ths); } + +const std::vector<GenSpec> map_layouts = { + G(), + G().idx("x", 3), + G().idx("x", 3).idx("y", 5), + G().idx("x", 3).idx("y", 5).idx("z", 7), + G().map("x", {"a","b","c"}), + G().map("x", {"a","b","c"}).map("y", {"foo","bar"}), + G().map("x", {"a","b","c"}).map("y", {"foo","bar"}).map("z", {"i","j","k","l"}), + G().idx("x", 3).map("y", {"foo", "bar"}).idx("z", 7), + G().map("x", {"a","b","c"}).idx("y", 5).map("z", {"i","j","k","l"}) }; TensorSpec perform_generic_map(const TensorSpec &a, map_fun_t func, const ValueBuilderFactory &factory) @@ -40,14 +42,16 @@ TensorSpec perform_generic_map(const TensorSpec &a, map_fun_t func, const ValueB } void test_generic_map_with(const ValueBuilderFactory &factory) { - for (const auto & layout : map_layouts) { - TensorSpec lhs = spec(layout, Div16(N())); - ValueType lhs_type = ValueType::from_spec(lhs.type()); - for (auto func : {operation::Floor::f, operation::Fabs::f, operation::Square::f, operation::Inv::f}) { - SCOPED_TRACE(fmt("\n===\nLHS: %s\n===\n", lhs.to_string().c_str())); - auto expect = ReferenceOperations::map(lhs, func); - auto actual = perform_generic_map(lhs, func, factory); - EXPECT_EQ(actual, expect); + for (const auto &layout : map_layouts) { + for (TensorSpec lhs : { layout.cpy().cells_float().gen(), + layout.cpy().cells_double().gen() }) + { + for (auto func : {operation::Floor::f, operation::Fabs::f, operation::Square::f, operation::Inv::f}) { + SCOPED_TRACE(fmt("\n===\nLHS: %s\n===\n", lhs.to_string().c_str())); + auto expect = ReferenceOperations::map(lhs, func); + auto actual = perform_generic_map(lhs, func, factory); + EXPECT_EQ(actual, expect); + } } } } diff --git a/eval/src/tests/instruction/generic_merge/generic_merge_test.cpp b/eval/src/tests/instruction/generic_merge/generic_merge_test.cpp index 60a27e6f6e9..d5f7bc071f6 100644 --- a/eval/src/tests/instruction/generic_merge/generic_merge_test.cpp +++ b/eval/src/tests/instruction/generic_merge/generic_merge_test.cpp @@ -6,7 +6,7 @@ #include <vespa/eval/instruction/generic_merge.h> #include <vespa/eval/eval/interpreted_function.h> #include <vespa/eval/eval/test/reference_operations.h> -#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/gen_spec.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/gtest/gtest.h> #include <optional> @@ -18,20 +18,22 @@ using namespace vespalib::eval::test; using vespalib::make_string_short::fmt; -std::vector<Layout> merge_layouts = { - {}, {}, - {x(5)}, {x(5)}, - {x(3),y(5)}, {x(3),y(5)}, - float_cells({x(3),y(5)}), {x(3),y(5)}, - {x(3),y(5)}, float_cells({x(3),y(5)}), - {x({"a","b","c"})}, {x({"a","b","c"})}, - {x({"a","b","c"})}, {x({"c","d","e"})}, - {x({"a","c","e"})}, {x({"b","c","d"})}, - {x({"b","c","d"})}, {x({"a","c","e"})}, - {x({"a","b","c"})}, {x({"c","d"})}, - {x({"a","b"}),y({"foo","bar","baz"})}, {x({"b","c"}),y({"any","foo","bar"})}, - {x(3),y({"foo", "bar"})}, {x(3),y({"baz", "bar"})}, - {x({"a","b","c"}),y(5)}, {x({"b","c","d"}),y(5)} +GenSpec G() { return GenSpec(); } + +GenSpec::seq_t N_16ths = [] (size_t i) noexcept { return (i + 1.0) / 16.0; }; + +const std::vector<GenSpec> merge_layouts = { + G(), G(), + G().idx("x", 5), G().idx("x", 5), + G().idx("x", 3).idx("y", 5), G().idx("x", 3).idx("y", 5), + G().map("x", {"a","b","c"}), G().map("x", {"a","b","c"}), + G().map("x", {"a","b","c"}), G().map("x", {"c","d","e"}), + G().map("x", {"a","c","e"}), G().map("x", {"b","c","d"}), + G().map("x", {"b","c","d"}), G().map("x", {"a","c","e"}), + G().map("x", {"a","b","c"}), G().map("x", {"c","d"}), + G().map("x", {"a","b"}).map("y", {"foo","bar","baz"}), G().map("x", {"b","c"}).map("y", {"any","foo","bar"}), + G().idx("x", 3).map("y", {"foo", "bar"}), G().idx("x", 3).map("y", {"baz", "bar"}), + G().map("x", {"a","b","c"}).idx("y", 5), G().map("x", {"b","c","d"}).idx("y", 5) }; TensorSpec perform_generic_merge(const TensorSpec &a, const TensorSpec &b, join_fun_t fun, const ValueBuilderFactory &factory) { @@ -46,13 +48,21 @@ TensorSpec perform_generic_merge(const TensorSpec &a, const TensorSpec &b, join_ void test_generic_merge_with(const ValueBuilderFactory &factory) { ASSERT_TRUE((merge_layouts.size() % 2) == 0); for (size_t i = 0; i < merge_layouts.size(); i += 2) { - TensorSpec lhs = spec(merge_layouts[i], N()); - TensorSpec rhs = spec(merge_layouts[i + 1], Div16(N())); - SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); - for (auto fun: {operation::Add::f, operation::Mul::f, operation::Sub::f, operation::Max::f}) { - auto expect = ReferenceOperations::merge(lhs, rhs, fun); - auto actual = perform_generic_merge(lhs, rhs, fun, factory); - EXPECT_EQ(actual, expect); + const auto l = merge_layouts[i]; + const auto r = merge_layouts[i+1].cpy().seq(N_16ths); + for (TensorSpec lhs : { l.cpy().cells_float().gen(), + l.cpy().cells_double().gen() }) + { + for (TensorSpec rhs : { r.cpy().cells_float().gen(), + r.cpy().cells_double().gen() }) + { + SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); + for (auto fun: {operation::Add::f, operation::Mul::f, operation::Sub::f, operation::Max::f}) { + auto expect = ReferenceOperations::merge(lhs, rhs, fun); + auto actual = perform_generic_merge(lhs, rhs, fun, factory); + EXPECT_EQ(actual, expect); + } + } } } } diff --git a/eval/src/tests/instruction/generic_peek/generic_peek_test.cpp b/eval/src/tests/instruction/generic_peek/generic_peek_test.cpp index 6841215038a..c80e8a1296b 100644 --- a/eval/src/tests/instruction/generic_peek/generic_peek_test.cpp +++ b/eval/src/tests/instruction/generic_peek/generic_peek_test.cpp @@ -7,7 +7,7 @@ #include <vespa/eval/instruction/generic_peek.h> #include <vespa/eval/eval/interpreted_function.h> #include <vespa/eval/eval/test/reference_operations.h> -#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/gen_spec.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/overload.h> #include <vespa/vespalib/gtest/gtest.h> @@ -22,17 +22,16 @@ using namespace vespalib::eval::test; using vespalib::make_string_short::fmt; -std::vector<Layout> peek_layouts = { - {x(4)}, - {x(4),y(5)}, - {x(4),y(5),z(3)}, - float_cells({x(4),y(5),z(3)}), - {x({"-1","0","2"})}, - {x({"-1","0","2"}),y({"-2","0","1"}),z({"-2","-1","0","1","2"})}, - float_cells({x({"-1","0","2"}),y({"-2","0","1"})}), - {x(4),y({"-2","0","1"}),z(3)}, - {x({"-1","0","2"}),y(5),z({"-2","-1","0","1","2"})}, - float_cells({x({"-1","0","2"}),y(5),z({"-2","-1","0","1","2"})}) +GenSpec G() { return GenSpec(); } + +const std::vector<GenSpec> peek_layouts = { + G().idx("x", 4), + G().idx("x", 4).idx("y", 5), + G().idx("x", 4).idx("y", 5).idx("z", 3), + G().map("x", {"-1","0","2"}), + G().map("x", {"-1","0","2"}).map("y", {"-2","0","1"}).map("z", {"-2","-1","0","1","2"}), + G().idx("x", 4).map("y", {"-2","0","1"}).idx("z", 3), + G().map("x", {"-1","0","2"}).idx("y", 5).map("z", {"-2","-1","0","1","2"}) }; using PeekSpec = GenericPeek::SpecMap; @@ -194,12 +193,15 @@ void fill_dims_and_check(const TensorSpec &input, } void test_generic_peek_with(const ValueBuilderFactory &factory) { - for (const auto & layout : peek_layouts) { - TensorSpec input = spec(layout, N()); - ValueType input_type = ValueType::from_spec(input.type()); - const auto &dims = input_type.dimensions(); - PeekSpec spec; - fill_dims_and_check(input, spec, dims, factory); + for (const auto &layout : peek_layouts) { + for (TensorSpec input : { layout.cpy().cells_float().gen(), + layout.cpy().cells_double().gen() }) + { + ValueType input_type = ValueType::from_spec(input.type()); + const auto &dims = input_type.dimensions(); + PeekSpec spec; + fill_dims_and_check(input, spec, dims, factory); + } } } diff --git a/eval/src/tests/instruction/generic_reduce/generic_reduce_test.cpp b/eval/src/tests/instruction/generic_reduce/generic_reduce_test.cpp index 9e2090fa968..2c5baf234c4 100644 --- a/eval/src/tests/instruction/generic_reduce/generic_reduce_test.cpp +++ b/eval/src/tests/instruction/generic_reduce/generic_reduce_test.cpp @@ -6,7 +6,7 @@ #include <vespa/eval/instruction/generic_reduce.h> #include <vespa/eval/eval/interpreted_function.h> #include <vespa/eval/eval/test/reference_operations.h> -#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/gen_spec.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/gtest/gtest.h> #include <optional> @@ -18,22 +18,23 @@ using namespace vespalib::eval::test; using vespalib::make_string_short::fmt; -std::vector<Layout> layouts = { - {}, - {x(3)}, - {x(3),y(5)}, - {x(3),y(5),z(7)}, - float_cells({x(3),y(5),z(7)}), - {x({"a","b","c"})}, - {x({})}, - {x({}),y(10)}, - {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), - {x(3),y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}), - {x(3),y({}),z(7)} +GenSpec::seq_t N_16ths = [] (size_t i) noexcept { return (i + 1.0) / 16.0; }; + +GenSpec G() { return GenSpec().seq(N_16ths); } + +const std::vector<GenSpec> layouts = { + G(), + G().idx("x", 3), + G().idx("x", 3).idx("y", 5), + G().idx("x", 3).idx("y", 5).idx("z", 7), + G().map("x", {"a","b","c"}), + G().map("x", {}), + G().map("x", {}).idx("y", 10), + G().map("x", {"a","b","c"}).map("y", {"foo","bar"}), + G().map("x", {"a","b","c"}).map("y", {"foo","bar"}).map("z", {"i","j","k","l"}), + G().idx("x", 3).map("y", {"foo", "bar"}).idx("z", 7), + G().map("x", {"a","b","c"}).idx("y", 5).map("z", {"i","j","k","l"}), + G().idx("x", 3).map("y", {}).idx("z", 7) }; TensorSpec perform_generic_reduce(const TensorSpec &a, Aggr aggr, const std::vector<vespalib::string> &dims, @@ -68,19 +69,23 @@ TEST(GenericReduceTest, sparse_reduce_plan_can_be_created) { } void test_generic_reduce_with(const ValueBuilderFactory &factory) { - for (const Layout &layout: layouts) { - TensorSpec input = spec(layout, Div16(N())); - SCOPED_TRACE(fmt("tensor type: %s, num_cells: %zu", input.type().c_str(), input.cells().size())); - for (Aggr aggr: {Aggr::SUM, Aggr::AVG, Aggr::MIN, Aggr::MAX}) { - SCOPED_TRACE(fmt("aggregator: %s", AggrNames::name_of(aggr)->c_str())); - for (const Domain &domain: layout) { - auto expect = ReferenceOperations::reduce(input, aggr, {domain.dimension}).normalize(); - auto actual = perform_generic_reduce(input, aggr, {domain.dimension}, factory); + for (const auto &layout: layouts) { + for (TensorSpec input : { layout.cpy().cells_float().gen(), + layout.cpy().cells_double().gen() }) + { + SCOPED_TRACE(fmt("tensor type: %s, num_cells: %zu", input.type().c_str(), input.cells().size())); + for (Aggr aggr: {Aggr::SUM, Aggr::AVG, Aggr::MIN, Aggr::MAX}) { + SCOPED_TRACE(fmt("aggregator: %s", AggrNames::name_of(aggr)->c_str())); + auto t = layout.type(); + for (const auto & dim: t.dimensions()) { + auto expect = ReferenceOperations::reduce(input, aggr, {dim.name}).normalize(); + auto actual = perform_generic_reduce(input, aggr, {dim.name}, factory); + EXPECT_EQ(actual, expect); + } + auto expect = ReferenceOperations::reduce(input, aggr, {}).normalize(); + auto actual = perform_generic_reduce(input, aggr, {}, factory); EXPECT_EQ(actual, expect); } - auto expect = ReferenceOperations::reduce(input, aggr, {}).normalize(); - auto actual = perform_generic_reduce(input, aggr, {}, factory); - EXPECT_EQ(actual, expect); } } } diff --git a/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp b/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp index 20d155822b5..f0c2241202e 100644 --- a/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp +++ b/eval/src/tests/instruction/generic_rename/generic_rename_test.cpp @@ -5,7 +5,7 @@ #include <vespa/eval/eval/value_codec.h> #include <vespa/eval/instruction/generic_rename.h> #include <vespa/eval/eval/interpreted_function.h> -#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/gen_spec.h> #include <vespa/eval/eval/test/reference_operations.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/gtest/gtest.h> @@ -17,18 +17,17 @@ using namespace vespalib::eval::test; using vespalib::make_string_short::fmt; -std::vector<Layout> rename_layouts = { - {x(3)}, - {x(3),y(5)}, - {x(3),y(5),z(7)}, - float_cells({x(3),y(5),z(7)}), - {x({"a","b","c"})}, - {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), - {x(3),y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) +GenSpec G() { return GenSpec(); } + +const std::vector<GenSpec> rename_layouts = { + G().idx("x", 3), + G().idx("x", 3).idx("y", 5), + G().idx("x", 3).idx("y", 5).idx("z", 7), + G().map("x", {"a","b","c"}), + G().map("x", {"a","b","c"}).map("y", {"foo","bar"}), + G().map("x", {"a","b","c"}).map("y", {"foo","bar"}).map("z", {"i","j","k","l"}), + G().idx("x", 3).map("y", {"foo", "bar"}).idx("z", 7), + G().map("x", {"a","b","c"}).idx("y", 5).map("z", {"i","j","k","l"}) }; struct FromTo { @@ -110,18 +109,20 @@ TensorSpec perform_generic_rename(const TensorSpec &a, } void test_generic_rename_with(const ValueBuilderFactory &factory) { - for (const auto & layout : rename_layouts) { - TensorSpec lhs = spec(layout, N()); - ValueType lhs_type = ValueType::from_spec(lhs.type()); - // printf("lhs_type: %s\n", lhs_type.to_spec().c_str()); - for (const auto & from_to : rename_from_to) { - ValueType renamed_type = lhs_type.rename(from_to.from, from_to.to); - if (renamed_type.is_error()) continue; - // printf("type %s -> %s\n", lhs_type.to_spec().c_str(), renamed_type.to_spec().c_str()); - SCOPED_TRACE(fmt("\n===\nLHS: %s\n===\n", lhs.to_string().c_str())); - auto expect = ReferenceOperations::rename(lhs, from_to.from, from_to.to); - auto actual = perform_generic_rename(lhs, from_to, factory); - EXPECT_EQ(actual, expect); + for (const auto &layout : rename_layouts) { + for (TensorSpec lhs : { layout.cpy().cells_float().gen(), + layout.cpy().cells_double().gen() }) + { + ValueType lhs_type = ValueType::from_spec(lhs.type()); + for (const auto & from_to : rename_from_to) { + ValueType renamed_type = lhs_type.rename(from_to.from, from_to.to); + if (renamed_type.is_error()) continue; + // printf("type %s -> %s\n", lhs_type.to_spec().c_str(), renamed_type.to_spec().c_str()); + SCOPED_TRACE(fmt("\n===\nLHS: %s\n===\n", lhs.to_string().c_str())); + auto expect = ReferenceOperations::rename(lhs, from_to.from, from_to.to); + auto actual = perform_generic_rename(lhs, from_to, factory); + EXPECT_EQ(actual, expect); + } } } } diff --git a/eval/src/tests/instruction/join_with_number/join_with_number_function_test.cpp b/eval/src/tests/instruction/join_with_number/join_with_number_function_test.cpp index a3fbb3ed529..e6a256a493b 100644 --- a/eval/src/tests/instruction/join_with_number/join_with_number_function_test.cpp +++ b/eval/src/tests/instruction/join_with_number/join_with_number_function_test.cpp @@ -4,8 +4,7 @@ #include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/tensor_function.h> #include <vespa/eval/eval/test/eval_fixture.h> -#include <vespa/eval/eval/test/tensor_model.hpp> -#include <vespa/eval/eval/test/param_variants.h> +#include <vespa/eval/eval/test/gen_spec.h> #include <vespa/eval/instruction/join_with_number_function.h> #include <vespa/vespalib/util/stringfmt.h> @@ -36,13 +35,12 @@ const ValueBuilderFactory &prod_factory = FastValueBuilderFactory::get(); EvalFixture::ParamRepo make_params() { auto repo = EvalFixture::ParamRepo() - .add("a", spec(1.5)) - .add("number", spec(2.5)) - .add("dense", spec({y(5)}, N())) - .add_matrix("x", 3, "y", 5); - - add_variants(repo, "mixed", {x({"a"}),y(5),z({"d","e"})}, N()); - add_variants(repo, "sparse", {x({"a","b","c"}),z({"d","e","f"})}, N()); + .add("a", GenSpec().seq_bias(1.5).gen()) + .add("number", GenSpec().seq_bias(2.5).gen()) + .add("dense", GenSpec().idx("y", 5).gen()) + .add_variants("x3y5", GenSpec().idx("x", 3).idx("y", 5)) + .add_variants("mixed", GenSpec().map("x", {"a"}).idx("y", 5).map("z", {"d","e"})) + .add_variants("sparse", GenSpec().map("x", {"a","b","c"}).map("z", {"d","e","f"})); return repo; } @@ -81,22 +79,22 @@ void verify_not_optimized(const vespalib::string &expr) { TEST("require that dense number join can be optimized") { TEST_DO(verify_optimized("x3y5+a", Primary::LHS, false)); TEST_DO(verify_optimized("a+x3y5", Primary::RHS, false)); - TEST_DO(verify_optimized("x3y5f*a", Primary::LHS, false)); - TEST_DO(verify_optimized("a*x3y5f", Primary::RHS, false)); + TEST_DO(verify_optimized("x3y5_f*a", Primary::LHS, false)); + TEST_DO(verify_optimized("a*x3y5_f", Primary::RHS, false)); } TEST("require that dense number join can be inplace") { TEST_DO(verify_optimized("@x3y5*a", Primary::LHS, true)); TEST_DO(verify_optimized("a*@x3y5", Primary::RHS, true)); - TEST_DO(verify_optimized("@x3y5f+a", Primary::LHS, true)); - TEST_DO(verify_optimized("a+@x3y5f", Primary::RHS, true)); + TEST_DO(verify_optimized("@x3y5_f+a", Primary::LHS, true)); + TEST_DO(verify_optimized("a+@x3y5_f", Primary::RHS, true)); } TEST("require that asymmetric operations work") { TEST_DO(verify_optimized("x3y5/a", Primary::LHS, false)); TEST_DO(verify_optimized("a/x3y5", Primary::RHS, false)); - TEST_DO(verify_optimized("x3y5f-a", Primary::LHS, false)); - TEST_DO(verify_optimized("a-x3y5f", Primary::RHS, false)); + TEST_DO(verify_optimized("x3y5_f-a", Primary::LHS, false)); + TEST_DO(verify_optimized("a-x3y5_f", Primary::RHS, false)); } TEST("require that sparse number join can be optimized") { diff --git a/eval/src/tests/instruction/mixed_inner_product_function/mixed_inner_product_function_test.cpp b/eval/src/tests/instruction/mixed_inner_product_function/mixed_inner_product_function_test.cpp index fbe71f3ed63..6b549b4d4d4 100644 --- a/eval/src/tests/instruction/mixed_inner_product_function/mixed_inner_product_function_test.cpp +++ b/eval/src/tests/instruction/mixed_inner_product_function/mixed_inner_product_function_test.cpp @@ -3,7 +3,7 @@ #include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/tensor_function.h> #include <vespa/eval/eval/test/eval_fixture.h> -#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/gen_spec.h> #include <vespa/eval/instruction/dense_dot_product_function.h> #include <vespa/eval/instruction/dense_matmul_function.h> #include <vespa/eval/instruction/dense_multi_matmul_function.h> @@ -22,34 +22,25 @@ using namespace vespalib::eval::test; const ValueBuilderFactory &prod_factory = FastValueBuilderFactory::get(); -struct MyVecSeq : Sequence { - double bias; - double operator[](size_t i) const override { return (i + bias); } - MyVecSeq(double cellBias) : bias(cellBias) {} -}; - -std::function<double(size_t)> my_vec_gen(double cellBias) { - return [=] (size_t i) noexcept { return i + cellBias; }; -} //----------------------------------------------------------------------------- EvalFixture::ParamRepo make_params() { return EvalFixture::ParamRepo() - .add_vector("x", 3, my_vec_gen(2.0)) - .add_vector("x", 3, my_vec_gen(13.25)) - .add_vector("y", 3, my_vec_gen(4.0)) - .add_vector("z", 3, my_vec_gen(0.25)) - .add_matrix("x", 3, "y", 1, my_vec_gen(5.0)) - .add_matrix("x", 1, "y", 3, my_vec_gen(6.0)) - .add_matrix("x", 3, "y", 3, my_vec_gen(1.5)) - .add_matrix("x", 3, "z", 3, my_vec_gen(2.5)) - .add_cube("x", 3, "y", 3, "z", 3, my_vec_gen(-4.0)) - .add("mix_x3zm", spec({x(3),z({"c","d"})}, MyVecSeq(0.5))) - .add("mix_y3zm", spec({y(3),z({"c","d"})}, MyVecSeq(3.5))) - .add("mix_x3zm_f", spec(float_cells({x(3),z({"c","d"})}), MyVecSeq(0.5))) - .add("mix_y3zm_f", spec(float_cells({y(3),z({"c","d"})}), MyVecSeq(3.5))) - .add("mix_x3y3zm", spec({x(3),y(3),z({"c","d"})}, MyVecSeq(0.0))) + .add_variants("x3", GenSpec().idx("x", 3).seq_bias(2.0)) + .add_variants("x3$2", GenSpec().idx("x", 3).seq_bias(13.25)) + .add_variants("y3", GenSpec().idx("y", 3).seq_bias(4.0)) + .add_variants("z3", GenSpec().idx("z", 3).seq_bias(0.25)) + .add_variants("x3y3", GenSpec().idx("x", 3).idx("y", 3).seq_bias(5.0)) + .add_variants("x1y3", GenSpec().idx("x", 1).idx("y", 3).seq_bias(6.0)) + .add_variants("x3y1", GenSpec().idx("x", 3).idx("y", 1).seq_bias(1.5)) + .add_variants("x3z3", GenSpec().idx("x", 3).idx("z", 3).seq_bias(2.5)) + .add_variants("x3y3z3", GenSpec().idx("x", 3).idx("y", 3).idx("z", 3).seq_bias(-4.0)) + .add("mix_x3zm", GenSpec().idx("x", 3).map("z", {"c","d"}).seq_bias(0.5).gen()) + .add("mix_y3zm", GenSpec().idx("y", 3).map("z", {"c","d"}).seq_bias(3.5).gen()) + .add("mix_x3zm_f", GenSpec().idx("x", 3).map("z", {"c","d"}).cells_float().seq_bias(0.5).gen()) + .add("mix_y3zm_f", GenSpec().idx("y", 3).map("z", {"c","d"}).cells_float().seq_bias(3.5).gen()) + .add("mix_x3y3zm", GenSpec().idx("x", 3).idx("y", 3).map("z", {"c","d"}).seq_bias(0.0).gen()) ; } @@ -101,35 +92,35 @@ TEST(MixedInnerProduct, use_dense_optimizers_when_possible) { TEST(MixedInnerProduct, trigger_optimizer_when_possible) { assert_mixed_optimized("reduce(x3 * mix_x3zm,sum,x)"); - assert_mixed_optimized("reduce(x3f * mix_x3zm,sum,x)"); + assert_mixed_optimized("reduce(x3_f * mix_x3zm,sum,x)"); assert_mixed_optimized("reduce(x3 * mix_x3zm_f,sum,x)"); - assert_mixed_optimized("reduce(x3f * mix_x3zm_f,sum,x)"); + assert_mixed_optimized("reduce(x3_f * mix_x3zm_f,sum,x)"); assert_mixed_optimized("reduce(x3$2 * mix_x3zm,sum,x)"); - assert_mixed_optimized("reduce(x3f$2 * mix_x3zm,sum,x)"); + assert_mixed_optimized("reduce(x3$2_f * mix_x3zm,sum,x)"); assert_mixed_optimized("reduce(y3 * mix_y3zm,sum,y)"); - assert_mixed_optimized("reduce(y3f * mix_y3zm,sum,y)"); + assert_mixed_optimized("reduce(y3_f * mix_y3zm,sum,y)"); assert_mixed_optimized("reduce(y3 * mix_y3zm_f,sum,y)"); - assert_mixed_optimized("reduce(y3f * mix_y3zm_f,sum,y)"); + assert_mixed_optimized("reduce(y3_f * mix_y3zm_f,sum,y)"); assert_mixed_optimized("reduce(x3y1 * mix_x3zm,sum,x)"); - assert_mixed_optimized("reduce(x3y1f * mix_x3zm,sum,x)"); + assert_mixed_optimized("reduce(x3y1_f * mix_x3zm,sum,x)"); assert_mixed_optimized("reduce(x3y1 * mix_x3zm,sum,x,y)"); - assert_mixed_optimized("reduce(x3y1f * mix_x3zm,sum,x,y)"); + assert_mixed_optimized("reduce(x3y1_f * mix_x3zm,sum,x,y)"); assert_mixed_optimized("reduce(x1y3 * mix_y3zm,sum,y)"); - assert_mixed_optimized("reduce(x1y3f * mix_y3zm,sum,y)"); + assert_mixed_optimized("reduce(x1y3_f * mix_y3zm,sum,y)"); assert_mixed_optimized("reduce(x1y3 * x1y3,sum,y)"); - assert_mixed_optimized("reduce(x1y3 * x1y3f,sum,y)"); - assert_mixed_optimized("reduce(x1y3f * x1y3,sum,y)"); - assert_mixed_optimized("reduce(x1y3f * x1y3f,sum,y)"); + assert_mixed_optimized("reduce(x1y3 * x1y3_f,sum,y)"); + assert_mixed_optimized("reduce(x1y3_f * x1y3,sum,y)"); + assert_mixed_optimized("reduce(x1y3_f * x1y3_f,sum,y)"); assert_mixed_optimized("reduce(x1y3 * mix_y3zm,sum,y)"); - assert_mixed_optimized("reduce(x1y3f * mix_y3zm,sum,y)"); + assert_mixed_optimized("reduce(x1y3_f * mix_y3zm,sum,y)"); assert_mixed_optimized("reduce(mix_x3zm * x3,sum,x)"); - assert_mixed_optimized("reduce(mix_x3zm * x3f,sum,x)"); + assert_mixed_optimized("reduce(mix_x3zm * x3_f,sum,x)"); assert_mixed_optimized("reduce(mix_x3zm * x3y1,sum,x)"); - assert_mixed_optimized("reduce(mix_x3zm * x3y1f,sum,x)"); + assert_mixed_optimized("reduce(mix_x3zm * x3y1_f,sum,x)"); assert_mixed_optimized("reduce(mix_y3zm * y3,sum,y)"); - assert_mixed_optimized("reduce(mix_y3zm * y3f,sum,y)"); + assert_mixed_optimized("reduce(mix_y3zm * y3_f,sum,y)"); assert_mixed_optimized("reduce(mix_y3zm * x1y3,sum,y)"); - assert_mixed_optimized("reduce(mix_y3zm * x1y3f,sum,y)"); + assert_mixed_optimized("reduce(mix_y3zm * x1y3_f,sum,y)"); } TEST(MixedInnerProduct, should_not_trigger_optimizer_for_other_cases) { diff --git a/eval/src/tests/instruction/mixed_map_function/mixed_map_function_test.cpp b/eval/src/tests/instruction/mixed_map_function/mixed_map_function_test.cpp index 3caebea7298..3a7d1368f03 100644 --- a/eval/src/tests/instruction/mixed_map_function/mixed_map_function_test.cpp +++ b/eval/src/tests/instruction/mixed_map_function/mixed_map_function_test.cpp @@ -3,7 +3,7 @@ #include <vespa/eval/eval/tensor_function.h> #include <vespa/eval/instruction/mixed_map_function.h> #include <vespa/eval/eval/test/eval_fixture.h> -#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/gen_spec.h> #include <vespa/vespalib/gtest/gtest.h> using namespace vespalib; @@ -15,13 +15,11 @@ const ValueBuilderFactory &prod_factory = FastValueBuilderFactory::get(); EvalFixture::ParamRepo make_params() { return EvalFixture::ParamRepo() - .add("a", spec(1.5)) - .add("b", spec(2.5)) - .add("sparse", spec({x({"a"})}, N())) - .add("mixed", spec({x({"a"}),y(5)}, N())) - .add_mutable("@sparse", spec({x({"a"})}, N())) - .add_mutable("@mixed", spec({x({"a"}),y(5)}, N())) - .add_matrix("x", 5, "y", 3); + .add("a", GenSpec().seq_bias(1.5).gen()) + .add("b", GenSpec().seq_bias(2.5).gen()) + .add_variants("sparse", GenSpec().map("x", {"a"})) + .add_variants("mixed", GenSpec().map("x", {"a"}).idx("y", 5)) + .add_variants("x5y3", GenSpec().idx("x", 5).idx("y", 3)); } EvalFixture::ParamRepo param_repo = make_params(); @@ -53,12 +51,12 @@ void verify_not_optimized(const vespalib::string &expr) { TEST(MapTest, dense_map_is_optimized) { verify_optimized("map(x5y3,f(x)(x+10))", false); - verify_optimized("map(x5y3f,f(x)(x+10))", false); + verify_optimized("map(x5y3_f,f(x)(x+10))", false); } TEST(MapTest, simple_dense_map_can_be_inplace) { verify_optimized("map(@x5y3,f(x)(x+10))", true); - verify_optimized("map(@x5y3f,f(x)(x+10))", true); + verify_optimized("map(@x5y3_f,f(x)(x+10))", true); } TEST(MapTest, scalar_map_is_not_optimized) { diff --git a/eval/src/tests/instruction/mixed_simple_join_function/mixed_simple_join_function_test.cpp b/eval/src/tests/instruction/mixed_simple_join_function/mixed_simple_join_function_test.cpp index 9c891adf179..105ae22e06e 100644 --- a/eval/src/tests/instruction/mixed_simple_join_function/mixed_simple_join_function_test.cpp +++ b/eval/src/tests/instruction/mixed_simple_join_function/mixed_simple_join_function_test.cpp @@ -4,7 +4,7 @@ #include <vespa/eval/eval/tensor_function.h> #include <vespa/eval/instruction/mixed_simple_join_function.h> #include <vespa/eval/eval/test/eval_fixture.h> -#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/gen_spec.h> #include <vespa/vespalib/util/stringfmt.h> @@ -45,24 +45,24 @@ const ValueBuilderFactory &prod_factory = FastValueBuilderFactory::get(); EvalFixture::ParamRepo make_params() { return EvalFixture::ParamRepo() - .add("a", spec(1.5)) - .add("b", spec(2.5)) - .add("sparse", spec({x({"a", "b", "c"})}, N())) - .add("mixed", spec({x({"a", "b", "c"}),y(5),z(3)}, N())) - .add("empty_mixed", spec({x({}),y(5),z(3)}, N())) - .add_mutable("@mixed", spec({x({"a", "b", "c"}),y(5),z(3)}, N())) - .add_cube("a", 1, "b", 1, "c", 1) - .add_cube("x", 1, "y", 1, "z", 1) - .add_cube("x", 3, "y", 5, "z", 3) - .add_vector("z", 3) - .add_dense({{"c", 5}, {"d", 1}}) - .add_dense({{"b", 1}, {"c", 5}}) - .add_matrix("x", 3, "y", 5, [](size_t idx) noexcept { return double((idx * 2) + 3); }) - .add_matrix("x", 3, "y", 5, [](size_t idx) noexcept { return double((idx * 3) + 2); }) - .add_vector("y", 5, [](size_t idx) noexcept { return double((idx * 2) + 3); }) - .add_vector("y", 5, [](size_t idx) noexcept { return double((idx * 3) + 2); }) - .add_matrix("y", 5, "z", 3, [](size_t idx) noexcept { return double((idx * 2) + 3); }) - .add_matrix("y", 5, "z", 3, [](size_t idx) noexcept { return double((idx * 3) + 2); }); + .add("a", GenSpec().seq_bias(1.5).gen()) + .add("b", GenSpec().seq_bias(2.5).gen()) + .add("sparse", GenSpec().map("x", {"a", "b", "c"}).gen()) + .add("mixed", GenSpec().map("x", {"a", "b", "c"}).idx("y", 5).idx("z", 3).gen()) + .add("empty_mixed", GenSpec().map("x", {}).idx("y", 5).idx("z", 3).gen()) + .add_mutable("@mixed", GenSpec().map("x", {"a", "b", "c"}).idx("y", 5).idx("z", 3).gen()) + .add_variants("a1b1c1", GenSpec().idx("a", 1).idx("b", 1).idx("c", 1)) + .add_variants("x1y1z1", GenSpec().idx("x", 1).idx("y", 1).idx("z", 1)) + .add_variants("x3y5z3", GenSpec().idx("x", 3).idx("y", 5).idx("z", 3)) + .add_variants("z3", GenSpec().idx("z", 3)) + .add_variants("c5d1", GenSpec().idx("c", 5).idx("d", 1)) + .add_variants("b1c5", GenSpec().idx("b", 1).idx("c", 5)) + .add_variants("x3y5", GenSpec().idx("x", 3).idx("y", 5).seq([](size_t idx) noexcept { return double((idx * 2) + 3); })) + .add_variants("x3y5$2", GenSpec().idx("x", 3).idx("y", 5).seq([](size_t idx) noexcept { return double((idx * 3) + 2); })) + .add_variants("y5", GenSpec().idx("y", 5).seq([](size_t idx) noexcept { return double((idx * 2) + 3); })) + .add_variants("y5$2", GenSpec().idx("y", 5).seq([](size_t idx) noexcept { return double((idx * 3) + 2); })) + .add_variants("y5z3", GenSpec().idx("y", 5).idx("z", 3).seq([](size_t idx) noexcept { return double((idx * 2) + 3); })) + .add_variants("y5z3$2", GenSpec().idx("y", 5).idx("z", 3).seq([](size_t idx) noexcept { return double((idx * 3) + 2); })); } EvalFixture::ParamRepo param_repo = make_params(); @@ -149,12 +149,12 @@ vespalib::string adjust_param(const vespalib::string &str, bool float_cells, boo if (mut_cells) { result = "@" + result; } - if (float_cells) { - result += "f"; - } if (is_rhs) { result += "$2"; } + if (float_cells) { + result += "_f"; + } return result; } diff --git a/eval/src/tests/instruction/pow_as_map_optimizer/pow_as_map_optimizer_test.cpp b/eval/src/tests/instruction/pow_as_map_optimizer/pow_as_map_optimizer_test.cpp index b4bf9ec5ef6..fe32a59bb78 100644 --- a/eval/src/tests/instruction/pow_as_map_optimizer/pow_as_map_optimizer_test.cpp +++ b/eval/src/tests/instruction/pow_as_map_optimizer/pow_as_map_optimizer_test.cpp @@ -3,7 +3,7 @@ #include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/tensor_function.h> #include <vespa/eval/eval/test/eval_fixture.h> -#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/gen_spec.h> #include <vespa/vespalib/gtest/gtest.h> using namespace vespalib::eval::operation; @@ -16,11 +16,11 @@ const ValueBuilderFactory &prod_factory = FastValueBuilderFactory::get(); EvalFixture::ParamRepo make_params() { return EvalFixture::ParamRepo() - .add("a", spec(1.5)) - .add("b", spec(2.5)) - .add("sparse", spec({x({"a","b"})}, N())) - .add("mixed", spec({x({"a"}),y(5)}, N())) - .add_matrix("x", 5, "y", 3); + .add("a", GenSpec().seq_bias(1.5).gen()) + .add("b", GenSpec().seq_bias(2.5).gen()) + .add("sparse", GenSpec().map("x", {"a","b"}).gen()) + .add("mixed", GenSpec().map("x", {"a"}).idx("y", 5).gen()) + .add_variants("x5y3", GenSpec().idx("x", 5).idx("y", 3)); } EvalFixture::ParamRepo param_repo = make_params(); @@ -55,9 +55,9 @@ TEST(PowAsMapTest, squared_dense_tensor_is_optimized) { verify_optimized("pow(x5y3,2.0)", Square::f); verify_optimized("join(x5y3,2.0,f(x,y)(x^y))", Square::f); verify_optimized("join(x5y3,2.0,f(x,y)(pow(x,y)))", Square::f); - verify_optimized("join(x5y3f,2.0,f(x,y)(pow(x,y)))", Square::f); + verify_optimized("join(x5y3_f,2.0,f(x,y)(pow(x,y)))", Square::f); verify_optimized("join(@x5y3,2.0,f(x,y)(pow(x,y)))", Square::f, true); - verify_optimized("join(@x5y3f,2.0,f(x,y)(pow(x,y)))", Square::f, true); + verify_optimized("join(@x5y3_f,2.0,f(x,y)(pow(x,y)))", Square::f, true); } TEST(PowAsMapTest, cubed_dense_tensor_is_optimized) { @@ -65,9 +65,9 @@ TEST(PowAsMapTest, cubed_dense_tensor_is_optimized) { verify_optimized("pow(x5y3,3.0)", Cube::f); verify_optimized("join(x5y3,3.0,f(x,y)(x^y))", Cube::f); verify_optimized("join(x5y3,3.0,f(x,y)(pow(x,y)))", Cube::f); - verify_optimized("join(x5y3f,3.0,f(x,y)(pow(x,y)))", Cube::f); + verify_optimized("join(x5y3_f,3.0,f(x,y)(pow(x,y)))", Cube::f); verify_optimized("join(@x5y3,3.0,f(x,y)(pow(x,y)))", Cube::f, true); - verify_optimized("join(@x5y3f,3.0,f(x,y)(pow(x,y)))", Cube::f, true); + verify_optimized("join(@x5y3_f,3.0,f(x,y)(pow(x,y)))", Cube::f, true); } TEST(PowAsMapTest, hypercubed_dense_tensor_is_not_optimized) { diff --git a/eval/src/tests/instruction/remove_trivial_dimension_optimizer/remove_trivial_dimension_optimizer_test.cpp b/eval/src/tests/instruction/remove_trivial_dimension_optimizer/remove_trivial_dimension_optimizer_test.cpp index 4de7e85074d..794725a8257 100644 --- a/eval/src/tests/instruction/remove_trivial_dimension_optimizer/remove_trivial_dimension_optimizer_test.cpp +++ b/eval/src/tests/instruction/remove_trivial_dimension_optimizer/remove_trivial_dimension_optimizer_test.cpp @@ -4,7 +4,7 @@ #include <vespa/eval/eval/tensor_function.h> #include <vespa/eval/instruction/replace_type_function.h> #include <vespa/eval/instruction/fast_rename_optimizer.h> -#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/gen_spec.h> #include <vespa/eval/eval/test/eval_fixture.h> #include <vespa/vespalib/util/stringfmt.h> @@ -19,10 +19,10 @@ const ValueBuilderFactory &prod_factory = FastValueBuilderFactory::get(); EvalFixture::ParamRepo make_params() { return EvalFixture::ParamRepo() - .add("x1y5z1", spec({x(1),y(5),z(1)}, N())) - .add("x1y5z1f", spec(float_cells({x(1),y(5),z(1)}), N())) - .add("x1y1z1", spec({x(1),y(1),z(1)}, N())) - .add("x1y5z_m", spec({x(1),y(5),z({"a"})}, N())); + .add("x1y5z1", GenSpec().idx("x", 1).idx("y", 5).idx("z", 1).gen()) + .add("x1y5z1f", GenSpec().idx("x", 1).idx("y", 5).idx("z", 1).cells_float().gen()) + .add("x1y1z1", GenSpec().idx("x", 1).idx("y", 1).idx("z", 1).gen()) + .add("x1y5z_m", GenSpec().idx("x", 1).idx("y", 5).map("z", {"a"}).gen()); } EvalFixture::ParamRepo param_repo = make_params(); diff --git a/eval/src/tests/instruction/sparse_dot_product_function/CMakeLists.txt b/eval/src/tests/instruction/sparse_dot_product_function/CMakeLists.txt new file mode 100644 index 00000000000..076f1d79796 --- /dev/null +++ b/eval/src/tests/instruction/sparse_dot_product_function/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_sparse_dot_product_function_test_app TEST + SOURCES + sparse_dot_product_function_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_sparse_dot_product_function_test_app COMMAND eval_sparse_dot_product_function_test_app) diff --git a/eval/src/tests/instruction/sparse_dot_product_function/sparse_dot_product_function_test.cpp b/eval/src/tests/instruction/sparse_dot_product_function/sparse_dot_product_function_test.cpp new file mode 100644 index 00000000000..65eab2778aa --- /dev/null +++ b/eval/src/tests/instruction/sparse_dot_product_function/sparse_dot_product_function_test.cpp @@ -0,0 +1,85 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/instruction/sparse_dot_product_function.h> +#include <vespa/eval/eval/test/eval_fixture.h> +#include <vespa/eval/eval/test/gen_spec.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib::eval; +using namespace vespalib::eval::test; + +const ValueBuilderFactory &prod_factory = FastValueBuilderFactory::get(); +const ValueBuilderFactory &test_factory = SimpleValueBuilderFactory::get(); + +//----------------------------------------------------------------------------- + +EvalFixture::ParamRepo make_params() { + return EvalFixture::ParamRepo() + .add("v1_x", GenSpec().map("x", 32, 1).seq_bias(3.0).gen()) + .add("v1_x_f", GenSpec().map("x", 32, 1).seq_bias(3.0).cells_float().gen()) + .add("v2_x", GenSpec().map("x", 16, 2).seq_bias(7.0).gen()) + .add("v2_x_f", GenSpec().map("x", 16, 2).seq_bias(7.0).cells_float().gen()) + .add("v3_y", GenSpec().map("y", 10, 1).gen()) + .add("v4_xd", GenSpec().idx("x", 10).gen()) + .add("m1_xy", GenSpec().map("x", 32, 1).map("y", 16, 2).seq_bias(3.0).gen()) + .add("m2_xy", GenSpec().map("x", 16, 2).map("y", 32, 1).seq_bias(7.0).gen()) + .add("m3_xym", GenSpec().map("x", 8, 1).idx("y", 5).gen()); +} +EvalFixture::ParamRepo param_repo = make_params(); + +void assert_optimized(const vespalib::string &expr) { + EvalFixture fast_fixture(prod_factory, expr, param_repo, true); + EvalFixture test_fixture(test_factory, expr, param_repo, true); + EvalFixture slow_fixture(prod_factory, expr, param_repo, false); + EXPECT_EQ(fast_fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQ(test_fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQ(slow_fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQ(fast_fixture.find_all<SparseDotProductFunction>().size(), 1u); + EXPECT_EQ(test_fixture.find_all<SparseDotProductFunction>().size(), 1u); + EXPECT_EQ(slow_fixture.find_all<SparseDotProductFunction>().size(), 0u); +} + +void assert_not_optimized(const vespalib::string &expr) { + EvalFixture fast_fixture(prod_factory, expr, param_repo, true); + EXPECT_EQ(fast_fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQ(fast_fixture.find_all<SparseDotProductFunction>().size(), 0u); +} + +//----------------------------------------------------------------------------- + +TEST(SparseDotProduct, expression_can_be_optimized) +{ + assert_optimized("reduce(v1_x*v2_x,sum,x)"); + assert_optimized("reduce(v2_x*v1_x,sum)"); + assert_optimized("reduce(v1_x*v2_x_f,sum)"); + assert_optimized("reduce(v1_x_f*v2_x,sum)"); + assert_optimized("reduce(v1_x_f*v2_x_f,sum)"); +} + +TEST(SparseDotProduct, multi_dimensional_expression_can_be_optimized) +{ + assert_optimized("reduce(m1_xy*m2_xy,sum,x,y)"); + assert_optimized("reduce(m1_xy*m2_xy,sum)"); +} + +TEST(SparseDotProduct, embedded_dot_product_is_not_optimized) +{ + assert_not_optimized("reduce(m1_xy*v1_x,sum,x)"); + assert_not_optimized("reduce(v1_x*m1_xy,sum,x)"); +} + +TEST(SparseDotProduct, similar_expressions_are_not_optimized) +{ + assert_not_optimized("reduce(m1_xy*v1_x,sum)"); + assert_not_optimized("reduce(v1_x*v3_y,sum)"); + assert_not_optimized("reduce(v2_x*v1_x,max)"); + assert_not_optimized("reduce(v2_x+v1_x,sum)"); + assert_not_optimized("reduce(v4_xd*v4_xd,sum)"); + assert_not_optimized("reduce(m3_xym*m3_xym,sum)"); +} + +//----------------------------------------------------------------------------- + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/tests/instruction/sum_max_dot_product_function/sum_max_dot_product_function_test.cpp b/eval/src/tests/instruction/sum_max_dot_product_function/sum_max_dot_product_function_test.cpp index 4b89f30d879..1013c98b424 100644 --- a/eval/src/tests/instruction/sum_max_dot_product_function/sum_max_dot_product_function_test.cpp +++ b/eval/src/tests/instruction/sum_max_dot_product_function/sum_max_dot_product_function_test.cpp @@ -3,7 +3,7 @@ #include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/tensor_function.h> #include <vespa/eval/eval/test/eval_fixture.h> -#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/gen_spec.h> #include <vespa/eval/instruction/sum_max_dot_product_function.h> #include <vespa/vespalib/gtest/gtest.h> @@ -13,12 +13,6 @@ using namespace vespalib::eval::test; const ValueBuilderFactory &prod_factory = FastValueBuilderFactory::get(); -struct MyVecSeq : Sequence { - double bias; - double operator[](size_t i) const override { return (i + bias); } - MyVecSeq(double cellBias) : bias(cellBias) {} -}; - //----------------------------------------------------------------------------- vespalib::string main_expr = "reduce(reduce(reduce(a*b,sum,z),max,y),sum,x)"; @@ -34,7 +28,7 @@ void assert_optimized(const TensorSpec &a, const TensorSpec &b, size_t dp_size) auto info = fast_fixture.find_all<SumMaxDotProductFunction>(); ASSERT_EQ(info.size(), 1u); EXPECT_TRUE(info[0]->result_is_mutable()); - EXPECT_EQUAL(info[0]->dp_size(), dp_size); + EXPECT_EQ(info[0]->dp_size(), dp_size); } void assert_not_optimized(const TensorSpec &a, const TensorSpec &b, const vespalib::string &expr = main_expr) { @@ -51,10 +45,23 @@ void assert_not_optimized(const TensorSpec &a, const TensorSpec &b, const vespal //----------------------------------------------------------------------------- -auto query = spec(float_cells({x({"0", "1", "2"}),z(5)}), MyVecSeq(0.5)); -auto document = spec(float_cells({y({"0", "1", "2", "3", "4", "5"}),z(5)}), MyVecSeq(2.5)); -auto empty_query = spec(float_cells({x({}),z(5)}), MyVecSeq(0.5)); -auto empty_document = spec(float_cells({y({}),z(5)}), MyVecSeq(2.5)); +GenSpec QueGen(size_t x_size, size_t z_size) { return GenSpec().cells_float().map("x", x_size).idx("z", z_size).seq_bias(0.5); } + +GenSpec DocGen(size_t y_size, size_t z_size) { return GenSpec().cells_float().map("y", y_size).idx("z", z_size).seq_bias(2.5); } + +GenSpec Que() { return QueGen(3, 5); } +GenSpec Doc() { return DocGen(6, 5); } + +GenSpec QueEmptyX() { return QueGen(0, 5); } +GenSpec DocEmptyX() { return DocGen(0, 5); } + +GenSpec QueTrivialZ() { return QueGen(3, 1); } +GenSpec DocTrivialZ() { return DocGen(6, 1); } + +auto query = Que().gen(); +auto document = Doc().gen(); +auto empty_query = QueEmptyX().gen(); +auto empty_document = DocEmptyX().gen(); TEST(SumMaxDotProduct, expressions_can_be_optimized) { @@ -66,24 +73,24 @@ TEST(SumMaxDotProduct, expressions_can_be_optimized) } TEST(SumMaxDotProduct, double_cells_are_not_optimized) { - auto double_query = spec({x({"0", "1", "2"}),z(5)}, MyVecSeq(0.5)); - auto double_document = spec({y({"0", "1", "2", "3", "4", "5"}),z(5)}, MyVecSeq(2.5)); + auto double_query = Que().cells_double().gen(); + auto double_document = Doc().cells_double().gen(); assert_not_optimized(query, double_document); assert_not_optimized(double_query, document); assert_not_optimized(double_query, double_document); } TEST(SumMaxDotProduct, trivial_dot_product_is_not_optimized) { - auto trivial_query = spec(float_cells({x({"0", "1", "2"}),z(1)}), MyVecSeq(0.5)); - auto trivial_document = spec(float_cells({y({"0", "1", "2", "3", "4", "5"}),z(1)}), MyVecSeq(2.5)); + auto trivial_query = QueTrivialZ().gen(); + auto trivial_document = DocTrivialZ().gen(); assert_not_optimized(trivial_query, trivial_document); } TEST(SumMaxDotProduct, additional_dimensions_are_not_optimized) { - auto extra_sparse_query = spec(float_cells({Domain("a", {"0"}),x({"0", "1", "2"}),z(5)}), MyVecSeq(0.5)); - auto extra_dense_query = spec(float_cells({Domain("a", 1),x({"0", "1", "2"}),z(5)}), MyVecSeq(0.5)); - auto extra_sparse_document = spec(float_cells({Domain("a", {"0"}),y({"0", "1", "2", "3", "4", "5"}),z(5)}), MyVecSeq(2.5)); - auto extra_dense_document = spec(float_cells({Domain("a", 1),y({"0", "1", "2", "3", "4", "5"}),z(5)}), MyVecSeq(2.5)); + auto extra_sparse_query = Que().map("a", 1).gen(); + auto extra_dense_query = Que().idx("a", 1).gen(); + auto extra_sparse_document = Doc().map("a", 1).gen(); + auto extra_dense_document = Doc().idx("a", 1).gen(); vespalib::string extra_sum_expr = "reduce(reduce(reduce(a*b,sum,z),max,y),sum,a,x)"; vespalib::string extra_max_expr = "reduce(reduce(reduce(a*b,sum,z),max,a,y),sum,x)"; assert_not_optimized(extra_sparse_query, document); @@ -97,8 +104,8 @@ TEST(SumMaxDotProduct, additional_dimensions_are_not_optimized) { } TEST(SumMaxDotProduct, more_dense_variants_are_not_optimized) { - auto dense_query = spec(float_cells({x(3),z(5)}), MyVecSeq(0.5)); - auto dense_document = spec(float_cells({y(5),z(5)}), MyVecSeq(2.5)); + auto dense_query = GenSpec().cells_float().idx("x", 3).idx("z", 5).seq_bias(0.5).gen(); + auto dense_document = GenSpec().cells_float().idx("y", 5).idx("z", 5).seq_bias(2.5).gen(); assert_not_optimized(dense_query, document); assert_not_optimized(query, dense_document); assert_not_optimized(dense_query, dense_document); diff --git a/eval/src/tests/streamed/value/streamed_value_test.cpp b/eval/src/tests/streamed/value/streamed_value_test.cpp index 2f91c3b9390..7aaa8cdebbc 100644 --- a/eval/src/tests/streamed/value/streamed_value_test.cpp +++ b/eval/src/tests/streamed/value/streamed_value_test.cpp @@ -5,7 +5,7 @@ #include <vespa/eval/eval/value_codec.h> #include <vespa/eval/instruction/generic_join.h> #include <vespa/eval/eval/interpreted_function.h> -#include <vespa/eval/eval/test/tensor_model.hpp> +#include <vespa/eval/eval/test/gen_spec.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/gtest/gtest.h> @@ -23,45 +23,36 @@ using Handle = SharedStringRepo::Handle; vespalib::string as_str(string_id label) { return Handle::string_from_id(label); } -std::vector<Layout> layouts = { - {}, - {x(3)}, - {x(3),y(5)}, - {x(3),y(5),z(7)}, - float_cells({x(3),y(5),z(7)}), - {x({"a","b","c"})}, - {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y({"foo","bar"}),z({"i","j","k","l"})}), - {x(3),y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y(5),z({"i","j","k","l"})}) +GenSpec G() { return GenSpec(); } + +const std::vector<GenSpec> layouts = { + G(), + G().idx("x", 3), + G().idx("x", 3).idx("y", 5), + G().idx("x", 3).idx("y", 5).idx("z", 7), + G().map("x", {"a","b","c"}), + G().map("x", {"a","b","c"}).map("y", {"foo","bar"}), + G().map("x", {"a","b","c"}).map("y", {"foo","bar"}).map("z", {"i","j","k","l"}), + G().idx("x", 3).map("y", {"foo", "bar"}).idx("z", 7), + G().map("x", {"a","b","c"}).idx("y", 5).map("z", {"i","j","k","l"}) }; -std::vector<Layout> join_layouts = { - {}, {}, - {x(5)}, {x(5)}, - {x(5)}, {y(5)}, - {x(5)}, {x(5),y(5)}, - {y(3)}, {x(2),z(3)}, - {x(3),y(5)}, {y(5),z(7)}, - float_cells({x(3),y(5)}), {y(5),z(7)}, - {x(3),y(5)}, float_cells({y(5),z(7)}), - float_cells({x(3),y(5)}), float_cells({y(5),z(7)}), - {x({"a","b","c"})}, {x({"a","b","c"})}, - {x({"a","b","c"})}, {x({"a","b"})}, - {x({"a","b","c"})}, {y({"foo","bar","baz"})}, - {x({"a","b","c"})}, {x({"a","b","c"}),y({"foo","bar","baz"})}, - {x({"a","b"}),y({"foo","bar","baz"})}, {x({"a","b","c"}),y({"foo","bar"})}, - {x({"a","b"}),y({"foo","bar","baz"})}, {y({"foo","bar"}),z({"i","j","k","l"})}, - float_cells({x({"a","b"}),y({"foo","bar","baz"})}), {y({"foo","bar"}),z({"i","j","k","l"})}, - {x({"a","b"}),y({"foo","bar","baz"})}, float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), - float_cells({x({"a","b"}),y({"foo","bar","baz"})}), float_cells({y({"foo","bar"}),z({"i","j","k","l"})}), - {x(3),y({"foo", "bar"})}, {y({"foo", "bar"}),z(7)}, - {x({"a","b","c"}),y(5)}, {y(5),z({"i","j","k","l"})}, - float_cells({x({"a","b","c"}),y(5)}), {y(5),z({"i","j","k","l"})}, - {x({"a","b","c"}),y(5)}, float_cells({y(5),z({"i","j","k","l"})}), - float_cells({x({"a","b","c"}),y(5)}), float_cells({y(5),z({"i","j","k","l"})}) +const std::vector<GenSpec> join_layouts = { + G(), G(), + G().idx("x", 5), G().idx("x", 5), + G().idx("x", 5), G().idx("y", 5), + G().idx("x", 5), G().idx("x", 5).idx("y", 5), + G().idx("y", 3), G().idx("x", 2).idx("z", 3), + G().idx("x", 3).idx("y", 5), G().idx("y", 5).idx("z", 7), + G().map("x", {"a","b","c"}), G().map("x", {"a","b","c"}), + G().map("x", {"a","b","c"}), G().map("x", {"a","b"}), + G().map("x", {"a","b","c"}), G().map("y", {"foo","bar","baz"}), + G().map("x", {"a","b","c"}), G().map("x", {"a","b","c"}).map("y", {"foo","bar","baz"}), + G().map("x", {"a","b"}).map("y", {"foo","bar","baz"}), G().map("x", {"a","b","c"}).map("y", {"foo","bar"}), + G().map("x", {"a","b"}).map("y", {"foo","bar","baz"}), G().map("y", {"foo","bar"}).map("z", {"i","j","k","l"}), + G().idx("x", 3).map("y", {"foo", "bar"}), G().map("y", {"foo","bar"}).idx("z", 7), + G().map("x", {"a","b","c"}).idx("y", 5), G().idx("y", 5).map("z", {"i","j","k","l"}) + }; TensorSpec streamed_value_join(const TensorSpec &a, const TensorSpec &b, join_fun_t function) { @@ -76,20 +67,26 @@ TensorSpec streamed_value_join(const TensorSpec &a, const TensorSpec &b, join_fu TEST(StreamedValueTest, streamed_values_can_be_converted_from_and_to_tensor_spec) { for (const auto &layout: layouts) { - TensorSpec expect = spec(layout, N()); - std::unique_ptr<Value> value = value_from_spec(expect, StreamedValueBuilderFactory::get()); - TensorSpec actual = spec_from_value(*value); - EXPECT_EQ(actual, expect); + for (TensorSpec expect : { layout.cpy().cells_float().gen(), + layout.cpy().cells_double().gen() }) + { + std::unique_ptr<Value> value = value_from_spec(expect, StreamedValueBuilderFactory::get()); + TensorSpec actual = spec_from_value(*value); + EXPECT_EQ(actual, expect); + } } } TEST(StreamedValueTest, streamed_values_can_be_copied) { for (const auto &layout: layouts) { - TensorSpec expect = spec(layout, N()); - std::unique_ptr<Value> value = value_from_spec(expect, StreamedValueBuilderFactory::get()); - std::unique_ptr<Value> copy = StreamedValueBuilderFactory::get().copy(*value); - TensorSpec actual = spec_from_value(*copy); - EXPECT_EQ(actual, expect); + for (TensorSpec expect : { layout.cpy().cells_float().gen(), + layout.cpy().cells_double().gen() }) + { + std::unique_ptr<Value> value = value_from_spec(expect, StreamedValueBuilderFactory::get()); + std::unique_ptr<Value> copy = StreamedValueBuilderFactory::get().copy(*value); + TensorSpec actual = spec_from_value(*copy); + EXPECT_EQ(actual, expect); + } } } @@ -126,16 +123,26 @@ TEST(StreamedValueTest, streamed_value_can_be_built_and_inspected) { EXPECT_EQ(result["bb"], 3); } +GenSpec::seq_t N_16ths = [] (size_t i) noexcept { return (i + 1.0) / 16.0; }; + TEST(StreamedValueTest, new_generic_join_works_for_streamed_values) { ASSERT_TRUE((join_layouts.size() % 2) == 0); for (size_t i = 0; i < join_layouts.size(); i += 2) { - TensorSpec lhs = spec(join_layouts[i], Div16(N())); - TensorSpec rhs = spec(join_layouts[i + 1], Div16(N())); - for (auto fun: {operation::Add::f, operation::Sub::f, operation::Mul::f, operation::Max::f}) { - SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); - auto expect = ReferenceOperations::join(lhs, rhs, fun); - auto actual = streamed_value_join(lhs, rhs, fun); - EXPECT_EQ(actual, expect); + const auto l = join_layouts[i].cpy().seq(N_16ths); + const auto r = join_layouts[i + 1].cpy().seq(N_16ths); + for (TensorSpec lhs : { l.cpy().cells_float().gen(), + l.cpy().cells_double().gen() }) + { + for (TensorSpec rhs : { r.cpy().cells_float().gen(), + r.cpy().cells_double().gen() }) + { + for (auto fun: {operation::Add::f, operation::Sub::f, operation::Mul::f, operation::Max::f}) { + SCOPED_TRACE(fmt("\n===\nLHS: %s\nRHS: %s\n===\n", lhs.to_string().c_str(), rhs.to_string().c_str())); + auto expect = ReferenceOperations::join(lhs, rhs, fun); + auto actual = streamed_value_join(lhs, rhs, fun); + EXPECT_EQ(actual, expect); + } + } } } } diff --git a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp index 3345d7dc8ee..000794aca7d 100644 --- a/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp +++ b/eval/src/tests/tensor/instruction_benchmark/instruction_benchmark.cpp @@ -43,6 +43,7 @@ #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/data/smart_buffer.h> +#include <vespa/eval/eval/test/gen_spec.h> #include <optional> #include <algorithm> @@ -60,51 +61,7 @@ template <typename T> using CREF = std::reference_wrapper<const T>; //----------------------------------------------------------------------------- -struct D { - vespalib::string name; - bool mapped; - size_t size; - size_t stride; - static D map(const vespalib::string &name_in, size_t size_in, size_t stride_in) { return D{name_in, true, size_in, stride_in}; } - static D idx(const vespalib::string &name_in, size_t size_in) { return D{name_in, false, size_in, 1}; } - operator ValueType::Dimension() const { - if (mapped) { - return ValueType::Dimension(name); - } else { - return ValueType::Dimension(name, size); - } - } - TensorSpec::Label operator()(size_t idx) const { - if (mapped) { - // need plain number as string for dynamic sparse peek - return TensorSpec::Label(fmt("%zu", idx)); - } else { - return TensorSpec::Label(idx); - } - } -}; - -void add_cells(TensorSpec &spec, double &seq, TensorSpec::Address addr) { - spec.add(addr, seq); - seq += 1.0; -} - -template <typename ...Ds> void add_cells(TensorSpec &spec, double &seq, TensorSpec::Address addr, const D &d, const Ds &...ds) { - for (size_t i = 0, idx = 0; i < d.size; ++i, idx += d.stride) { - addr.insert_or_assign(d.name, d(idx)); - add_cells(spec, seq, addr, ds...); - } -} - -template <typename ...Ds> TensorSpec make_spec(double seq, const Ds &...ds) { - TensorSpec spec(ValueType::tensor_type({ds...}, CellType::FLOAT).to_spec()); - add_cells(spec, seq, TensorSpec::Address(), ds...); - return spec; -} - -TensorSpec make_vector(const D &d1, double seq) { return make_spec(seq, d1); } -TensorSpec make_matrix(const D &d1, const D &d2, double seq) { return make_spec(seq, d1, d2); } -TensorSpec make_cube(const D &d1, const D &d2, const D &d3, double seq) { return make_spec(seq, d1, d2, d3); } +test::GenSpec GS(double bias) { return test::GenSpec().cells_float().seq_bias(bias); } //----------------------------------------------------------------------------- @@ -609,7 +566,7 @@ void benchmark_tensor_create(const vespalib::string &desc, const TensorSpec &pro ASSERT_FALSE(proto_type.is_error()); std::vector<CREF<TensorSpec>> stack_spec; for (const auto &cell: proto.cells()) { - stack_spec.emplace_back(stash.create<TensorSpec>(make_spec(cell.second))); + stack_spec.emplace_back(stash.create<TensorSpec>(GS(cell.second).gen())); } std::vector<EvalOp::UP> list; for (const Impl &impl: impl_list) { @@ -645,7 +602,7 @@ void benchmark_tensor_peek(const vespalib::string &desc, const TensorSpec &lhs, stack_spec.emplace_back(lhs); if (peek_spec.is_dynamic) { for (const auto &entry: peek_spec.spec) { - stack_spec.emplace_back(stash.create<TensorSpec>(make_spec(double(entry.second)))); + stack_spec.emplace_back(stash.create<TensorSpec>(GS(double(entry.second)).gen())); } } std::vector<EvalOp::UP> list; @@ -660,10 +617,10 @@ void benchmark_tensor_peek(const vespalib::string &desc, const TensorSpec &lhs, //----------------------------------------------------------------------------- TEST(MakeInputTest, print_some_test_input) { - auto number = make_spec(5.0); - auto sparse = make_vector(D::map("x", 5, 3), 1.0); - auto dense = make_vector(D::idx("x", 5), 10.0); - auto mixed = make_cube(D::map("x", 3, 7), D::idx("y", 2), D::idx("z", 2), 100.0); + auto number = GS(5.0).gen(); + auto sparse = GS(1.0).map("x", 5, 3).gen(); + auto dense = GS(10.0).idx("x", 5).gen(); + auto mixed = GS(100.0).map("x", 3, 7).idx("y", 2).idx("z", 2).gen(); fprintf(stderr, "--------------------------------------------------------\n"); fprintf(stderr, "simple number: %s\n", number.to_string().c_str()); fprintf(stderr, "sparse vector: %s\n", sparse.to_string().c_str()); @@ -728,197 +685,197 @@ void benchmark_encode_decode(const vespalib::string &desc, const TensorSpec &pro // relevant for the overall performance of the tensor implementation. TEST(EncodeDecodeBench, encode_decode_dense) { - auto proto = make_matrix(D::idx("a", 64), D::idx("b", 64), 1.0); + auto proto = GS(1.0).idx("a", 64).idx("b", 64).gen(); benchmark_encode_decode("dense tensor", proto); } TEST(EncodeDecodeBench, encode_decode_sparse) { - auto proto = make_matrix(D::map("a", 64, 1), D::map("b", 64, 1), 1.0); + auto proto = GS(1.0).map("a", 64, 1).map("b", 64, 1).gen(); benchmark_encode_decode("sparse tensor", proto); } TEST(EncodeDecodeBench, encode_decode_mixed) { - auto proto = make_matrix(D::map("a", 64, 1), D::idx("b", 64), 1.0); + auto proto = GS(1.0).map("a", 64, 1).idx("b", 64).gen(); benchmark_encode_decode("mixed tensor", proto); } //----------------------------------------------------------------------------- TEST(DenseConcat, small_vectors) { - auto lhs = make_vector(D::idx("x", 10), 1.0); - auto rhs = make_vector(D::idx("x", 10), 2.0); + auto lhs = GS(1.0).idx("x", 10).gen(); + auto rhs = GS(2.0).idx("x", 10).gen(); benchmark_concat("small dense vector append concat", lhs, rhs, "x"); } TEST(DenseConcat, cross_vectors) { - auto lhs = make_vector(D::idx("x", 10), 1.0); - auto rhs = make_vector(D::idx("x", 10), 2.0); + auto lhs = GS(1.0).idx("x", 10).gen(); + auto rhs = GS(2.0).idx("x", 10).gen(); benchmark_concat("small dense vector cross concat", lhs, rhs, "y"); } TEST(DenseConcat, cube_and_vector) { - auto lhs = make_cube(D::idx("a", 16), D::idx("b", 16), D::idx("c", 16), 1.0); - auto rhs = make_vector(D::idx("a", 16), 42.0); + auto lhs = GS(1.0).idx("a", 16).idx("b", 16).idx("c", 16).gen(); + auto rhs = GS(42.0).idx("a", 16).gen(); benchmark_concat("cube vs vector concat", lhs, rhs, "a"); } TEST(SparseConcat, small_vectors) { - auto lhs = make_vector(D::map("x", 10, 1), 1.0); - auto rhs = make_vector(D::map("x", 10, 2), 2.0); + auto lhs = GS(1.0).map("x", 10, 1).gen(); + auto rhs = GS(2.0).map("x", 10, 2).gen(); benchmark_concat("small sparse concat", lhs, rhs, "y"); } TEST(MixedConcat, mixed_vs_dense) { - auto lhs = make_cube(D::idx("a", 16), D::idx("b", 16), D::map("c", 16, 1), 1.0); - auto rhs = make_matrix(D::idx("a", 16), D::idx("b", 16), 2.0); + auto lhs = GS(1.0).idx("a", 16).idx("b", 16).map("c", 16, 1).gen(); + auto rhs = GS(2.0).idx("a", 16).idx("b", 16).gen(); benchmark_concat("mixed dense concat a", lhs, rhs, "a"); } TEST(MixedConcat, large_mixed_a) { - auto lhs = make_cube(D::idx("a", 16), D::idx("b", 16), D::map("c", 16, 1), 1.0); - auto rhs = make_cube(D::idx("a", 16), D::idx("b", 16), D::map("c", 16, 2), 2.0); + auto lhs = GS(1.0).idx("a", 16).idx("b", 16).map("c", 16, 1).gen(); + auto rhs = GS(2.0).idx("a", 16).idx("b", 16).map("c", 16, 2).gen(); benchmark_concat("mixed append concat a", lhs, rhs, "a"); } TEST(MixedConcat, large_mixed_b) { - auto lhs = make_cube(D::idx("a", 16), D::idx("b", 16), D::map("c", 16, 1), 1.0); - auto rhs = make_cube(D::idx("a", 16), D::idx("b", 16), D::map("c", 16, 2), 2.0); + auto lhs = GS(1.0).idx("a", 16).idx("b", 16).map("c", 16, 1).gen(); + auto rhs = GS(2.0).idx("a", 16).idx("b", 16).map("c", 16, 2).gen(); benchmark_concat("mixed append concat b", lhs, rhs, "b"); } //----------------------------------------------------------------------------- TEST(NumberJoin, plain_op2) { - auto lhs = make_spec(2.0); - auto rhs = make_spec(3.0); + auto lhs = GS(2.0).gen(); + auto rhs = GS(3.0).gen(); benchmark_join("simple numbers multiply", lhs, rhs, operation::Mul::f); } //----------------------------------------------------------------------------- TEST(DenseJoin, small_vectors) { - auto lhs = make_vector(D::idx("x", 10), 1.0); - auto rhs = make_vector(D::idx("x", 10), 2.0); + auto lhs = GS(1.0).idx("x", 10).gen(); + auto rhs = GS(2.0).idx("x", 10).gen(); benchmark_join("small dense vector multiply", lhs, rhs, operation::Mul::f); } TEST(DenseJoin, full_overlap) { - auto lhs = make_cube(D::idx("a", 16), D::idx("b", 16), D::idx("c", 16), 1.0); - auto rhs = make_cube(D::idx("a", 16), D::idx("b", 16), D::idx("c", 16), 2.0); + auto lhs = GS(1.0).idx("a", 16).idx("b", 16).idx("c", 16).gen(); + auto rhs = GS(2.0).idx("a", 16).idx("b", 16).idx("c", 16).gen(); benchmark_join("dense full overlap multiply", lhs, rhs, operation::Mul::f); } TEST(DenseJoin, partial_overlap) { - auto lhs = make_cube(D::idx("a", 8), D::idx("c", 8), D::idx("d", 8), 1.0); - auto rhs = make_cube(D::idx("b", 8), D::idx("c", 8), D::idx("d", 8), 2.0); + auto lhs = GS(1.0).idx("a", 8).idx("c", 8).idx("d", 8).gen(); + auto rhs = GS(2.0).idx("b", 8).idx("c", 8).idx("d", 8).gen(); benchmark_join("dense partial overlap multiply", lhs, rhs, operation::Mul::f); } TEST(DenseJoin, subset_overlap) { - auto lhs = make_cube(D::idx("a", 16), D::idx("b", 16), D::idx("c", 16), 1.0); - auto rhs_inner = make_matrix(D::idx("b", 16), D::idx("c", 16), 2.0); - auto rhs_outer = make_matrix(D::idx("a", 16), D::idx("b", 16), 3.0); + auto lhs = GS(1.0).idx("a", 16).idx("b", 16).idx("c", 16).gen(); + auto rhs_inner = GS(2.0).idx("b", 16).idx("c", 16).gen(); + auto rhs_outer = GS(3.0).idx("a", 16).idx("b", 16).gen(); benchmark_join("dense subset overlap inner multiply", lhs, rhs_inner, operation::Mul::f); benchmark_join("dense subset overlap outer multiply", lhs, rhs_outer, operation::Mul::f); } TEST(DenseJoin, no_overlap) { - auto lhs = make_cube(D::idx("a", 4), D::idx("e", 4), D::idx("f", 4), 1.0); - auto rhs = make_cube(D::idx("b", 4), D::idx("c", 4), D::idx("d", 4), 2.0); + auto lhs = GS(1.0).idx("a", 4).idx("e", 4).idx("f", 4).gen(); + auto rhs = GS(2.0).idx("b", 4).idx("c", 4).idx("d", 4).gen(); benchmark_join("dense no overlap multiply", lhs, rhs, operation::Mul::f); } TEST(DenseJoin, simple_expand) { - auto lhs = make_cube(D::idx("a", 5), D::idx("b", 4), D::idx("c", 4), 1.0); - auto rhs = make_cube(D::idx("d", 4), D::idx("e", 4), D::idx("f", 5), 2.0); + auto lhs = GS(1.0).idx("a", 5).idx("b", 4).idx("c", 4).gen(); + auto rhs = GS(2.0).idx("d", 4).idx("e", 4).idx("f", 5).gen(); benchmark_join("dense simple expand multiply", lhs, rhs, operation::Mul::f); } TEST(DenseJoin, multiply_by_number) { - auto lhs = make_spec(3.0); - auto rhs = make_cube(D::idx("a", 16), D::idx("b", 16), D::idx("c", 16), 2.0); + auto lhs = GS(3.0).gen(); + auto rhs = GS(2.0).idx("a", 16).idx("b", 16).idx("c", 16).gen(); benchmark_join("dense cube multiply by number", lhs, rhs, operation::Mul::f); } //----------------------------------------------------------------------------- TEST(SparseJoin, small_vectors) { - auto lhs = make_vector(D::map("x", 10, 1), 1.0); - auto rhs = make_vector(D::map("x", 10, 2), 2.0); + auto lhs = GS(1.0).map("x", 10, 1).gen(); + auto rhs = GS(2.0).map("x", 10, 2).gen(); benchmark_join("small sparse vector multiply", lhs, rhs, operation::Mul::f); } TEST(SparseJoin, large_vectors) { - auto lhs = make_vector(D::map("x", 1800, 1), 1.0); - auto rhs = make_vector(D::map("x", 1000, 2), 2.0); + auto lhs = GS(1.0).map("x", 1800, 1).gen(); + auto rhs = GS(2.0).map("x", 1000, 2).gen(); benchmark_join("large sparse vector multiply", lhs, rhs, operation::Mul::f); } TEST(SparseJoin, full_overlap) { - auto lhs = make_cube(D::map("a", 16, 1), D::map("b", 16, 1), D::map("c", 16, 1), 1.0); - auto rhs = make_cube(D::map("a", 16, 2), D::map("b", 16, 2), D::map("c", 16, 2), 2.0); + auto lhs = GS(1.0).map("a", 16, 1).map("b", 16, 1).map("c", 16, 1).gen(); + auto rhs = GS(2.0).map("a", 16, 2).map("b", 16, 2).map("c", 16, 2).gen(); benchmark_join("sparse full overlap multiply", lhs, rhs, operation::Mul::f); } TEST(SparseJoin, full_overlap_big_vs_small) { - auto lhs = make_cube(D::map("a", 16, 1), D::map("b", 16, 1), D::map("c", 16, 1), 1.0); - auto rhs = make_cube(D::map("a", 2, 1), D::map("b", 2, 1), D::map("c", 2, 1), 2.0); + auto lhs = GS(1.0).map("a", 16, 1).map("b", 16, 1).map("c", 16, 1).gen(); + auto rhs = GS(2.0).map("a", 2, 1).map("b", 2, 1).map("c", 2, 1).gen(); benchmark_join("sparse full overlap big vs small multiply", lhs, rhs, operation::Mul::f); } TEST(SparseJoin, partial_overlap) { - auto lhs = make_cube(D::map("a", 8, 1), D::map("c", 8, 1), D::map("d", 8, 1), 1.0); - auto rhs = make_cube(D::map("b", 8, 2), D::map("c", 8, 2), D::map("d", 8, 2), 2.0); + auto lhs = GS(1.0).map("a", 8, 1).map("c", 8, 1).map("d", 8, 1).gen(); + auto rhs = GS(2.0).map("b", 8, 2).map("c", 8, 2).map("d", 8, 2).gen(); benchmark_join("sparse partial overlap multiply", lhs, rhs, operation::Mul::f); } TEST(SparseJoin, no_overlap) { - auto lhs = make_cube(D::map("a", 4, 1), D::map("e", 4, 1), D::map("f", 4, 1), 1.0); - auto rhs = make_cube(D::map("b", 4, 1), D::map("c", 4, 1), D::map("d", 4, 1), 2.0); + auto lhs = GS(1.0).map("a", 4, 1).map("e", 4, 1).map("f", 4, 1).gen(); + auto rhs = GS(2.0).map("b", 4, 1).map("c", 4, 1).map("d", 4, 1).gen(); benchmark_join("sparse no overlap multiply", lhs, rhs, operation::Mul::f); } TEST(SparseJoin, multiply_by_number) { - auto lhs = make_spec(3.0); - auto rhs = make_cube(D::map("a", 16, 2), D::map("b", 16, 2), D::map("c", 16, 2), 2.0); + auto lhs = GS(3.0).gen(); + auto rhs = GS(2.0).map("a", 16, 2).map("b", 16, 2).map("c", 16, 2).gen(); benchmark_join("sparse multiply by number", lhs, rhs, operation::Mul::f); } //----------------------------------------------------------------------------- TEST(MixedJoin, full_overlap) { - auto lhs = make_cube(D::map("a", 16, 1), D::map("b", 16, 1), D::idx("c", 16), 1.0); - auto rhs = make_cube(D::map("a", 16, 2), D::map("b", 16, 2), D::idx("c", 16), 2.0); + auto lhs = GS(1.0).map("a", 16, 1).map("b", 16, 1).idx("c", 16).gen(); + auto rhs = GS(2.0).map("a", 16, 2).map("b", 16, 2).idx("c", 16).gen(); benchmark_join("mixed full overlap multiply", lhs, rhs, operation::Mul::f); } TEST(MixedJoin, partial_sparse_overlap) { - auto lhs = make_cube(D::map("a", 8, 1), D::map("c", 8, 1), D::idx("d", 8), 1.0); - auto rhs = make_cube(D::map("b", 8, 2), D::map("c", 8, 2), D::idx("d", 8), 2.0); + auto lhs = GS(1.0).map("a", 8, 1).map("c", 8, 1).idx("d", 8).gen(); + auto rhs = GS(2.0).map("b", 8, 2).map("c", 8, 2).idx("d", 8).gen(); benchmark_join("mixed partial sparse overlap multiply", lhs, rhs, operation::Mul::f); } TEST(MixedJoin, no_overlap) { - auto lhs = make_cube(D::map("a", 4, 1), D::map("e", 4, 1), D::idx("f", 4), 1.0); - auto rhs = make_cube(D::map("b", 4, 1), D::map("c", 4, 1), D::idx("d", 4), 2.0); + auto lhs = GS(1.0).map("a", 4, 1).map("e", 4, 1).idx("f", 4).gen(); + auto rhs = GS(2.0).map("b", 4, 1).map("c", 4, 1).idx("d", 4).gen(); benchmark_join("mixed no overlap multiply", lhs, rhs, operation::Mul::f); } TEST(MixedJoin, multiply_by_number) { - auto lhs = make_spec(3.0); - auto rhs = make_cube(D::map("a", 16, 2), D::map("b", 16, 2), D::idx("c", 16), 2.0); + auto lhs = GS(3.0).gen(); + auto rhs = GS(2.0).map("a", 16, 2).map("b", 16, 2).idx("c", 16).gen(); benchmark_join("mixed multiply by number", lhs, rhs, operation::Mul::f); } //----------------------------------------------------------------------------- TEST(ReduceBench, number_reduce) { - auto lhs = make_spec(1.0); + auto lhs = GS(1.0).gen(); benchmark_reduce("number reduce", lhs, Aggr::SUM, {}); } TEST(ReduceBench, dense_reduce) { - auto lhs = make_cube(D::idx("a", 16), D::idx("b", 16), D::idx("c", 16), 1.0); + auto lhs = GS(1.0).idx("a", 16).idx("b", 16).idx("c", 16).gen(); benchmark_reduce("dense reduce inner", lhs, Aggr::SUM, {"c"}); benchmark_reduce("dense reduce middle", lhs, Aggr::SUM, {"b"}); benchmark_reduce("dense reduce outer", lhs, Aggr::SUM, {"a"}); @@ -929,7 +886,7 @@ TEST(ReduceBench, dense_reduce) { } TEST(ReduceBench, sparse_reduce) { - auto lhs = make_cube(D::map("a", 16, 1), D::map("b", 16, 1), D::map("c", 16, 1), 1.0); + auto lhs = GS(1.0).map("a", 16, 1).map("b", 16, 1).map("c", 16, 1).gen(); benchmark_reduce("sparse reduce inner", lhs, Aggr::SUM, {"c"}); benchmark_reduce("sparse reduce middle", lhs, Aggr::SUM, {"b"}); benchmark_reduce("sparse reduce outer", lhs, Aggr::SUM, {"a"}); @@ -940,8 +897,8 @@ TEST(ReduceBench, sparse_reduce) { } TEST(ReduceBench, mixed_reduce) { - auto lhs = make_spec(1.0, D::map("a", 4, 1), D::map("b", 4, 1), D::map("c", 4, 1), - D::idx("d", 4), D::idx("e", 4), D::idx("f", 4)); + auto lhs = GS(1.0).map("a", 4, 1).map("b", 4, 1).map("c", 4, 1) + .idx("d", 4).idx("e", 4).idx("f", 4).gen(); benchmark_reduce("mixed reduce middle dense", lhs, Aggr::SUM, {"e"}); benchmark_reduce("mixed reduce middle sparse", lhs, Aggr::SUM, {"b"}); benchmark_reduce("mixed reduce middle sparse/dense", lhs, Aggr::SUM, {"b", "e"}); @@ -953,87 +910,87 @@ TEST(ReduceBench, mixed_reduce) { //----------------------------------------------------------------------------- TEST(RenameBench, dense_rename) { - auto lhs = make_matrix(D::idx("a", 64), D::idx("b", 64), 1.0); + auto lhs = GS(1.0).idx("a", 64).idx("b", 64).gen(); benchmark_rename("dense transpose", lhs, {"a", "b"}, {"b", "a"}); } TEST(RenameBench, sparse_rename) { - auto lhs = make_matrix(D::map("a", 64, 1), D::map("b", 64, 1), 1.0); + auto lhs = GS(1.0).map("a", 64, 1).map("b", 64, 1).gen(); benchmark_rename("sparse transpose", lhs, {"a", "b"}, {"b", "a"}); } TEST(RenameBench, mixed_rename) { - auto lhs = make_spec(1.0, D::map("a", 8, 1), D::map("b", 8, 1), D::idx("c", 8), D::idx("d", 8)); + auto lhs = GS(1.0).map("a", 8, 1).map("b", 8, 1).idx("c", 8).idx("d", 8).gen(); benchmark_rename("mixed multi-transpose", lhs, {"a", "b", "c", "d"}, {"b", "a", "d", "c"}); } //----------------------------------------------------------------------------- TEST(MergeBench, dense_merge) { - auto lhs = make_matrix(D::idx("a", 64), D::idx("b", 64), 1.0); - auto rhs = make_matrix(D::idx("a", 64), D::idx("b", 64), 2.0); + auto lhs = GS(1.0).idx("a", 64).idx("b", 64).gen(); + auto rhs = GS(2.0).idx("a", 64).idx("b", 64).gen(); benchmark_merge("dense merge", lhs, rhs, operation::Max::f); } TEST(MergeBench, sparse_merge_big_small) { - auto lhs = make_matrix(D::map("a", 64, 1), D::map("b", 64, 1), 1.0); - auto rhs = make_matrix(D::map("a", 8, 1), D::map("b", 8, 1), 2.0); + auto lhs = GS(1.0).map("a", 64, 1).map("b", 64, 1).gen(); + auto rhs = GS(2.0).map("a", 8, 1).map("b", 8, 1).gen(); benchmark_merge("sparse merge big vs small", lhs, rhs, operation::Max::f); } TEST(MergeBench, sparse_merge_minimal_overlap) { - auto lhs = make_matrix(D::map("a", 64, 11), D::map("b", 32, 11), 1.0); - auto rhs = make_matrix(D::map("a", 32, 13), D::map("b", 64, 13), 2.0); + auto lhs = GS(1.0).map("a", 64, 11).map("b", 32, 11).gen(); + auto rhs = GS(2.0).map("a", 32, 13).map("b", 64, 13).gen(); benchmark_merge("sparse merge minimal overlap", lhs, rhs, operation::Max::f); } TEST(MergeBench, mixed_merge) { - auto lhs = make_matrix(D::map("a", 64, 1), D::idx("b", 64), 1.0); - auto rhs = make_matrix(D::map("a", 64, 2), D::idx("b", 64), 2.0); + auto lhs = GS(1.0).map("a", 64, 1).idx("b", 64).gen(); + auto rhs = GS(2.0).map("a", 64, 2).idx("b", 64).gen(); benchmark_merge("mixed merge", lhs, rhs, operation::Max::f); } //----------------------------------------------------------------------------- TEST(MapBench, number_map) { - auto lhs = make_spec(1.75); + auto lhs = GS(1.75).gen(); benchmark_map("number map", lhs, operation::Floor::f); } TEST(MapBench, dense_map) { - auto lhs = make_matrix(D::idx("a", 64), D::idx("b", 64), 1.75); + auto lhs = GS(1.75).idx("a", 64).idx("b", 64).gen(); benchmark_map("dense map", lhs, operation::Floor::f); } TEST(MapBench, sparse_map_small) { - auto lhs = make_matrix(D::map("a", 4, 1), D::map("b", 4, 1), 1.75); + auto lhs = GS(1.75).map("a", 4, 1).map("b", 4, 1).gen(); benchmark_map("sparse map small", lhs, operation::Floor::f); } TEST(MapBench, sparse_map_big) { - auto lhs = make_matrix(D::map("a", 64, 1), D::map("b", 64, 1), 1.75); + auto lhs = GS(1.75).map("a", 64, 1).map("b", 64, 1).gen(); benchmark_map("sparse map big", lhs, operation::Floor::f); } TEST(MapBench, mixed_map) { - auto lhs = make_matrix(D::map("a", 64, 1), D::idx("b", 64), 1.75); + auto lhs = GS(1.75).map("a", 64, 1).idx("b", 64).gen(); benchmark_map("mixed map", lhs, operation::Floor::f); } //----------------------------------------------------------------------------- TEST(TensorCreateBench, create_dense) { - auto proto = make_matrix(D::idx("a", 32), D::idx("b", 32), 1.0); + auto proto = GS(1.0).idx("a", 32).idx("b", 32).gen(); benchmark_tensor_create("dense tensor create", proto); } TEST(TensorCreateBench, create_sparse) { - auto proto = make_matrix(D::map("a", 32, 1), D::map("b", 32, 1), 1.0); + auto proto = GS(1.0).map("a", 32, 1).map("b", 32, 1).gen(); benchmark_tensor_create("sparse tensor create", proto); } TEST(TensorCreateBench, create_mixed) { - auto proto = make_matrix(D::map("a", 32, 1), D::idx("b", 32), 1.0); + auto proto = GS(1.0).map("a", 32, 1).idx("b", 32).gen(); benchmark_tensor_create("mixed tensor create", proto); } @@ -1041,7 +998,7 @@ TEST(TensorCreateBench, create_mixed) { TEST(TensorLambdaBench, simple_lambda) { auto type = ValueType::from_spec("tensor<float>(a[64],b[64])"); - auto p0 = make_spec(3.5); + auto p0 = GS(3.5).gen(); auto function = Function::parse({"a", "b", "p0"}, "(a*64+b)*p0"); ASSERT_FALSE(function->has_error()); benchmark_tensor_lambda("simple tensor lambda", type, p0, *function); @@ -1049,7 +1006,7 @@ TEST(TensorLambdaBench, simple_lambda) { TEST(TensorLambdaBench, complex_lambda) { auto type = ValueType::from_spec("tensor<float>(a[64],b[64])"); - auto p0 = make_vector(D::idx("x", 3), 1.0); + auto p0 = GS(1.0).idx("x", 3).gen(); auto function = Function::parse({"a", "b", "p0"}, "(a*64+b)*reduce(p0,sum)"); ASSERT_FALSE(function->has_error()); benchmark_tensor_lambda("complex tensor lambda", type, p0, *function); @@ -1058,7 +1015,7 @@ TEST(TensorLambdaBench, complex_lambda) { //----------------------------------------------------------------------------- TEST(TensorPeekBench, dense_peek) { - auto lhs = make_matrix(D::idx("a", 64), D::idx("b", 64), 1.0); + auto lhs = GS(1.0).idx("a", 64).idx("b", 64).gen(); benchmark_tensor_peek("dense peek cell verbatim", lhs, verbatim_peek().add("a", 1).add("b", 2)); benchmark_tensor_peek("dense peek cell dynamic", lhs, dynamic_peek().add("a", 1).add("b", 2)); benchmark_tensor_peek("dense peek vector verbatim", lhs, verbatim_peek().add("a", 1)); @@ -1066,7 +1023,7 @@ TEST(TensorPeekBench, dense_peek) { } TEST(TensorPeekBench, sparse_peek) { - auto lhs = make_matrix(D::map("a", 64, 1), D::map("b", 64, 1), 1.0); + auto lhs = GS(1.0).map("a", 64, 1).map("b", 64, 1).gen(); benchmark_tensor_peek("sparse peek cell verbatim", lhs, verbatim_peek().add("a", 1).add("b", 2)); benchmark_tensor_peek("sparse peek cell dynamic", lhs, dynamic_peek().add("a", 1).add("b", 2)); benchmark_tensor_peek("sparse peek vector verbatim", lhs, verbatim_peek().add("a", 1)); @@ -1074,7 +1031,7 @@ TEST(TensorPeekBench, sparse_peek) { } TEST(TensorPeekBench, mixed_peek) { - auto lhs = make_spec(1.0, D::map("a", 8, 1), D::map("b", 8, 1), D::idx("c", 8), D::idx("d", 8)); + auto lhs = GS(1.0).map("a", 8, 1).map("b", 8, 1).idx("c", 8).idx("d", 8).gen(); benchmark_tensor_peek("mixed peek cell verbatim", lhs, verbatim_peek().add("a", 1).add("b", 2).add("c", 3).add("d", 4)); benchmark_tensor_peek("mixed peek cell dynamic", lhs, dynamic_peek().add("a", 1).add("b", 2).add("c", 3).add("d", 4)); benchmark_tensor_peek("mixed peek dense verbatim", lhs, verbatim_peek().add("a", 1).add("b", 2)); diff --git a/eval/src/vespa/eval/eval/optimize_tensor_function.cpp b/eval/src/vespa/eval/eval/optimize_tensor_function.cpp index 2e8c89f88fc..25612b8d5fd 100644 --- a/eval/src/vespa/eval/eval/optimize_tensor_function.cpp +++ b/eval/src/vespa/eval/eval/optimize_tensor_function.cpp @@ -5,6 +5,7 @@ #include "simple_value.h" #include <vespa/eval/instruction/dense_dot_product_function.h> +#include <vespa/eval/instruction/sparse_dot_product_function.h> #include <vespa/eval/instruction/mixed_inner_product_function.h> #include <vespa/eval/instruction/sum_max_dot_product_function.h> #include <vespa/eval/instruction/dense_xw_product_function.h> @@ -31,11 +32,7 @@ namespace vespalib::eval { namespace { -const TensorFunction &optimize_for_factory(const ValueBuilderFactory &factory, const TensorFunction &expr, Stash &stash) { - if (&factory == &SimpleValueBuilderFactory::get()) { - // never optimize simple value evaluation - return expr; - } +const TensorFunction &optimize_for_factory(const ValueBuilderFactory &, const TensorFunction &expr, Stash &stash) { using Child = TensorFunction::Child; Child root(expr); { @@ -47,6 +44,7 @@ const TensorFunction &optimize_for_factory(const ValueBuilderFactory &factory, c const Child &child = nodes.back().get(); child.set(SumMaxDotProductFunction::optimize(child.get(), stash)); child.set(DenseDotProductFunction::optimize(child.get(), stash)); + child.set(SparseDotProductFunction::optimize(child.get(), stash)); child.set(DenseXWProductFunction::optimize(child.get(), stash)); child.set(DenseMatMulFunction::optimize(child.get(), stash)); child.set(DenseMultiMatMulFunction::optimize(child.get(), stash)); diff --git a/eval/src/vespa/eval/eval/test/CMakeLists.txt b/eval/src/vespa/eval/eval/test/CMakeLists.txt index 2e9b50da5e6..e82b85d1890 100644 --- a/eval/src/vespa/eval/eval/test/CMakeLists.txt +++ b/eval/src/vespa/eval/eval/test/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(eval_eval_test OBJECT SOURCES eval_fixture.cpp eval_spec.cpp + gen_spec.cpp reference_evaluation.cpp reference_operations.cpp tensor_conformance.cpp diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.cpp b/eval/src/vespa/eval/eval/test/eval_fixture.cpp index 58d8905baf3..966954b9026 100644 --- a/eval/src/vespa/eval/eval/test/eval_fixture.cpp +++ b/eval/src/vespa/eval/eval/test/eval_fixture.cpp @@ -28,7 +28,10 @@ NodeTypes get_types(const Function &function, const ParamRepo ¶m_repo) { std::vector<ValueType> param_types; for (size_t i = 0; i < function.num_params(); ++i) { auto pos = param_repo.map.find(function.param_name(i)); - ASSERT_TRUE(pos != param_repo.map.end()); + if (pos == param_repo.map.end()) { + TEST_STATE(fmt("param name: '%s'", function.param_name(i).data()).c_str()); + ASSERT_TRUE(pos != param_repo.map.end()); + } param_types.push_back(ValueType::from_spec(pos->second.value.type())); ASSERT_TRUE(!param_types.back().is_error()); } @@ -181,6 +184,23 @@ EvalFixture::ParamRepo::add_dense(const std::vector<std::pair<vespalib::string, return *this; } +// produce 4 variants: float/double * mutable/const +EvalFixture::ParamRepo & +EvalFixture::ParamRepo::add_variants(const vespalib::string &name_base, + const GenSpec &spec) +{ + auto name_f = name_base + "_f"; + auto name_m = "@" + name_base; + auto name_m_f = "@" + name_base + "_f"; + auto dbl_ts = spec.cpy().cells_double().gen(); + auto flt_ts = spec.cpy().cells_float().gen(); + add(name_base, dbl_ts); + add(name_f, flt_ts); + add_mutable(name_m, dbl_ts); + add_mutable(name_m_f, flt_ts); + return *this; +} + void EvalFixture::detect_param_tampering(const ParamRepo ¶m_repo, bool allow_mutable) const { diff --git a/eval/src/vespa/eval/eval/test/eval_fixture.h b/eval/src/vespa/eval/eval/test/eval_fixture.h index dc49cf7e4dc..44adaca3298 100644 --- a/eval/src/vespa/eval/eval/test/eval_fixture.h +++ b/eval/src/vespa/eval/eval/test/eval_fixture.h @@ -10,6 +10,7 @@ #include <vespa/vespalib/util/stash.h> #include <set> #include <functional> +#include "gen_spec.h" namespace vespalib::eval::test { @@ -40,6 +41,10 @@ public: ParamRepo &add_matrix(const char *d1, size_t s1, const char *d2, size_t s2, gen_fun_t gen = gen_N); ParamRepo &add_cube(const char *d1, size_t s1, const char *d2, size_t s2, const char *d3, size_t s3, gen_fun_t gen = gen_N); ParamRepo &add_dense(const std::vector<std::pair<vespalib::string, size_t> > &dims, gen_fun_t gen = gen_N); + + // produce 4 variants: float/double * mutable/const + ParamRepo &add_variants(const vespalib::string &name_base, + const GenSpec &spec); ~ParamRepo() {} }; diff --git a/eval/src/vespa/eval/eval/test/gen_spec.cpp b/eval/src/vespa/eval/eval/test/gen_spec.cpp new file mode 100644 index 00000000000..c20e9005318 --- /dev/null +++ b/eval/src/vespa/eval/eval/test/gen_spec.cpp @@ -0,0 +1,63 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "gen_spec.h" +#include <vespa/eval/eval/string_stuff.h> +#include <vespa/vespalib/util/stringfmt.h> + +using vespalib::make_string_short::fmt; + +namespace vespalib::eval::test { + +DimSpec::~DimSpec() = default; + +std::vector<vespalib::string> +DimSpec::make_dict(size_t size, size_t stride, const vespalib::string &prefix) +{ + std::vector<vespalib::string> dict; + for (size_t i = 0; i < size; ++i) { + dict.push_back(fmt("%s%zu", prefix.c_str(), i * stride)); + } + return dict; +} + +GenSpec::GenSpec(GenSpec &&other) = default; +GenSpec::GenSpec(const GenSpec &other) = default; +GenSpec &GenSpec::operator=(GenSpec &&other) = default; +GenSpec &GenSpec::operator=(const GenSpec &other) = default; + +GenSpec::~GenSpec() = default; + +ValueType +GenSpec::type() const +{ + std::vector<ValueType::Dimension> dim_types; + for (const auto &dim: _dims) { + dim_types.push_back(dim.type()); + } + auto type = ValueType::tensor_type(dim_types, _cells); + assert(!type.is_error()); + return type; +} + +TensorSpec +GenSpec::gen() const +{ + size_t idx = 0; + TensorSpec::Address addr; + TensorSpec result(type().to_spec()); + std::function<void(size_t)> add_cells = [&](size_t dim_idx) { + if (dim_idx == _dims.size()) { + result.add(addr, _seq(idx++)); + } else { + const auto &dim = _dims[dim_idx]; + for (size_t i = 0; i < dim.size(); ++i) { + addr.insert_or_assign(dim.name(), dim.label(i)); + add_cells(dim_idx + 1); + } + } + }; + add_cells(0); + return result; +} + +} // namespace diff --git a/eval/src/vespa/eval/eval/test/gen_spec.h b/eval/src/vespa/eval/eval/test/gen_spec.h new file mode 100644 index 00000000000..36bbd554125 --- /dev/null +++ b/eval/src/vespa/eval/eval/test/gen_spec.h @@ -0,0 +1,109 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/value_type.h> +#include <functional> +#include <cassert> + +namespace vespalib::eval::test { + +/** + * Type and labels for a single dimension of a TensorSpec to be + * generated. Dimensions are specified independent of each other for + * simplicity. All dense subspaces will be padded during conversion to + * actual values, which means that indexed dimensions are inherently + * independent already. Using different labels for the same mapped + * dimension for different tensors should enable us to exhibit + * sufficient levels of partial overlap. + **/ +class DimSpec +{ +private: + vespalib::string _name; + size_t _size; + std::vector<vespalib::string> _dict; +public: + DimSpec(const vespalib::string &name, size_t size) noexcept + : _name(name), _size(size), _dict() + { + assert(_size); + } + DimSpec(const vespalib::string &name, std::vector<vespalib::string> dict) noexcept + : _name(name), _size(), _dict(std::move(dict)) + { + assert(!_size); + } + ~DimSpec(); + static std::vector<vespalib::string> make_dict(size_t size, size_t stride, const vespalib::string &prefix); + ValueType::Dimension type() const { + return _size ? ValueType::Dimension{_name, uint32_t(_size)} : ValueType::Dimension{_name}; + } + const vespalib::string &name() const { return _name; } + size_t size() const { + return _size ? _size : _dict.size(); + } + TensorSpec::Label label(size_t idx) const { + assert(idx < size()); + return _size ? TensorSpec::Label{idx} : TensorSpec::Label{_dict[idx]}; + } +}; + +/** + * Specification defining how to generate a TensorSpec. Typically used + * to generate complex values for testing and benchmarking. + **/ +class GenSpec +{ +public: + using seq_t = std::function<double(size_t)>; +private: + std::vector<DimSpec> _dims; + CellType _cells; + seq_t _seq; + + static double default_seq(size_t idx) { return (idx + 1.0); } +public: + GenSpec() : _dims(), _cells(CellType::DOUBLE), _seq(default_seq) {} + GenSpec(GenSpec &&other); + GenSpec(const GenSpec &other); + GenSpec &operator=(GenSpec &&other); + GenSpec &operator=(const GenSpec &other); + ~GenSpec(); + std::vector<DimSpec> dims() const { return _dims; } + CellType cells() const { return _cells; } + seq_t seq() const { return _seq; } + GenSpec cpy() const { return *this; } + GenSpec &idx(const vespalib::string &name, size_t size) { + _dims.emplace_back(name, size); + return *this; + } + GenSpec &map(const vespalib::string &name, size_t size, size_t stride = 1, const vespalib::string &prefix = "") { + _dims.emplace_back(name, DimSpec::make_dict(size, stride, prefix)); + return *this; + } + GenSpec &map(const vespalib::string &name, std::vector<vespalib::string> dict) { + _dims.emplace_back(name, std::move(dict)); + return *this; + } + GenSpec &cells(CellType cell_type) { + _cells = cell_type; + return *this; + } + GenSpec &cells_double() { return cells(CellType::DOUBLE); } + GenSpec &cells_float() { return cells(CellType::FLOAT); } + GenSpec &seq(seq_t seq_in) { + _seq = seq_in; + return *this; + } + GenSpec &seq_n() { return seq(default_seq); } + GenSpec &seq_bias(double bias) { + seq_t fun = [bias](size_t idx) noexcept { return (idx + bias); }; + return seq(fun); + } + ValueType type() const; + TensorSpec gen() const; +}; + +} // namespace diff --git a/eval/src/vespa/eval/eval/test/param_variants.h b/eval/src/vespa/eval/eval/test/param_variants.h deleted file mode 100644 index 41a43ebca08..00000000000 --- a/eval/src/vespa/eval/eval/test/param_variants.h +++ /dev/null @@ -1,23 +0,0 @@ -#include "eval_fixture.h" -#include "tensor_model.hpp" - -namespace vespalib::eval::test { - -// for testing of optimizers / tensor functions -// we produce the same param three times: -// as-is, with float cells, and tagged as mutable. -void add_variants(EvalFixture::ParamRepo &repo, - const vespalib::string &name_base, - const Layout &base_layout, - const Sequence &seq) -{ - auto name_f = name_base + "_f"; - auto name_m = "@" + name_base; - auto name_m_f = "@" + name_base + "_f"; - repo.add(name_base, spec(base_layout, seq)); - repo.add(name_f, spec(float_cells(base_layout), seq)); - repo.add_mutable(name_m, spec(base_layout, seq)); - repo.add_mutable(name_m_f, spec(float_cells(base_layout), seq)); -} - -} // namespace diff --git a/eval/src/vespa/eval/instruction/CMakeLists.txt b/eval/src/vespa/eval/instruction/CMakeLists.txt index 58d5290f5d9..cac69d23640 100644 --- a/eval/src/vespa/eval/instruction/CMakeLists.txt +++ b/eval/src/vespa/eval/instruction/CMakeLists.txt @@ -32,6 +32,7 @@ vespa_add_library(eval_instruction OBJECT pow_as_map_optimizer.cpp remove_trivial_dimension_optimizer.cpp replace_type_function.cpp + sparse_dot_product_function.cpp sum_max_dot_product_function.cpp vector_from_doubles_function.cpp ) diff --git a/eval/src/vespa/eval/instruction/generic_join.cpp b/eval/src/vespa/eval/instruction/generic_join.cpp index abe29b8228c..6d6f86b7c4d 100644 --- a/eval/src/vespa/eval/instruction/generic_join.cpp +++ b/eval/src/vespa/eval/instruction/generic_join.cpp @@ -308,6 +308,17 @@ SparseJoinPlan::SparseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_t [](const auto &a, const auto &b){ return (a.name < b.name); }); } +SparseJoinPlan::SparseJoinPlan(size_t num_mapped_dims) + : sources(num_mapped_dims, Source::BOTH), lhs_overlap(), rhs_overlap() +{ + lhs_overlap.reserve(num_mapped_dims); + rhs_overlap.reserve(num_mapped_dims); + for (size_t i = 0; i < num_mapped_dims; ++i) { + lhs_overlap.push_back(i); + rhs_overlap.push_back(i); + } +} + bool SparseJoinPlan::should_forward_lhs_index() const { diff --git a/eval/src/vespa/eval/instruction/generic_join.h b/eval/src/vespa/eval/instruction/generic_join.h index 1fcfcf416cc..026a2938971 100644 --- a/eval/src/vespa/eval/instruction/generic_join.h +++ b/eval/src/vespa/eval/instruction/generic_join.h @@ -58,6 +58,7 @@ struct SparseJoinPlan { bool should_forward_lhs_index() const; bool should_forward_rhs_index() const; SparseJoinPlan(const ValueType &lhs_type, const ValueType &rhs_type); + SparseJoinPlan(size_t num_mapped_dims); // full overlap plan ~SparseJoinPlan(); }; @@ -70,15 +71,14 @@ struct SparseJoinState { const Value::Index &first_index; const Value::Index &second_index; const std::vector<size_t> &second_view_dims; - std::vector<string_id> full_address; - std::vector<string_id*> first_address; - std::vector<const string_id*> address_overlap; - std::vector<string_id*> second_only_address; + std::vector<string_id> full_address; + std::vector<string_id*> first_address; + std::vector<const string_id*> address_overlap; + std::vector<string_id*> second_only_address; size_t lhs_subspace; size_t rhs_subspace; size_t &first_subspace; size_t &second_subspace; - SparseJoinState(const SparseJoinPlan &plan, const Value::Index &lhs, const Value::Index &rhs); ~SparseJoinState(); }; diff --git a/eval/src/vespa/eval/instruction/sparse_dot_product_function.cpp b/eval/src/vespa/eval/instruction/sparse_dot_product_function.cpp new file mode 100644 index 00000000000..93ae2856372 --- /dev/null +++ b/eval/src/vespa/eval/instruction/sparse_dot_product_function.cpp @@ -0,0 +1,111 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "sparse_dot_product_function.h" +#include "generic_join.h" +#include "detect_type.h" +#include <vespa/eval/eval/fast_value.hpp> + +namespace vespalib::eval { + +using namespace tensor_function; +using namespace operation; +using namespace instruction; + +namespace { + +template <typename SCT, typename BCT> +double my_fast_sparse_dot_product(const FastValueIndex &small_idx, const FastValueIndex &big_idx, + const SCT *small_cells, const BCT *big_cells) +{ + double result = 0.0; + small_idx.map.each_map_entry([&](auto small_subspace, auto hash) { + auto small_addr = small_idx.map.get_addr(small_subspace); + auto big_subspace = big_idx.map.lookup(small_addr, hash); + if (big_subspace != FastAddrMap::npos()) { + result += (small_cells[small_subspace] * big_cells[big_subspace]); + } + }); + return result; +} + +template <typename LCT, typename RCT> +void my_sparse_dot_product_op(InterpretedFunction::State &state, uint64_t num_mapped_dims) { + const auto &lhs_idx = state.peek(1).index(); + const auto &rhs_idx = state.peek(0).index(); + const LCT *lhs_cells = state.peek(1).cells().typify<LCT>().cbegin(); + const RCT *rhs_cells = state.peek(0).cells().typify<RCT>().cbegin(); + if (auto indexes = detect_type<FastValueIndex>(lhs_idx, rhs_idx)) { +#if __has_cpp_attribute(likely) + [[likely]]; +#endif + const auto &lhs_fast = indexes.get<0>(); + const auto &rhs_fast = indexes.get<1>(); + double result = (rhs_fast.map.size() < lhs_fast.map.size()) + ? my_fast_sparse_dot_product(rhs_fast, lhs_fast, rhs_cells, lhs_cells) + : my_fast_sparse_dot_product(lhs_fast, rhs_fast, lhs_cells, rhs_cells); + state.pop_pop_push(state.stash.create<ScalarValue<double>>(result)); + } else { +#if __has_cpp_attribute(unlikely) + [[unlikely]]; +#endif + double result = 0.0; + SparseJoinPlan plan(num_mapped_dims); + SparseJoinState sparse(plan, lhs_idx, rhs_idx); + auto outer = sparse.first_index.create_view({}); + auto inner = sparse.second_index.create_view(sparse.second_view_dims); + outer->lookup({}); + while (outer->next_result(sparse.first_address, sparse.first_subspace)) { + inner->lookup(sparse.address_overlap); + if (inner->next_result(sparse.second_only_address, sparse.second_subspace)) { + result += (lhs_cells[sparse.lhs_subspace] * rhs_cells[sparse.rhs_subspace]); + } + } + state.pop_pop_push(state.stash.create<ScalarValue<double>>(result)); + } +} + +struct MyGetFun { + template <typename LCT, typename RCT> + static auto invoke() { return my_sparse_dot_product_op<LCT,RCT>; } +}; + +} // namespace <unnamed> + +SparseDotProductFunction::SparseDotProductFunction(const TensorFunction &lhs_in, + const TensorFunction &rhs_in) + : tensor_function::Op2(ValueType::make_type(CellType::DOUBLE, {}), lhs_in, rhs_in) +{ +} + +InterpretedFunction::Instruction +SparseDotProductFunction::compile_self(const ValueBuilderFactory &, Stash &) const +{ + auto op = typify_invoke<2,TypifyCellType,MyGetFun>(lhs().result_type().cell_type(), rhs().result_type().cell_type()); + return InterpretedFunction::Instruction(op, lhs().result_type().count_mapped_dimensions()); +} + +bool +SparseDotProductFunction::compatible_types(const ValueType &res, const ValueType &lhs, const ValueType &rhs) +{ + return (res.is_scalar() && (res.cell_type() == CellType::DOUBLE) && + lhs.is_sparse() && (rhs.dimensions() == lhs.dimensions())); +} + +const TensorFunction & +SparseDotProductFunction::optimize(const TensorFunction &expr, Stash &stash) +{ + auto reduce = as<Reduce>(expr); + if (reduce && (reduce->aggr() == Aggr::SUM)) { + auto join = as<Join>(reduce->child()); + if (join && (join->function() == Mul::f)) { + const TensorFunction &lhs = join->lhs(); + const TensorFunction &rhs = join->rhs(); + if (compatible_types(expr.result_type(), lhs.result_type(), rhs.result_type())) { + return stash.create<SparseDotProductFunction>(lhs, rhs); + } + } + } + return expr; +} + +} // namespace diff --git a/eval/src/vespa/eval/instruction/sparse_dot_product_function.h b/eval/src/vespa/eval/instruction/sparse_dot_product_function.h new file mode 100644 index 00000000000..ccc7a61f5e8 --- /dev/null +++ b/eval/src/vespa/eval/instruction/sparse_dot_product_function.h @@ -0,0 +1,23 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/tensor_function.h> + +namespace vespalib::eval { + +/** + * Tensor function for a dot product between two sparse tensors. + */ +class SparseDotProductFunction : public tensor_function::Op2 +{ +public: + SparseDotProductFunction(const TensorFunction &lhs_in, + const TensorFunction &rhs_in); + InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + bool result_is_mutable() const override { return true; } + static bool compatible_types(const ValueType &res, const ValueType &lhs, const ValueType &rhs); + static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); +}; + +} // namespace diff --git a/fastos/src/vespa/fastos/ringbuffer.h b/fastos/src/vespa/fastos/ringbuffer.h index 41c0af7385b..89b1bb84c9c 100644 --- a/fastos/src/vespa/fastos/ringbuffer.h +++ b/fastos/src/vespa/fastos/ringbuffer.h @@ -42,18 +42,6 @@ public: _closed = false; } - FastOS_RingBufferData *GetData () { return _data; } - - void RepositionDataAt0 () - { - uint8_t *src = &_data->_buffer[_dataIndex]; - uint8_t *dst = _data->_buffer; - - for(int i=0; i<_dataSize; i++) - *dst++ = *src++; - _dataIndex = 0; - } - FastOS_RingBuffer (int bufferSize) : _closed(false), _data(0), @@ -93,11 +81,6 @@ public: _dataSize += bytes; } - int GetDataSize () - { - return _dataSize; - } - int GetWriteSpace () { int spaceLeft = _bufferSize - _dataSize; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 3a67ca3c374..23dd047cc4c 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -54,75 +54,62 @@ public class Flags { "Takes effect on the next run of RetiredExpirer.", HOSTNAME); - public static final UnboundBooleanFlag WAIT_FOR_RESOURCES_IN_PREPARE = defineFeatureFlag( - "wait-for-resources-in-prepare", false, - List.of("freva"), "2021-01-21", "2021-04-01", - "Whether deployment prepare should wait until all resources (parent hosts, LBs, etc.) are ready before returning", - "Takes effect at redeployment."); - public static final UnboundDoubleFlag DEFAULT_TERM_WISE_LIMIT = defineDoubleFlag( "default-term-wise-limit", 1.0, - List.of("baldersheim"), "2020-12-02", "2021-02-01", + List.of("baldersheim"), "2020-12-02", "2022-01-01", "Default limit for when to apply termwise query evaluation", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundStringFlag FEED_SEQUENCER_TYPE = defineStringFlag( "feed-sequencer-type", "LATENCY", - List.of("baldersheim"), "2020-12-02", "2021-02-01", + List.of("baldersheim"), "2020-12-02", "2022-01-01", "Selects type of sequenced executor used for feeding, valid values are LATENCY, ADAPTIVE, THROUGHPUT", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundStringFlag RESPONSE_SEQUENCER_TYPE = defineStringFlag( "response-sequencer-type", "ADAPTIVE", - List.of("baldersheim"), "2020-12-02", "2021-02-01", + List.of("baldersheim"), "2020-12-02", "2022-01-01", "Selects type of sequenced executor used for mbus responses, valid values are LATENCY, ADAPTIVE, THROUGHPUT", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag RESPONSE_NUM_THREADS = defineIntFlag( "response-num-threads", 2, - List.of("baldersheim"), "2020-12-02", "2021-02-01", + List.of("baldersheim"), "2020-12-02", "2022-01-01", "Number of threads used for mbus responses, default is 2, negative number = numcores/4", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag SKIP_COMMUNICATIONMANAGER_THREAD = defineFeatureFlag( "skip-communicatiomanager-thread", false, - List.of("baldersheim"), "2020-12-02", "2021-02-01", + List.of("baldersheim"), "2020-12-02", "2022-01-01", "Should we skip the communicationmanager thread", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag SKIP_MBUS_REQUEST_THREAD = defineFeatureFlag( "skip-mbus-request-thread", false, - List.of("baldersheim"), "2020-12-02", "2021-02-01", + List.of("baldersheim"), "2020-12-02", "2022-01-01", "Should we skip the mbus request thread", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag SKIP_MBUS_REPLY_THREAD = defineFeatureFlag( "skip-mbus-reply-thread", false, - List.of("baldersheim"), "2020-12-02", "2021-02-01", + List.of("baldersheim"), "2020-12-02", "2022-01-01", "Should we skip the mbus reply thread", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag USE_THREE_PHASE_UPDATES = defineFeatureFlag( "use-three-phase-updates", false, - List.of("vekterli"), "2020-12-02", "2021-02-01", + List.of("vekterli"), "2020-12-02", "2021-03-01", "Whether to enable the use of three-phase updates when bucket replicas are out of sync.", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag USE_FAST_VALUE_TENSOR_IMPLEMENTATION = defineFeatureFlag( - "use-fast-value-tensor-implementation", false, - List.of("geirst"), "2020-12-02", "2021-02-01", - "Whether to use FastValueBuilderFactory as the tensor implementation on all content nodes.", - "Takes effect at restart of content node process", - ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag TCP_ABORT_ON_OVERFLOW = defineFeatureFlag( "tcp-abort-on-overflow", false, List.of("andreer"), "2020-12-02", "2021-02-01", @@ -132,14 +119,14 @@ public class Flags { public static final UnboundStringFlag TLS_FOR_ZOOKEEPER_CLIENT_SERVER_COMMUNICATION = defineStringFlag( "tls-for-zookeeper-client-server-communication", "OFF", - List.of("hmusum"), "2020-12-02", "2021-02-01", + List.of("hmusum"), "2020-12-02", "2021-04-01", "How to setup TLS for ZooKeeper client/server communication. Valid values are OFF, PORT_UNIFICATION, TLS_WITH_PORT_UNIFICATION, TLS_ONLY", "Takes effect on restart of config server", NODE_TYPE, HOSTNAME); public static final UnboundBooleanFlag USE_TLS_FOR_ZOOKEEPER_CLIENT = defineFeatureFlag( "use-tls-for-zookeeper-client", false, - List.of("hmusum"), "2020-12-02", "2021-02-01", + List.of("hmusum"), "2020-12-02", "2021-04-01", "Whether to use TLS for ZooKeeper clients", "Takes effect on restart of process", NODE_TYPE, HOSTNAME); @@ -150,40 +137,22 @@ public class Flags { "Whether endpoint certificates should be validated before use", "Takes effect on the next deployment of the application"); - public static final UnboundStringFlag DELETE_UNUSED_ENDPOINT_CERTIFICATES = defineStringFlag( - "delete-unused-endpoint-certificates", "disable", - List.of("andreer"), "2020-12-02", "2021-02-01", - "Whether the endpoint certificate maintainer should delete unused certificates in cameo/zk", - "Takes effect on next scheduled run of maintainer - set to \"disable\", \"dryrun\" or \"enable\""); - - public static final UnboundBooleanFlag USE_ALTERNATIVE_ENDPOINT_CERTIFICATE_PROVIDER = defineFeatureFlag( - "use-alternative-endpoint-certificate-provider", false, - List.of("andreer"), "2020-12-02", "2021-02-01", - "Whether to use an alternative CA when provisioning new certificates", - "Takes effect only on initial application deployment - not on later certificate refreshes!"); - public static final UnboundStringFlag YUM_DIST_HOST = defineStringFlag( "yum-dist-host", "", - List.of("aressem"), "2020-12-02", "2021-02-01", + List.of("aressem"), "2020-12-02", "2021-03-01", "Override the default dist host for yum.", "Takes effect on next tick or on host-admin restart (may vary where used)."); - public static final UnboundBooleanFlag ENDPOINT_CERT_IN_SHARED_ROUTING = defineFeatureFlag( - "endpoint-cert-in-shared-routing", false, - List.of("andreer"), "2020-12-02", "2021-02-01", - "Whether to provision and use endpoint certs for apps in shared routing zones", - "Takes effect on next deployment of the application", APPLICATION_ID); - public static final UnboundBooleanFlag PROVISION_TENANT_ROLES = defineFeatureFlag( - "provision-application-roles", false, - List.of("tokle"), "2020-12-02", "2021-08-01", - "Whether application roles should be provisioned", + "provision-tenant-roles", false, + List.of("tokle"), "2020-12-02", "2021-04-01", + "Whether tenant roles should be provisioned", "Takes effect on next deployment (controller)", TENANT_ID); public static final UnboundBooleanFlag TENANT_IAM_ROLE = defineFeatureFlag( "application-iam-roles", false, - List.of("tokle"), "2020-12-02", "2021-08-01", + List.of("tokle"), "2020-12-02", "2021-04-01", "Allow separate iam roles when provisioning/assigning hosts", "Takes effect immediately on new hosts, on next redeploy for applications", TENANT_ID); @@ -228,39 +197,25 @@ public class Flags { public static final UnboundBooleanFlag USE_ACCESS_CONTROL_CLIENT_AUTHENTICATION = defineFeatureFlag( "use-access-control-client-authentication", false, - List.of("tokle"), "2020-12-02", "2021-02-01", + List.of("tokle"), "2020-12-02", "2021-03-01", "Whether application container should set up client authentication on default port based on access control element", "Takes effect on next internal redeployment", APPLICATION_ID); public static final UnboundBooleanFlag USE_ASYNC_MESSAGE_HANDLING_ON_SCHEDULE = defineFeatureFlag( "async-message-handling-on-schedule", false, - List.of("baldersheim"), "2020-12-02", "2021-02-01", + List.of("baldersheim"), "2020-12-02", "2022-01-01", "Optionally deliver async messages in own thread", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundIntFlag MERGE_CHUNK_SIZE = defineIntFlag( - "merge-chunk-size", 0x2000000, - List.of("baldersheim"), "2020-12-02", "2021-02-01", - "Size of baldersheim buffer in service layer", - "Takes effect at redeployment", - ZONE_ID, APPLICATION_ID); - public static final UnboundDoubleFlag FEED_CONCURRENCY = defineDoubleFlag( "feed-concurrency", 0.5, - List.of("baldersheim"), "2020-12-02", "2021-02-01", + List.of("baldersheim"), "2020-12-02", "2022-01-01", "How much concurrency should be allowed for feed", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag ENABLE_AUTOMATIC_REINDEXING = defineFeatureFlag( - "enable-automatic-reindexing", true, - List.of("bjorncs", "jonmv"), "2020-12-02", "2021-02-01", - "Whether to automatically trigger reindexing from config change", - "Takes effect on next internal redeployment", - APPLICATION_ID); - public static final UnboundDoubleFlag REINDEXER_WINDOW_SIZE_INCREMENT = defineDoubleFlag( "reindexer-window-size-increment", 0.2, List.of("jonmv"), "2020-12-09", "2021-02-07", @@ -277,7 +232,7 @@ public class Flags { public static final UnboundBooleanFlag USE_POWER_OF_TWO_CHOICES_LOAD_BALANCING = defineFeatureFlag( "use-power-of-two-choices-load-balancing", false, - List.of("tokle"), "2020-12-02", "2021-02-01", + List.of("tokle"), "2020-12-02", "2021-02-15", "Whether to use Power of two load balancing algorithm for application", "Takes effect on next internal redeployment", APPLICATION_ID); @@ -308,12 +263,6 @@ public class Flags { "Whether to enable zstd compression of jdisc access logs", "Takes effect on (re)deployment"); - public static final UnboundBooleanFlag USE_ENDPOINT_CERTIFICATE_MAINTAINER = defineFeatureFlag( - "use-endpoint-certificate-maintainer", false, - List.of("andreer"), "2021-01-12", "2021-02-12", - "Use EndpointCertificateMaintainer instead of EndpointCertificateManager cleanup thread to handle certificate refreshes and deletions", - "Takes effect on next run of maintainer / next manager cleanup thread run"); - public static final UnboundBooleanFlag ENABLE_FEED_BLOCK_IN_DISTRIBUTOR = defineFeatureFlag( "enable-feed-block-in-distributor", false, List.of("geirst"), "2021-01-27", "2021-04-01", diff --git a/fnet/src/tests/connect/connect_test.cpp b/fnet/src/tests/connect/connect_test.cpp index 2b4a2bbe9f0..50722e7d16e 100644 --- a/fnet/src/tests/connect/connect_test.cpp +++ b/fnet/src/tests/connect/connect_test.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/time_bomb.h> #include <vespa/fnet/transport.h> #include <vespa/fnet/transport_thread.h> #include <vespa/fnet/simplepacketstreamer.h> @@ -13,7 +14,7 @@ using namespace vespalib; -int short_time = 20; // ms +constexpr vespalib::duration short_time = 20ms; struct BlockingHostResolver : public AsyncResolver::HostResolver { AsyncResolver::SimpleHostResolver resolver; diff --git a/fnet/src/tests/frt/rpc/detach_return_invoke.cpp b/fnet/src/tests/frt/rpc/detach_return_invoke.cpp index 68bd58d2f82..8d65c65f768 100644 --- a/fnet/src/tests/frt/rpc/detach_return_invoke.cpp +++ b/fnet/src/tests/frt/rpc/detach_return_invoke.cpp @@ -5,6 +5,7 @@ #include <vespa/fnet/frt/rpcrequest.h> #include <vespa/fnet/frt/invoker.h> #include <vespa/vespalib/util/stringfmt.h> +#include <thread> struct Receptor : public FRT_IRequestWait { diff --git a/fnet/src/tests/sync_execute/sync_execute.cpp b/fnet/src/tests/sync_execute/sync_execute.cpp index 55a898496ae..d577b5b7e92 100644 --- a/fnet/src/tests/sync_execute/sync_execute.cpp +++ b/fnet/src/tests/sync_execute/sync_execute.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/util/gate.h> #include <vespa/fnet/transport.h> #include <vespa/fnet/iexecutable.h> #include <vespa/fastos/thread.h> diff --git a/jrt/src/com/yahoo/jrt/CryptoEngine.java b/jrt/src/com/yahoo/jrt/CryptoEngine.java index 6d1955d7f66..318ee333fee 100644 --- a/jrt/src/com/yahoo/jrt/CryptoEngine.java +++ b/jrt/src/com/yahoo/jrt/CryptoEngine.java @@ -1,10 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jrt; - -import com.yahoo.security.tls.AuthorizationMode; import com.yahoo.security.tls.MixedMode; -import com.yahoo.security.tls.ConfigFileBasedTlsContext; import com.yahoo.security.tls.TlsContext; import com.yahoo.security.tls.TransportSecurityUtils; @@ -24,8 +21,7 @@ public interface CryptoEngine extends AutoCloseable { if (!TransportSecurityUtils.isTransportSecurityEnabled()) { return new NullCryptoEngine(); } - AuthorizationMode mode = TransportSecurityUtils.getInsecureAuthorizationMode(); - TlsContext tlsContext = new ConfigFileBasedTlsContext(TransportSecurityUtils.getConfigFile().get(), mode); + TlsContext tlsContext = TransportSecurityUtils.getSystemTlsContext().get(); TlsCryptoEngine tlsCryptoEngine = new TlsCryptoEngine(tlsContext); MixedMode mixedMode = TransportSecurityUtils.getInsecureMixedMode(); switch (mixedMode) { diff --git a/messagebus/src/tests/context/context.cpp b/messagebus/src/tests/context/context.cpp index 713e01cae73..dd734cbbb26 100644 --- a/messagebus/src/tests/context/context.cpp +++ b/messagebus/src/tests/context/context.cpp @@ -10,6 +10,7 @@ #include <vespa/messagebus/testlib/simplemessage.h> #include <vespa/messagebus/testlib/testserver.h> #include <vespa/vespalib/testkit/testapp.h> +#include <thread> using namespace mbus; diff --git a/messagebus/src/tests/messagebus/messagebus.cpp b/messagebus/src/tests/messagebus/messagebus.cpp index 367bfc997d0..259c24ad176 100644 --- a/messagebus/src/tests/messagebus/messagebus.cpp +++ b/messagebus/src/tests/messagebus/messagebus.cpp @@ -13,6 +13,7 @@ #include <vespa/messagebus/testlib/testserver.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/stringfmt.h> +#include <thread> using namespace mbus; diff --git a/messagebus/src/tests/serviceaddress/serviceaddress.cpp b/messagebus/src/tests/serviceaddress/serviceaddress.cpp index c5d07bc437a..0ecba42bf4e 100644 --- a/messagebus/src/tests/serviceaddress/serviceaddress.cpp +++ b/messagebus/src/tests/serviceaddress/serviceaddress.cpp @@ -4,6 +4,7 @@ #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> #include <vespa/messagebus/network/rpcservice.h> +#include <thread> using namespace mbus; diff --git a/messagebus/src/tests/slobrok/slobrok.cpp b/messagebus/src/tests/slobrok/slobrok.cpp index 439ee0b23b5..a9e6d0f4847 100644 --- a/messagebus/src/tests/slobrok/slobrok.cpp +++ b/messagebus/src/tests/slobrok/slobrok.cpp @@ -5,8 +5,8 @@ #include <vespa/slobrok/sbmirror.h> #include <vespa/messagebus/network/rpcnetwork.h> #include <vespa/messagebus/network/rpcnetworkparams.h> - #include <vespa/vespalib/util/host_name.h> +#include <thread> using slobrok::api::IMirrorAPI; diff --git a/messagebus/src/tests/sourcesession/sourcesession.cpp b/messagebus/src/tests/sourcesession/sourcesession.cpp index 3ffe0f8e19d..25d75aa180c 100644 --- a/messagebus/src/tests/sourcesession/sourcesession.cpp +++ b/messagebus/src/tests/sourcesession/sourcesession.cpp @@ -17,6 +17,7 @@ #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/messagebus/testlib/testserver.h> #include <vespa/vespalib/testkit/testapp.h> +#include <thread> using namespace mbus; diff --git a/messagebus/src/tests/throttling/throttling.cpp b/messagebus/src/tests/throttling/throttling.cpp index a23e0b61550..e11aabfd0ba 100644 --- a/messagebus/src/tests/throttling/throttling.cpp +++ b/messagebus/src/tests/throttling/throttling.cpp @@ -13,6 +13,7 @@ #include <vespa/messagebus/testlib/simplemessage.h> #include <vespa/messagebus/testlib/simplereply.h> #include <vespa/messagebus/testlib/testserver.h> +#include <thread> using namespace mbus; diff --git a/messagebus_test/src/tests/trace/trace.cpp b/messagebus_test/src/tests/trace/trace.cpp index 1ab30303d2c..a5e9251f9cc 100644 --- a/messagebus_test/src/tests/trace/trace.cpp +++ b/messagebus_test/src/tests/trace/trace.cpp @@ -19,6 +19,7 @@ #include <vespa/messagebus/testlib/simplereply.h> #include <vespa/messagebus/testlib/simpleprotocol.h> #include <iostream> +#include <thread> #include <vespa/log/log.h> LOG_SETUP("trace_test"); diff --git a/metrics-proxy/pom.xml b/metrics-proxy/pom.xml index 8bf5a30e584..90d9f093da1 100644 --- a/metrics-proxy/pom.xml +++ b/metrics-proxy/pom.xml @@ -101,11 +101,6 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> - <dependency> - <groupId>org.json</groupId> - <artifactId>json</artifactId> - <scope>provided</scope> - </dependency> <!-- compile scope --> diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java index 3cd9f526387..0e0511967a3 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/node/NodeMetricGatherer.java @@ -9,13 +9,11 @@ import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.metricsproxy.metric.model.ServiceId; import ai.vespa.metricsproxy.service.SystemPollerProvider; import ai.vespa.metricsproxy.service.VespaServices; +import com.fasterxml.jackson.databind.JsonNode; import com.google.inject.Inject; import com.yahoo.container.jdisc.state.CoredumpGatherer; import com.yahoo.container.jdisc.state.FileWrapper; import com.yahoo.container.jdisc.state.HostLifeGatherer; -import com.yahoo.yolean.Exceptions; -import org.json.JSONException; -import org.json.JSONObject; import java.util.ArrayList; import java.util.Iterator; @@ -54,10 +52,10 @@ public class NodeMetricGatherer { List<MetricsPacket.Builder> metricPacketBuilders = new ArrayList<>(); metricPacketBuilders.addAll(gatherServiceHealthMetrics(vespaServices)); - JSONObject coredumpPacket = CoredumpGatherer.gatherCoredumpMetrics(fileWrapper); + JsonNode coredumpPacket = CoredumpGatherer.gatherCoredumpMetrics(fileWrapper); addObjectToBuilders(metricPacketBuilders, coredumpPacket); if (SystemPollerProvider.runningOnLinux()) { - JSONObject packet = HostLifeGatherer.getHostLifePacket(fileWrapper); + JsonNode packet = HostLifeGatherer.getHostLifePacket(fileWrapper); addObjectToBuilders(metricPacketBuilders, packet); } @@ -69,24 +67,20 @@ public class NodeMetricGatherer { ).collect(Collectors.toList()); } - protected static void addObjectToBuilders(List<MetricsPacket.Builder> builders, JSONObject object) { - try { - MetricsPacket.Builder builder = new MetricsPacket.Builder(ServiceId.toServiceId(object.getString("application"))); - builder.timestamp(object.getLong("timestamp")); - if (object.has("status_code")) builder.statusCode(object.getInt("status_code")); - if (object.has("status_msg")) builder.statusMessage(object.getString("status_msg")); - if (object.has("metrics")) { - JSONObject metrics = object.getJSONObject("metrics"); - Iterator<?> keys = metrics.keys(); - while(keys.hasNext()) { - String key = (String) keys.next(); - builder.putMetric(MetricId.toMetricId(key), metrics.getLong(key)); - } + protected static void addObjectToBuilders(List<MetricsPacket.Builder> builders, JsonNode object) { + MetricsPacket.Builder builder = new MetricsPacket.Builder(ServiceId.toServiceId(object.get("application").textValue())); + builder.timestamp(object.get("timestamp").longValue()); + if (object.has("status_code")) builder.statusCode(object.get("status_code").intValue()); + if (object.has("status_msg")) builder.statusMessage(object.get("status_msg").textValue()); + if (object.has("metrics")) { + JsonNode metrics = object.get("metrics"); + Iterator<?> keys = metrics.fieldNames(); + while(keys.hasNext()) { + String key = (String) keys.next(); + builder.putMetric(MetricId.toMetricId(key), metrics.get(key).asLong()); } - builders.add(builder); - } catch (JSONException e) { - Exceptions.toMessageString(e); } + builders.add(builder); } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java index 1cab1a859a9..827f513a418 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java @@ -2,9 +2,10 @@ package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.metric.HealthMetric; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + import java.util.logging.Level; -import org.json.JSONException; -import org.json.JSONObject; import java.io.IOException; import java.util.logging.Logger; @@ -15,6 +16,7 @@ import java.util.logging.Logger; * @author Jo Kristian Bergum */ public class RemoteHealthMetricFetcher extends HttpMetricFetcher { + private static final ObjectMapper jsonMapper = new ObjectMapper(); private final static Logger log = Logger.getLogger(RemoteHealthMetricFetcher.class.getPackage().getName()); private final static String HEALTH_PATH = STATE_PATH + "health"; @@ -54,16 +56,16 @@ public class RemoteHealthMetricFetcher extends HttpMetricFetcher { return HealthMetric.getUnknown("Empty response from status page"); } try { - JSONObject o = new JSONObject(data); - JSONObject status = o.getJSONObject("status"); - String code = status.getString("code"); + JsonNode o = jsonMapper.readTree(data); + JsonNode status = o.get("status"); + String code = status.get("code").asText(); String message = ""; if (status.has("message")) { - message = status.getString("message"); + message = status.get("message").textValue(); } return HealthMetric.get(code, message); - } catch (JSONException e) { + } catch (IOException e) { log.log(Level.FINE, "Failed to parse json response from metrics page:" + e + ":" + data); return HealthMetric.getUnknown("Not able to parse json from status page"); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java index 442ebc0d38d..464f215edc4 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteMetricsFetcher.java @@ -4,9 +4,9 @@ package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.metric.Metric; import ai.vespa.metricsproxy.metric.Metrics; import ai.vespa.metricsproxy.metric.model.DimensionId; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import java.io.IOException; import java.util.Collections; @@ -23,6 +23,8 @@ import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; */ public class RemoteMetricsFetcher extends HttpMetricFetcher { + private static final ObjectMapper jsonMapper = new ObjectMapper(); + final static String METRICS_PATH = STATE_PATH + "metrics"; RemoteMetricsFetcher(VespaService service, int port) { @@ -57,21 +59,21 @@ public class RemoteMetricsFetcher extends HttpMetricFetcher { return remoteMetrics; } - private Metrics parse(String data) throws JSONException { - JSONObject o = new JSONObject(data); + private Metrics parse(String data) throws IOException { + JsonNode o = jsonMapper.readTree(data); if (!(o.has("metrics"))) { return new Metrics(); //empty } - JSONObject metrics = o.getJSONObject("metrics"); - JSONArray values; + JsonNode metrics = o.get("metrics"); + ArrayNode values; long timestamp; try { - JSONObject snapshot = metrics.getJSONObject("snapshot"); - timestamp = (long) snapshot.getDouble("to"); - values = metrics.getJSONArray("values"); - } catch (JSONException e) { + JsonNode snapshot = metrics.get("snapshot"); + timestamp = snapshot.get("to").asLong(); + values = (ArrayNode) metrics.get("values"); + } catch (Exception e) { // snapshot might not have been produced. Do not throw exception into log return new Metrics(); } @@ -81,29 +83,29 @@ public class RemoteMetricsFetcher extends HttpMetricFetcher { Map<DimensionId, String> noDims = Collections.emptyMap(); Map<String, Map<DimensionId, String>> uniqueDimensions = new HashMap<>(); - for (int i = 0; i < values.length(); i++) { - JSONObject metric = values.getJSONObject(i); - String name = metric.getString("name"); + for (int i = 0; i < values.size(); i++) { + JsonNode metric = values.get(i); + String name = metric.get("name").textValue(); String description = ""; if (metric.has("description")) { - description = metric.getString("description"); + description = metric.get("description").textValue(); } Map<DimensionId, String> dim = noDims; if (metric.has("dimensions")) { - JSONObject dimensions = metric.getJSONObject("dimensions"); + JsonNode dimensions = metric.get("dimensions"); StringBuilder sb = new StringBuilder(); - for (Iterator<?> it = dimensions.keys(); it.hasNext(); ) { + for (Iterator<?> it = dimensions.fieldNames(); it.hasNext(); ) { String k = (String) it.next(); - String v = dimensions.getString(k); + String v = dimensions.get(k).asText(); sb.append(toDimensionId(k)).append(v); } if ( ! uniqueDimensions.containsKey(sb.toString())) { dim = new HashMap<>(); - for (Iterator<?> it = dimensions.keys(); it.hasNext(); ) { + for (Iterator<?> it = dimensions.fieldNames(); it.hasNext(); ) { String k = (String) it.next(); - String v = dimensions.getString(k); + String v = dimensions.get(k).textValue(); dim.put(toDimensionId(k), v); } uniqueDimensions.put(sb.toString(), Collections.unmodifiableMap(dim)); @@ -111,10 +113,17 @@ public class RemoteMetricsFetcher extends HttpMetricFetcher { dim = uniqueDimensions.get(sb.toString()); } - JSONObject aggregates = metric.getJSONObject("values"); - for (Iterator<?> it = aggregates.keys(); it.hasNext(); ) { + JsonNode aggregates = metric.get("values"); + for (Iterator<?> it = aggregates.fieldNames(); it.hasNext(); ) { String aggregator = (String) it.next(); - Number value = (Number) aggregates.get(aggregator); + JsonNode aggregatorValue = aggregates.get(aggregator); + if (aggregatorValue == null) { + throw new IllegalArgumentException("Value for aggregator '" + aggregator + "' is missing"); + } + Number value = aggregatorValue.numberValue(); + if (value == null) { + throw new IllegalArgumentException("Value for aggregator '" + aggregator + "' is not a number"); + } StringBuilder metricName = (new StringBuilder()).append(name).append(".").append(aggregator); m.add(new Metric(metricName.toString(), value, timestamp, dim, description)); } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java index d7576718e8a..cf1eac3c691 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsHandlerTest.java @@ -8,13 +8,11 @@ import ai.vespa.metricsproxy.metric.model.json.GenericApplicationModel; import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel; import ai.vespa.metricsproxy.metric.model.json.GenericMetrics; import ai.vespa.metricsproxy.metric.model.json.GenericService; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.github.tomakehurst.wiremock.junit.WireMockRule; import com.yahoo.container.jdisc.RequestHandlerTestDriver; -import java.util.regex.Pattern; - -import org.json.JSONArray; -import org.json.JSONObject; import org.junit.Before; import org.junit.Ignore; import org.junit.Rule; @@ -24,6 +22,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Map; import java.util.concurrent.Executors; +import java.util.regex.Pattern; import static ai.vespa.metricsproxy.TestUtil.getFileContents; import static ai.vespa.metricsproxy.http.ValuesFetcher.defaultMetricsConsumerId; @@ -49,6 +48,8 @@ import static org.junit.Assert.fail; @SuppressWarnings("UnstableApiUsage") public class ApplicationMetricsHandlerTest { + private static final ObjectMapper jsonMapper = new ObjectMapper(); + private static final String HOST = "localhost"; private static final String URI_BASE = "http://" + HOST; private static final String METRICS_V1_URI = URI_BASE + METRICS_V1_PATH; @@ -102,16 +103,16 @@ public class ApplicationMetricsHandlerTest { @Test public void v1_response_contains_values_uri() throws Exception { String response = testDriver.sendRequest(METRICS_V1_URI).readAll(); - JSONObject root = new JSONObject(response); + JsonNode root = jsonMapper.readTree(response); assertTrue(root.has("resources")); - JSONArray resources = root.getJSONArray("resources"); - assertEquals(2, resources.length()); + ArrayNode resources = (ArrayNode) root.get("resources"); + assertEquals(2, resources.size()); - JSONObject valuesUrl = resources.getJSONObject(0); - assertEquals(METRICS_VALUES_URI, valuesUrl.getString("url")); - JSONObject prometheusUrl = resources.getJSONObject(1); - assertEquals(PROMETHEUS_VALUES_URI, prometheusUrl.getString("url")); + JsonNode valuesUrl = resources.get(0); + assertEquals(METRICS_VALUES_URI, valuesUrl.get("url").textValue()); + JsonNode prometheusUrl = resources.get(1); + assertEquals(PROMETHEUS_VALUES_URI, prometheusUrl.get("url").textValue()); } @Ignore @@ -199,7 +200,7 @@ public class ApplicationMetricsHandlerTest { @Test public void invalid_path_yields_error_response() throws Exception { String response = testDriver.sendRequest(METRICS_V1_URI + "/invalid").readAll(); - JSONObject root = new JSONObject(response); + JsonNode root = jsonMapper.readTree(response); assertTrue(root.has("error")); } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsHandlerTestBase.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsHandlerTestBase.java index 1c5ce695155..379ef04d38d 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsHandlerTestBase.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/metrics/MetricsHandlerTestBase.java @@ -6,9 +6,9 @@ import ai.vespa.metricsproxy.metric.model.json.GenericJsonModel; import ai.vespa.metricsproxy.metric.model.json.GenericMetrics; import ai.vespa.metricsproxy.metric.model.json.GenericService; import ai.vespa.metricsproxy.service.DownService; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import org.json.JSONArray; -import org.json.JSONObject; +import com.fasterxml.jackson.databind.node.ArrayNode; import org.junit.Ignore; import org.junit.Test; @@ -32,6 +32,8 @@ import static org.junit.Assert.fail; */ public abstract class MetricsHandlerTestBase<MODEL> extends HttpHandlerTestBase { + private static final ObjectMapper jsonMapper = new ObjectMapper(); + static String rootUri; static String valuesUri; @@ -56,21 +58,21 @@ public abstract class MetricsHandlerTestBase<MODEL> extends HttpHandlerTestBase @Test public void invalid_path_yields_error_response() throws Exception { String response = testDriver.sendRequest(rootUri + "/invalid").readAll(); - JSONObject root = new JSONObject(response); + JsonNode root = jsonMapper.readTree(response); assertTrue(root.has("error")); } @Test public void root_response_contains_values_uri() throws Exception { String response = testDriver.sendRequest(rootUri).readAll(); - JSONObject root = new JSONObject(response); + JsonNode root = jsonMapper.readTree(response); assertTrue(root.has("resources")); - JSONArray resources = root.getJSONArray("resources"); - assertEquals(1, resources.length()); + ArrayNode resources = (ArrayNode) root.get("resources"); + assertEquals(1, resources.size()); - JSONObject valuesUrl = resources.getJSONObject(0); - assertEquals(valuesUri, valuesUrl.getString("url")); + JsonNode valuesUrl = resources.get(0); + assertEquals(valuesUri, valuesUrl.get("url").textValue()); } @Ignore diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java index a224c4090b3..89186e63b93 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java @@ -3,9 +3,10 @@ package ai.vespa.metricsproxy.http.prometheus; import ai.vespa.metricsproxy.http.HttpHandlerTestBase; import ai.vespa.metricsproxy.service.DummyService; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.yahoo.container.jdisc.RequestHandlerTestDriver; -import org.json.JSONArray; -import org.json.JSONObject; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; @@ -22,6 +23,8 @@ import static org.junit.Assert.assertTrue; @SuppressWarnings("UnstableApiUsage") public class PrometheusHandlerTest extends HttpHandlerTestBase { + private static final ObjectMapper jsonMapper = new ObjectMapper(); + private static final String V1_URI = URI_BASE + PrometheusHandler.V1_PATH; private static final String VALUES_URI = URI_BASE + PrometheusHandler.VALUES_PATH; @@ -40,14 +43,14 @@ public class PrometheusHandlerTest extends HttpHandlerTestBase { @Test public void v1_response_contains_values_uri() throws Exception { String response = testDriver.sendRequest(V1_URI).readAll(); - JSONObject root = new JSONObject(response); + JsonNode root = jsonMapper.readTree(response); assertTrue(root.has("resources")); - JSONArray resources = root.getJSONArray("resources"); - assertEquals(1, resources.length()); + ArrayNode resources = (ArrayNode) root.get("resources"); + assertEquals(1, resources.size()); - JSONObject valuesUrl = resources.getJSONObject(0); - assertEquals(VALUES_URI, valuesUrl.getString("url")); + JsonNode valuesUrl = resources.get(0); + assertEquals(VALUES_URI, valuesUrl.get("url").textValue()); } @Ignore diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/NodeMetricGathererTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/NodeMetricGathererTest.java index e2ad0ccd504..c2fc23a878d 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/NodeMetricGathererTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/node/NodeMetricGathererTest.java @@ -3,8 +3,9 @@ package ai.vespa.metricsproxy.node; import ai.vespa.metricsproxy.metric.model.MetricId; import ai.vespa.metricsproxy.metric.model.MetricsPacket; -import org.json.JSONException; -import org.json.JSONObject; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.Test; import java.util.ArrayList; @@ -17,10 +18,12 @@ import static org.junit.Assert.assertEquals; */ public class NodeMetricGathererTest { + private static final ObjectMapper jsonMapper = new ObjectMapper(); + @Test - public void testJSONObjectIsCorrectlyConvertedToMetricsPacket() throws JSONException { + public void testJSONObjectIsCorrectlyConvertedToMetricsPacket() { List<MetricsPacket.Builder> builders = new ArrayList<>(); - JSONObject hostLifePacket = generateHostLifePacket(); + JsonNode hostLifePacket = generateHostLifePacket(); NodeMetricGatherer.addObjectToBuilders(builders, hostLifePacket); MetricsPacket packet = builders.remove(0).build(); @@ -32,17 +35,17 @@ public class NodeMetricGathererTest { assertEquals(1l, packet.metrics().get(MetricId.toMetricId("alive"))); } - private JSONObject generateHostLifePacket() throws JSONException { + private JsonNode generateHostLifePacket() { - JSONObject jsonObject = new JSONObject(); + ObjectNode jsonObject = jsonMapper.createObjectNode(); jsonObject.put("status_code", 0); jsonObject.put("status_msg", "OK"); jsonObject.put("timestamp", 123); jsonObject.put("application", "host_life"); - JSONObject metrics = new JSONObject(); + ObjectNode metrics = jsonMapper.createObjectNode(); metrics.put("uptime", 12); metrics.put("alive", 1); - jsonObject.put("metrics", metrics); + jsonObject.set("metrics", metrics); return jsonObject; } } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java index 8d5bba77844..70970bfe8da 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/rpc/RpcMetricsTest.java @@ -5,17 +5,18 @@ import ai.vespa.metricsproxy.metric.Metric; import ai.vespa.metricsproxy.metric.Metrics; import ai.vespa.metricsproxy.metric.model.ConsumerId; import ai.vespa.metricsproxy.service.VespaService; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.yahoo.jrt.Request; import com.yahoo.jrt.Spec; import com.yahoo.jrt.StringValue; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; import org.junit.Test; +import java.io.IOException; import java.util.List; import static ai.vespa.metricsproxy.TestUtil.getFileContents; @@ -40,6 +41,8 @@ import static org.junit.Assert.assertTrue; */ public class RpcMetricsTest { + private static final ObjectMapper jsonMapper = new ObjectMapper(); + private static final String METRICS_RESPONSE = getFileContents("metrics-storage-simple.json").trim(); private static final String EXTRA_APP = "extra"; @@ -67,9 +70,9 @@ public class RpcMetricsTest { String allServicesResponse = getMetricsForYamas(ALL_SERVICES, rpcClient).trim(); // Verify that application is used as serviceId, and that metric exists. - JSONObject extraMetrics = findExtraMetricsObject(allServicesResponse); - assertThat(extraMetrics.getJSONObject("metrics").getInt("foo.count"), is(3)); - assertThat(extraMetrics.getJSONObject("dimensions").getString("role"), is("extra-role")); + JsonNode extraMetrics = findExtraMetricsObject(allServicesResponse); + assertThat(extraMetrics.get("metrics").get("foo.count").intValue(), is(3)); + assertThat(extraMetrics.get("dimensions").get("role").textValue(), is("extra-role")); } } } @@ -85,7 +88,7 @@ public class RpcMetricsTest { // Verify that no extra metrics exists String allServicesResponse = getMetricsForYamas(ALL_SERVICES, rpcClient).trim(); - JSONObject extraMetrics = findExtraMetricsObject(allServicesResponse); + JsonNode extraMetrics = findExtraMetricsObject(allServicesResponse); assertEquals(extraMetrics.toString(), "{}"); } } @@ -130,28 +133,28 @@ public class RpcMetricsTest { } } - private static void verifyMetricsFromRpcRequest(VespaService service, RpcClient client) throws JSONException { + private static void verifyMetricsFromRpcRequest(VespaService service, RpcClient client) throws IOException { String jsonResponse = getMetricsForYamas(service.getMonitoringName(), client).trim(); - JSONArray metrics = new JSONObject(jsonResponse).getJSONArray("metrics"); - assertThat("Expected 3 metric messages", metrics.length(), is(3)); - for (int i = 0; i < metrics.length() - 1; i++) { // The last "metric message" contains only status code/message - JSONObject jsonObject = metrics.getJSONObject(i); + ArrayNode metrics = (ArrayNode) jsonMapper.readTree(jsonResponse).get("metrics"); + assertThat("Expected 3 metric messages", metrics.size(), is(3)); + for (int i = 0; i < metrics.size() - 1; i++) { // The last "metric message" contains only status code/message + JsonNode jsonObject = metrics.get(i); assertFalse(jsonObject.has("status_code")); assertFalse(jsonObject.has("status_msg")); - assertThat(jsonObject.getJSONObject("dimensions").getString("foo"), is("bar")); - assertThat(jsonObject.getJSONObject("dimensions").getString("bar"), is("foo")); - assertThat(jsonObject.getJSONObject("dimensions").getString("serviceDim"), is("serviceDimValue")); - assertThat(jsonObject.getJSONObject("routing").getJSONObject("yamas").getJSONArray("namespaces").length(), is(1)); - if (jsonObject.getJSONObject("metrics").has("foo_count")) { - assertThat(jsonObject.getJSONObject("metrics").getInt("foo_count"), is(1)); - assertThat(jsonObject.getJSONObject("routing").getJSONObject("yamas").getJSONArray("namespaces").get(0), is(vespaMetricsConsumerId.id)); + assertThat(jsonObject.get("dimensions").get("foo").textValue(), is("bar")); + assertThat(jsonObject.get("dimensions").get("bar").textValue(), is("foo")); + assertThat(jsonObject.get("dimensions").get("serviceDim").textValue(), is("serviceDimValue")); + assertThat(jsonObject.get("routing").get("yamas").get("namespaces").size(), is(1)); + if (jsonObject.get("metrics").has("foo_count")) { + assertThat(jsonObject.get("metrics").get("foo_count").intValue(), is(1)); + assertThat(jsonObject.get("routing").get("yamas").get("namespaces").get(0).textValue(), is(vespaMetricsConsumerId.id)); } else { - assertThat(jsonObject.getJSONObject("metrics").getInt("foo.count"), is(1)); - assertThat(jsonObject.getJSONObject("routing").getJSONObject("yamas").getJSONArray("namespaces").get(0), is(CUSTOM_CONSUMER_ID.id)); + assertThat(jsonObject.get("metrics").get("foo.count").intValue(), is(1)); + assertThat(jsonObject.get("routing").get("yamas").get("namespaces").get(0).textValue(), is(CUSTOM_CONSUMER_ID.id)); } } - verifyStatusMessage(metrics.getJSONObject(metrics.length() - 1)); + verifyStatusMessage(metrics.get(metrics.size() - 1)); } private void verfiyMetricsFromServiceObject(VespaService service) { @@ -166,15 +169,15 @@ public class RpcMetricsTest { assertThat("Metric foo did not contain correct dimension for key = bar", foo.getDimensions().get(toDimensionId("bar")), is("foo")); } - private void verifyMetricsFromRpcRequestForAllServices(RpcClient client) throws JSONException { + private void verifyMetricsFromRpcRequestForAllServices(RpcClient client) throws IOException { // Verify that metrics for all services can be retrieved in one request. String allServicesResponse = getMetricsForYamas(ALL_SERVICES, client).trim(); - JSONArray allServicesMetrics = new JSONObject(allServicesResponse).getJSONArray("metrics"); - assertThat(allServicesMetrics.length(), is(5)); + ArrayNode allServicesMetrics = (ArrayNode) jsonMapper.readTree(allServicesResponse).get("metrics"); + assertThat(allServicesMetrics.size(), is(5)); } @Test - public void testGetAllMetricNames() throws Exception { + public void testGetAllMetricNames() { try (IntegrationTester tester = new IntegrationTester()) { tester.httpServer().setResponse(METRICS_RESPONSE); @@ -205,14 +208,14 @@ public class RpcMetricsTest { invoke(req, rpcClient, false); } - private JSONObject findExtraMetricsObject(String jsonResponse) throws JSONException { - JSONArray metrics = new JSONObject(jsonResponse).getJSONArray("metrics"); - for (int i = 0; i < metrics.length(); i++) { - JSONObject jsonObject = metrics.getJSONObject(i); + private JsonNode findExtraMetricsObject(String jsonResponse) throws IOException { + ArrayNode metrics = (ArrayNode) jsonMapper.readTree(jsonResponse).get("metrics"); + for (int i = 0; i < metrics.size(); i++) { + JsonNode jsonObject = metrics.get(i); assertTrue(jsonObject.has("application")); - if (jsonObject.getString("application").equals(EXTRA_APP)) return jsonObject; + if (jsonObject.get("application").textValue().equals(EXTRA_APP)) return jsonObject; } - return new JSONObject(); + return jsonMapper.createObjectNode(); } private static String getMetricsForYamas(String service, RpcClient client) { @@ -250,12 +253,12 @@ public class RpcMetricsTest { return returnValue; } - private static void verifyStatusMessage(JSONObject jsonObject) throws JSONException { - assertThat(jsonObject.getInt("status_code"), is(0)); - assertThat(jsonObject.getString("status_msg"), notNullValue()); - assertThat(jsonObject.getString("application"), notNullValue()); - assertThat(jsonObject.getString("routing"), notNullValue()); - assertThat(jsonObject.length(), is(4)); + private static void verifyStatusMessage(JsonNode jsonObject) { + assertThat(jsonObject.get("status_code").intValue(), is(0)); + assertThat(jsonObject.get("status_msg").textValue(), notNullValue()); + assertThat(jsonObject.get("application").textValue(), notNullValue()); + assertThat(jsonObject.get("routing"), notNullValue()); + assertThat(jsonObject.size(), is(4)); } } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java index 0d53f988ac7..7ff179e5528 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java @@ -2,7 +2,6 @@ package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.metric.Metric; -import org.json.JSONException; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; @@ -37,7 +36,7 @@ public class ContainerServiceTest { } @Test - public void testMultipleQueryDimensions() throws JSONException { + public void testMultipleQueryDimensions() { int count = 0; VespaService service = VespaService.create("service1", "id", httpServer.port()); for (Metric m : service.getMetrics().getMetrics()) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java index 6094b497fff..46ffe14d4e2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java @@ -161,7 +161,7 @@ public class CapacityChecker { int timesHostCanBeRemoved = 0; Optional<Node> unallocatedNode; - while (timesHostCanBeRemoved < 1000) { // Arbitrary upper bound + while (timesHostCanBeRemoved < 100) { // Arbitrary upper bound unallocatedNode = tryAllocateNodes(nodeChildren.get(host), hosts, resourceMap, containedAllocations); if (unallocatedNode.isEmpty()) { timesHostCanBeRemoved++; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 3d61dd39b31..53ba73e4d82 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -61,12 +61,17 @@ public class GroupPreparer { public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, List<Node> surplusActiveNodes, MutableInteger highestIndex, int wantedGroups) { + String allocateOsRequirement = allocateOsRequirementFlag + .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()) + .value(); + // Try preparing in memory without global unallocated lock. Most of the time there should be no changes and we // can return nodes previously allocated. { MutableInteger probePrepareHighestIndex = new MutableInteger(highestIndex.get()); NodeAllocation probeAllocation = prepareAllocation(application, cluster, requestedNodes, surplusActiveNodes, - probePrepareHighestIndex, wantedGroups, PROBE_LOCK); + probePrepareHighestIndex, wantedGroups, PROBE_LOCK, + allocateOsRequirement); if (probeAllocation.fulfilledAndNoChanges()) { List<Node> acceptedNodes = probeAllocation.finalNodes(); surplusActiveNodes.removeAll(acceptedNodes); @@ -80,10 +85,16 @@ public class GroupPreparer { Mutex allocationLock = nodeRepository.lockUnallocated()) { NodeAllocation allocation = prepareAllocation(application, cluster, requestedNodes, surplusActiveNodes, - highestIndex, wantedGroups, allocationLock); + highestIndex, wantedGroups, allocationLock, allocateOsRequirement); if (nodeRepository.zone().getCloud().dynamicProvisioning()) { - Version osVersion = nodeRepository.osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion); + final Version osVersion; + if (allocateOsRequirement.equals("rhel8")) { + osVersion = new Version(8); + } else { + osVersion = nodeRepository.osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion); + } + List<ProvisionedHost> provisionedHosts = allocation.getFulfilledDockerDeficit() .map(deficit -> hostProvisioner.get().provisionHosts(nodeRepository.database().getProvisionIndexes(deficit.getCount()), deficit.getFlavor(), @@ -122,13 +133,10 @@ public class GroupPreparer { private NodeAllocation prepareAllocation(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, List<Node> surplusActiveNodes, MutableInteger highestIndex, int wantedGroups, - Mutex allocationLock) { + Mutex allocationLock, String allocateOsRequirement) { LockedNodeList allNodes = nodeRepository.list(allocationLock); NodeAllocation allocation = new NodeAllocation(allNodes, application, cluster, requestedNodes, highestIndex, nodeRepository); - String allocateOsRequirement = allocateOsRequirementFlag - .with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()) - .value(); NodePrioritizer prioritizer = new NodePrioritizer( allNodes, application, cluster, requestedNodes, wantedGroups, nodeRepository.zone().getCloud().dynamicProvisioning(), nodeRepository.nameResolver(), diff --git a/persistence/src/vespa/persistence/spi/attribute_resource_usage.h b/persistence/src/vespa/persistence/spi/attribute_resource_usage.h index 8d28370e504..02e0c248e3d 100644 --- a/persistence/src/vespa/persistence/spi/attribute_resource_usage.h +++ b/persistence/src/vespa/persistence/spi/attribute_resource_usage.h @@ -29,6 +29,7 @@ public: double get_usage() const noexcept { return _usage; } const vespalib::string& get_name() const noexcept { return _name; } + bool valid() const noexcept { return !_name.empty(); } bool operator==(const AttributeResourceUsage& rhs) const noexcept { return ((_usage == rhs._usage) && (_name == rhs._name)); diff --git a/persistence/src/vespa/persistence/spi/bucket.h b/persistence/src/vespa/persistence/spi/bucket.h index 30393109cc9..507cc80ad76 100644 --- a/persistence/src/vespa/persistence/spi/bucket.h +++ b/persistence/src/vespa/persistence/spi/bucket.h @@ -21,12 +21,12 @@ public: explicit Bucket(const document::Bucket& b) noexcept : _bucket(b) {} - const document::Bucket &getBucket() const { return _bucket; } - document::BucketId getBucketId() const { return _bucket.getBucketId(); } - document::BucketSpace getBucketSpace() const { return _bucket.getBucketSpace(); } + const document::Bucket &getBucket() const noexcept { return _bucket; } + document::BucketId getBucketId() const noexcept { return _bucket.getBucketId(); } + document::BucketSpace getBucketSpace() const noexcept { return _bucket.getBucketSpace(); } /** Convert easily to a document bucket id to make class easy to use. */ - operator document::BucketId() const { return _bucket.getBucketId(); } + operator document::BucketId() const noexcept { return _bucket.getBucketId(); } bool operator==(const Bucket& o) const noexcept { return (_bucket == o._bucket); diff --git a/protocols/getnodestate/host_info.json b/protocols/getnodestate/host_info.json index 7ae5b0043ff..7eddf506e63 100644 --- a/protocols/getnodestate/host_info.json +++ b/protocols/getnodestate/host_info.json @@ -110,7 +110,8 @@ "usage": 0.85 }, "disk": { - "usage": 0.6 + "usage": 0.6, + "name": "a cool disk" } } } diff --git a/searchcommon/src/vespa/searchcommon/common/CMakeLists.txt b/searchcommon/src/vespa/searchcommon/common/CMakeLists.txt index 23e6e8dd394..bf5686280e6 100644 --- a/searchcommon/src/vespa/searchcommon/common/CMakeLists.txt +++ b/searchcommon/src/vespa/searchcommon/common/CMakeLists.txt @@ -1,7 +1,9 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchcommon_searchcommon_common OBJECT SOURCES + compaction_strategy.cpp datatype.cpp + growstrategy.cpp schema.cpp schemaconfigurer.cpp DEPENDS diff --git a/searchcommon/src/vespa/searchcommon/common/compaction_strategy.cpp b/searchcommon/src/vespa/searchcommon/common/compaction_strategy.cpp new file mode 100644 index 00000000000..c3377ed5857 --- /dev/null +++ b/searchcommon/src/vespa/searchcommon/common/compaction_strategy.cpp @@ -0,0 +1,15 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "compaction_strategy.h" +#include <iostream> +namespace search { + +std::ostream& operator<<(std::ostream& os, const CompactionStrategy& compaction_strategy) +{ + os << "{maxDeadBytesRatio=" << compaction_strategy.getMaxDeadBytesRatio() << + ", maxDeadAddressSpaceRatio=" << compaction_strategy.getMaxDeadAddressSpaceRatio() << + "}"; + return os; +} + +} diff --git a/searchcommon/src/vespa/searchcommon/common/compaction_strategy.h b/searchcommon/src/vespa/searchcommon/common/compaction_strategy.h index 0b3caf44481..ae354a4c4d2 100644 --- a/searchcommon/src/vespa/searchcommon/common/compaction_strategy.h +++ b/searchcommon/src/vespa/searchcommon/common/compaction_strategy.h @@ -3,6 +3,7 @@ #pragma once #include <stdint.h> +#include <iosfwd> namespace search { @@ -34,4 +35,6 @@ public: bool operator!=(const CompactionStrategy & rhs) const { return !(operator==(rhs)); } }; +std::ostream& operator<<(std::ostream& os, const CompactionStrategy& compaction_strategy); + } // namespace search diff --git a/searchcommon/src/vespa/searchcommon/common/growstrategy.cpp b/searchcommon/src/vespa/searchcommon/common/growstrategy.cpp new file mode 100644 index 00000000000..534be3060a7 --- /dev/null +++ b/searchcommon/src/vespa/searchcommon/common/growstrategy.cpp @@ -0,0 +1,18 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "growstrategy.h" +#include <iostream> + +namespace search { + +std::ostream& operator<<(std::ostream& os, const GrowStrategy& grow_strategy) +{ + os << "{docsInitialCapacity=" << grow_strategy.getDocsInitialCapacity() << + ", docsGrowFactor=" << grow_strategy.getDocsGrowFactor() << + ", docsGrowDelta=" << grow_strategy.getDocsGrowDelta() << + ", multiValueAllocGrowFactor=" << grow_strategy.getMultiValueAllocGrowFactor() << + "}"; + return os; +} + +} diff --git a/searchcommon/src/vespa/searchcommon/common/growstrategy.h b/searchcommon/src/vespa/searchcommon/common/growstrategy.h index 4194541b8e5..f7c5c030d95 100644 --- a/searchcommon/src/vespa/searchcommon/common/growstrategy.h +++ b/searchcommon/src/vespa/searchcommon/common/growstrategy.h @@ -4,6 +4,7 @@ #include <vespa/vespalib/util/growstrategy.h> #include <cstdint> +#include <iosfwd> namespace search { @@ -18,10 +19,10 @@ public: GrowStrategy() : GrowStrategy(1024, 0.5, 0, 0.2) {} - GrowStrategy(uint32_t docsInitialCapacity, float docsGrowPercent, + GrowStrategy(uint32_t docsInitialCapacity, float docsGrowFactor, uint32_t docsGrowDelta, float multiValueAllocGrowFactor) : _docsInitialCapacity(docsInitialCapacity), - _docsGrowFactor(docsGrowPercent), + _docsGrowFactor(docsGrowFactor), _docsGrowDelta(docsGrowDelta), _multiValueAllocGrowFactor(multiValueAllocGrowFactor) { @@ -54,5 +55,7 @@ public: } }; +std::ostream& operator<<(std::ostream& os, const GrowStrategy& grow_strategy); + } diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index 04cface5d40..d77bc69ba62 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -69,6 +69,7 @@ vespa_define_module( src/tests/proton/attribute/imported_attributes_repo src/tests/proton/bucketdb/bucketdb src/tests/proton/common + src/tests/proton/common/alloc_config src/tests/proton/common/attribute_updater src/tests/proton/common/document_type_inspector src/tests/proton/common/hw_info_sampler diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index 4715ff80d03..b47a1954c6f 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -13,6 +13,7 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/test/make_bucket_space.h> #include <vespa/searchcommon/common/schemaconfigurer.h> +#include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/matching/querylimiter.h> #include <vespa/searchcore/proton/metrics/metricswireservice.h> @@ -145,6 +146,7 @@ public: std::make_shared<DocumentDBMaintenanceConfig>(), search::LogDocumentStore::Config(), std::make_shared<const ThreadingServiceConfig>(ThreadingServiceConfig::make(1)), + std::make_shared<const AllocConfig>(), "client", docTypeName.getName()); } diff --git a/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp b/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp index 90942e9aef4..6cca2e4bd48 100644 --- a/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp +++ b/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp @@ -38,6 +38,7 @@ #include <vespa/messagebus/testlib/slobrok.h> #include <vespa/metrics/config-metricsmanager.h> #include <vespa/searchcommon/common/schemaconfigurer.h> +#include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/matching/querylimiter.h> #include <vespa/searchcore/proton/metrics/metricswireservice.h> @@ -198,6 +199,7 @@ std::shared_ptr<DocumentDBConfig> make_document_db_config(std::shared_ptr<Docume std::make_shared<DocumentDBMaintenanceConfig>(), search::LogDocumentStore::Config(), std::make_shared<const ThreadingServiceConfig>(ThreadingServiceConfig::make(1)), + std::make_shared<const AllocConfig>(), "client", doc_type_name.getName()); } diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp index d25a234c6f8..3987a8685ea 100644 --- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp @@ -232,8 +232,7 @@ struct ParallelAttributeManager InitializerTask::SP documentMetaStoreInitTask; BucketDBOwner::SP bucketDbOwner; DocumentMetaStore::SP documentMetaStore; - search::GrowStrategy attributeGrow; - size_t attributeGrowNumDocs; + AllocStrategy alloc_strategy; bool fastAccessAttributesOnly; std::shared_ptr<AttributeManager::SP> mgr; vespalib::ThreadStackExecutor masterExecutor; @@ -250,15 +249,14 @@ ParallelAttributeManager::ParallelAttributeManager(search::SerialNum configSeria : documentMetaStoreInitTask(std::make_shared<DummyInitializerTask>()), bucketDbOwner(std::make_shared<BucketDBOwner>()), documentMetaStore(std::make_shared<DocumentMetaStore>(bucketDbOwner)), - attributeGrow(), - attributeGrowNumDocs(1), + alloc_strategy(), fastAccessAttributesOnly(false), mgr(std::make_shared<AttributeManager::SP>()), masterExecutor(1, 128 * 1024), master(masterExecutor), initializer(std::make_shared<AttributeManagerInitializer>(configSerialNum, documentMetaStoreInitTask, documentMetaStore, baseAttrMgr, attrCfg, - attributeGrow, attributeGrowNumDocs, + alloc_strategy, fastAccessAttributesOnly, master, mgr)) { documentMetaStore->setCommittedDocIdLimit(docIdLimit); diff --git a/searchcore/src/tests/proton/attribute/attribute_test.cpp b/searchcore/src/tests/proton/attribute/attribute_test.cpp index b67eaec1a8d..ebd3e27aeca 100644 --- a/searchcore/src/tests/proton/attribute/attribute_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_test.cpp @@ -535,7 +535,7 @@ public: AttributeCollectionSpecFactory _factory; AttributeCollectionSpecTest(bool fastAccessOnly) : _builder(), - _factory(search::GrowStrategy(), 100, fastAccessOnly) + _factory(AllocStrategy(search::GrowStrategy(), search::CompactionStrategy(), 100), fastAccessOnly) { addAttribute("a1", false); addAttribute("a2", true); diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp b/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp index 0fced6e0bff..b35027eac2a 100644 --- a/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_usage_filter/attribute_usage_filter_test.cpp @@ -3,9 +3,13 @@ LOG_SETUP("attribute_usage_filter_test"); #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> +#include <vespa/searchcore/proton/attribute/i_attribute_usage_listener.h> using proton::AttributeUsageFilter; using proton::AttributeUsageStats; +using proton::IAttributeUsageListener; +using search::AddressSpaceUsage; +using vespalib::AddressSpace; namespace { @@ -38,33 +42,47 @@ public: } }; +class MyListener : public IAttributeUsageListener { +public: + AttributeUsageStats stats; + MyListener() : stats() {} + void notify_attribute_usage(const AttributeUsageStats &stats_in) override { + stats = stats_in; + } +}; + struct Fixture { - AttributeUsageFilter _filter; + AttributeUsageFilter filter; + const MyListener* listener; using State = AttributeUsageFilter::State; using Config = AttributeUsageFilter::Config; Fixture() - : _filter() + : filter(), + listener() { + auto my_listener = std::make_unique<MyListener>(); + listener = my_listener.get(); + filter.set_listener(std::move(my_listener)); } void testWrite(const vespalib::string &exp) { if (exp.empty()) { - EXPECT_TRUE(_filter.acceptWriteOperation()); - State state = _filter.getAcceptState(); + EXPECT_TRUE(filter.acceptWriteOperation()); + State state = filter.getAcceptState(); EXPECT_TRUE(state.acceptWriteOperation()); EXPECT_EQUAL(exp, state.message()); } else { - EXPECT_FALSE(_filter.acceptWriteOperation()); - State state = _filter.getAcceptState(); + EXPECT_FALSE(filter.acceptWriteOperation()); + State state = filter.getAcceptState(); EXPECT_FALSE(state.acceptWriteOperation()); EXPECT_EQUAL(exp, state.message()); } } void setAttributeStats(const AttributeUsageStats &stats) { - _filter.setAttributeStats(stats); + filter.setAttributeStats(stats); } }; @@ -78,7 +96,7 @@ TEST_F("Check that default filter allows write", Fixture) TEST_F("Check that enum store limit can be reached", Fixture) { - f._filter.setConfig(Fixture::Config(0.8, 1.0)); + f.filter.setConfig(Fixture::Config(0.8, 1.0)); MyAttributeStats stats; stats.triggerEnumStoreLimit(); f.setAttributeStats(stats); @@ -95,7 +113,7 @@ TEST_F("Check that enum store limit can be reached", Fixture) TEST_F("Check that multivalue limit can be reached", Fixture) { - f._filter.setConfig(Fixture::Config(1.0, 0.8)); + f.filter.setConfig(Fixture::Config(1.0, 0.8)); MyAttributeStats stats; stats.triggerMultiValueLimit(); f.setAttributeStats(stats); @@ -113,7 +131,7 @@ TEST_F("Check that multivalue limit can be reached", Fixture) TEST_F("Check that both enumstore limit and multivalue limit can be reached", Fixture) { - f._filter.setConfig(Fixture::Config(0.8, 0.8)); + f.filter.setConfig(Fixture::Config(0.8, 0.8)); MyAttributeStats stats; stats.triggerEnumStoreLimit(); stats.triggerMultiValueLimit(); @@ -139,4 +157,13 @@ TEST_F("Check that both enumstore limit and multivalue limit can be reached", "attributeName: \"multiValueName\", subdb: \"ready\"}"); } +TEST_F("listener is updated when attribute stats change", Fixture) +{ + AttributeUsageStats stats; + AddressSpaceUsage usage(AddressSpace(12, 10, 15), AddressSpace(22, 20, 25)); + stats.merge(usage, "my_attr", "my_subdb"); + f.setAttributeStats(stats); + EXPECT_EQUAL(stats, f.listener->stats); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp index 8a814e25bd5..d6c40b0d70d 100644 --- a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp +++ b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp @@ -18,6 +18,7 @@ #include <vespa/vespalib/util/foreground_thread_executor.h> #include <vespa/vespalib/util/foregroundtaskexecutor.h> #include <vespa/vespalib/util/threadstackexecutor.h> +#include <thread> #include <vespa/log/log.h> LOG_SETUP("attributeflush_test"); diff --git a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp index e284f5ed1d9..cb64a1a59d8 100644 --- a/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp +++ b/searchcore/src/tests/proton/attribute/exclusive_attribute_read_accessor/exclusive_attribute_read_accessor_test.cpp @@ -40,7 +40,7 @@ TEST_F("require that attribute write thread is blocked while guard is held", Fix ReadGuard::UP guard = f.accessor.takeGuard(); Gate gate; f.writer->execute(f.writer->getExecutorIdFromName(f.attribute->getNamePrefix()), [&gate]() { gate.countDown(); }); - bool reachedZero = gate.await(100); + bool reachedZero = gate.await(100ms); EXPECT_FALSE(reachedZero); EXPECT_EQUAL(1u, gate.getCount()); diff --git a/searchcore/src/tests/proton/common/alloc_config/CMakeLists.txt b/searchcore/src/tests/proton/common/alloc_config/CMakeLists.txt new file mode 100644 index 00000000000..26a2dc72cc2 --- /dev/null +++ b/searchcore/src/tests/proton/common/alloc_config/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_alloc_config_test_app TEST + SOURCES + alloc_config_test.cpp + DEPENDS + searchcore_pcommon + GTest::GTest +) +vespa_add_test(NAME searchcore_alloc_config_test_app COMMAND searchcore_alloc_config_test_app) diff --git a/searchcore/src/tests/proton/common/alloc_config/alloc_config_test.cpp b/searchcore/src/tests/proton/common/alloc_config/alloc_config_test.cpp new file mode 100644 index 00000000000..18a0ee47a47 --- /dev/null +++ b/searchcore/src/tests/proton/common/alloc_config/alloc_config_test.cpp @@ -0,0 +1,35 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchcore/proton/common/alloc_config.h> +#include <vespa/searchcore/proton/common/subdbtype.h> +#include <vespa/vespalib/gtest/gtest.h> + +using proton::AllocConfig; +using proton::AllocStrategy; +using proton::SubDbType; +using search::CompactionStrategy; +using search::GrowStrategy; + +namespace { + +CompactionStrategy baseline_compaction_strategy(0.2, 0.25); + +GrowStrategy make_grow_strategy(uint32_t initial_docs) { + return GrowStrategy(initial_docs, 0.1, 1, 0.15); +} + +AllocStrategy make_alloc_strategy(uint32_t initial_docs) { + return AllocStrategy(make_grow_strategy(initial_docs), baseline_compaction_strategy, 10000); +} + +}; + +TEST(AllocConfigTest, can_make_allocation_strategy_for_sub_dbs) +{ + AllocConfig config(make_alloc_strategy(10000000), 5, 2); + EXPECT_EQ(make_alloc_strategy(20000000), config.make_alloc_strategy(SubDbType::READY)); + EXPECT_EQ(make_alloc_strategy(100000), config.make_alloc_strategy(SubDbType::REMOVED)); + EXPECT_EQ(make_alloc_strategy(30000000), config.make_alloc_strategy(SubDbType::NOTREADY)); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index d231040aeda..8f6b2f0bc10 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -613,6 +613,12 @@ TEST("require that attribute manager (imported attributes) should change when vi EXPECT_TRUE(params.shouldAttributeManagerChange()); } +TEST("require that attribute manager should change when alloc config has changed") +{ + ReconfigParams params(CCR().set_alloc_config_changed(true)); + EXPECT_TRUE(params.shouldAttributeManagerChange()); +} + void assertMaintenanceControllerShouldNotChange(DocumentDBConfig::ComparisonResult result) { @@ -684,6 +690,7 @@ TEST("require that subdbs should change if relevant config changed") TEST_DO(assertSubDbsShouldChange(CCR().setRankingConstantsChanged(true))); TEST_DO(assertSubDbsShouldChange(CCR().setOnnxModelsChanged(true))); TEST_DO(assertSubDbsShouldChange(CCR().setSchemaChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().set_alloc_config_changed(true))); } TEST_MAIN() diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index 9cdd984b152..40a9656ae41 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -123,8 +123,7 @@ struct MyStoreOnlyConfig : _cfg(DocTypeName(DOCTYPE_NAME), SUB_NAME, BASE_DIR, - search::GrowStrategy(), - 0, 0, SubDbType::READY) + 0, SubDbType::READY) { } }; diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp index 0cc9a92895b..a952efdecdc 100644 --- a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp @@ -10,6 +10,7 @@ #include <vespa/searchcore/proton/server/idocumentmovehandler.h> #include <vespa/searchcore/proton/server/imaintenancejobrunner.h> #include <vespa/searchcore/proton/server/maintenancedocumentsubdb.h> +#include <vespa/searchcore/proton/server/ibucketmodifiedhandler.h> #include <vespa/searchcore/proton/test/buckethandler.h> #include <vespa/searchcore/proton/test/clusterstatehandler.h> #include <vespa/searchcore/proton/test/disk_mem_usage_notifier.h> @@ -43,15 +44,8 @@ using BucketIdSet = std::set<BucketId>; using BucketIdVector = BucketId::List; using DocumentVector = std::vector<Document::SP>; using MoveOperationVector = std::vector<MoveOperation>; -using ScanItr = BucketMoveJob::ScanIterator; -using ScanPos = BucketMoveJob::ScanPosition; - -namespace { - -const uint32_t FIRST_SCAN_PASS = 1; -const uint32_t SECOND_SCAN_PASS = 2; - -} +using ScanItr = bucketdb::ScanIterator; +using ScanPass = ScanItr::Pass; struct MyMoveOperationLimiter : public IMoveOperationLimiter { uint32_t beginOpCount; @@ -346,9 +340,8 @@ struct ScanFixtureBase return ScanItr(_bucketDB->takeGuard(), BucketId()); } - ScanItr getItr(BucketId bucket, BucketId endBucket = BucketId(), uint32_t pass = FIRST_SCAN_PASS) { - return ScanItr(_bucketDB->takeGuard(), pass, - bucket, endBucket); + ScanItr getItr(BucketId bucket, BucketId endBucket = BucketId(), ScanPass pass = ScanPass::FIRST) { + return ScanItr(_bucketDB->takeGuard(), pass, bucket, endBucket); } }; @@ -451,12 +444,12 @@ TEST_F("require that we can iterate from the middle of not ready buckets", ScanF { BucketId bucket = f._notReady.bucket(2); { - ScanItr itr = f.getItr(bucket, bucket, FIRST_SCAN_PASS); + ScanItr itr = f.getItr(bucket, bucket, ScanPass::FIRST); assertEquals(BucketVector(). add(f._notReady.bucket(4)), itr, SubDbType::NOTREADY); } { - ScanItr itr = f.getItr(BucketId(), bucket, SECOND_SCAN_PASS); + ScanItr itr = f.getItr(BucketId(), bucket, ScanPass::SECOND); assertEquals(BucketVector(). add(f._notReady.bucket(2)), itr, SubDbType::NOTREADY); } @@ -478,12 +471,12 @@ TEST_F("require that we can iterate from the middle of ready buckets", ScanFixtu add(f._notReady.bucket(4)), itr, SubDbType::NOTREADY); } { - ScanItr itr = f.getItr(bucket, bucket, FIRST_SCAN_PASS); + ScanItr itr = f.getItr(bucket, bucket, ScanPass::FIRST); assertEquals(BucketVector(). add(f._ready.bucket(8)), itr, SubDbType::READY); } { - ScanItr itr = f.getItr(BucketId(), bucket, SECOND_SCAN_PASS); + ScanItr itr = f.getItr(BucketId(), bucket, ScanPass::SECOND); assertEquals(BucketVector(). add(f._ready.bucket(6)), itr, SubDbType::READY); } diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp index 1127191a1a2..8840d4778e0 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp @@ -78,20 +78,20 @@ struct Rendezvous { vespalib::Gate gone; typedef std::unique_ptr<Rendezvous> UP; Rendezvous() : enter(), leave(), gone() {} - bool run(uint32_t timeout = 80000) { + bool run(vespalib::duration timeout = 80s) { enter.countDown(); bool retval = leave.await(timeout); gone.countDown(); return retval; } - bool waitForEnter(uint32_t timeout = 80000) { + bool waitForEnter(vespalib::duration timeout = 80s) { return enter.await(timeout); } - bool leaveAndWait(uint32_t timeout = 80000) { + bool leaveAndWait(vespalib::duration timeout = 80s) { leave.countDown(); return gone.await(timeout); } - bool await(uint32_t timeout = 80000) { + bool await(vespalib::duration timeout = 80s) { if (waitForEnter(timeout)) { return leaveAndWait(timeout); } @@ -373,7 +373,7 @@ struct FeedTokenContext { FeedTokenContext(); ~FeedTokenContext(); - bool await(uint32_t timeout = 80000) { return transport.gate.await(timeout); } + bool await(vespalib::duration timeout = 80s) { return transport.gate.await(timeout); } const Result *getResult() { if (transport.result.get()) { return transport.result.get(); @@ -399,36 +399,6 @@ struct PutContext { {} }; - -struct PutHandler { - FeedHandler &handler; - DocBuilder &builder; - Timestamp timestamp; - std::vector<PutContext::SP> puts; - PutHandler(FeedHandler &fh, DocBuilder &db) : - handler(fh), - builder(db), - timestamp(0), - puts() - {} - void put(const vespalib::string &docId) { - PutContext::SP pc(new PutContext(docId, builder)); - FeedOperation::UP op(new PutOperation(pc->docCtx.bucketId, timestamp, pc->docCtx.doc)); - handler.handleOperation(pc->tokenCtx.token, std::move(op)); - timestamp = Timestamp(timestamp + 1); - puts.push_back(pc); - } - bool await(uint32_t timeout = 80000) { - for (const auto & put : puts) { - if (!put->tokenCtx.await(timeout)) { - return false; - } - } - return true; - } -}; - - struct MyTlsWriter : TlsWriter { int store_count; int erase_count; diff --git a/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp index 2bde5761aea..c12b91c4c1e 100644 --- a/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp +++ b/searchcore/src/tests/proton/documentdb/job_tracked_maintenance_job/job_tracked_maintenance_job_test.cpp @@ -24,7 +24,7 @@ getGateVector(size_t size) { GateVector retval; for (size_t i = 0; i < size; ++i) { - retval.push_back(GateUP(new Gate())); + retval.push_back(std::make_unique<Gate>()); } return retval; } @@ -46,7 +46,7 @@ struct MyMaintenanceJob : public IBlockableMaintenanceJob void unBlock(BlockedReason) override { _blocked = false; } bool isBlocked() const override { return _blocked; } bool run() override { - _runGates[_runIdx++]->await(5000); + _runGates[_runIdx++]->await(5s); return _runIdx == _runGates.size(); } }; @@ -82,10 +82,10 @@ struct Fixture } void runJobAndWait(size_t runIdx, size_t startedGateCount, size_t endedGateCount) { _exec.execute(vespalib::makeLambdaTask([this]() { runJob(); })); - _tracker->_started.await(5000); + _tracker->_started.await(5s); assertTracker(startedGateCount, endedGateCount); _myJob->_runGates[runIdx]->countDown(); - _runGates[runIdx]->await(5000); + _runGates[runIdx]->await(5s); } }; diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index c6ffa167dbf..44afe1ddd75 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -72,8 +72,7 @@ using BlockedReason = IBlockableMaintenanceJob::BlockedReason; typedef BucketId::List BucketIdVector; typedef std::set<BucketId> BucketIdSet; -constexpr int TIMEOUT_MS = 60000; -constexpr vespalib::duration TIMEOUT_SEC = 60s; +constexpr vespalib::duration TIMEOUT = 60s; namespace { @@ -947,14 +946,14 @@ TEST_F("require that bucket move controller is active", MaintenanceControllerFix EXPECT_EQUAL(5u, f._notReady.getNumUsedLids()); EXPECT_EQUAL(5u, f._notReady.getDocumentCount()); f.startMaintenance(); - ASSERT_TRUE(f._executor.waitIdle(TIMEOUT_SEC)); + ASSERT_TRUE(f._executor.waitIdle(TIMEOUT)); EXPECT_EQUAL(0u, f._ready.getNumUsedLids()); EXPECT_EQUAL(0u, f._ready.getDocumentCount()); EXPECT_EQUAL(10u, f._notReady.getNumUsedLids()); EXPECT_EQUAL(10u, f._notReady.getDocumentCount()); f._calc->addReady(bucketId1); f.notifyClusterStateChanged(); - ASSERT_TRUE(f._executor.waitIdle(TIMEOUT_SEC)); + ASSERT_TRUE(f._executor.waitIdle(TIMEOUT)); EXPECT_EQUAL(3u, f._ready.getNumUsedLids()); EXPECT_EQUAL(3u, f._ready.getDocumentCount()); EXPECT_EQUAL(7u, f._notReady.getNumUsedLids()); @@ -963,13 +962,13 @@ TEST_F("require that bucket move controller is active", MaintenanceControllerFix f._calc->addReady(bucketId2); f._calc->addReady(bucketId4); f.notifyClusterStateChanged(); - ASSERT_TRUE(f._executor.waitIdle(TIMEOUT_SEC)); + ASSERT_TRUE(f._executor.waitIdle(TIMEOUT)); EXPECT_EQUAL(6u, f._ready.getNumUsedLids()); EXPECT_EQUAL(6u, f._ready.getDocumentCount()); EXPECT_EQUAL(4u, f._notReady.getNumUsedLids()); EXPECT_EQUAL(4u, f._notReady.getDocumentCount()); frozen2.reset(); - ASSERT_TRUE(f._executor.waitIdle(TIMEOUT_SEC)); + ASSERT_TRUE(f._executor.waitIdle(TIMEOUT)); EXPECT_EQUAL(8u, f._ready.getNumUsedLids()); EXPECT_EQUAL(8u, f._ready.getDocumentCount()); EXPECT_EQUAL(2u, f._notReady.getNumUsedLids()); @@ -997,14 +996,14 @@ TEST_F("require that document pruner is active", MaintenanceControllerFixture) EXPECT_EQUAL(10u, f._removed.getNumUsedLids()); EXPECT_EQUAL(10u, f._removed.getDocumentCount()); f.startMaintenance(); - ASSERT_TRUE(f._executor.waitIdle(TIMEOUT_SEC)); + ASSERT_TRUE(f._executor.waitIdle(TIMEOUT)); EXPECT_EQUAL(10u, f._removed.getNumUsedLids()); EXPECT_EQUAL(10u, f._removed.getDocumentCount()); MyFrozenBucket::UP frozen3(new MyFrozenBucket(f._mc, bucketId3)); f.setPruneConfig(DocumentDBPruneRemovedDocumentsConfig(200ms, 900s)); for (uint32_t i = 0; i < 6; ++i) { std::this_thread::sleep_for(100ms); - ASSERT_TRUE(f._executor.waitIdle(TIMEOUT_SEC)); + ASSERT_TRUE(f._executor.waitIdle(TIMEOUT)); if (f._removed.getNumUsedLids() != 10u) break; } @@ -1013,7 +1012,7 @@ TEST_F("require that document pruner is active", MaintenanceControllerFixture) frozen3.reset(); for (uint32_t i = 0; i < 600; ++i) { std::this_thread::sleep_for(100ms); - ASSERT_TRUE(f._executor.waitIdle(TIMEOUT_SEC)); + ASSERT_TRUE(f._executor.waitIdle(TIMEOUT)); if (f._removed.getNumUsedLids() != 10u) break; } @@ -1075,7 +1074,7 @@ TEST_F("require that active bucket is not moved until de-activated", Maintenance EXPECT_EQUAL(5u, f._notReady.getDocumentCount()); f.startMaintenance(); - ASSERT_TRUE(f._executor.waitIdle(TIMEOUT_SEC)); + ASSERT_TRUE(f._executor.waitIdle(TIMEOUT)); EXPECT_EQUAL(5u, f._ready.getNumUsedLids()); EXPECT_EQUAL(5u, f._ready.getDocumentCount()); EXPECT_EQUAL(5u, f._notReady.getNumUsedLids()); @@ -1084,7 +1083,7 @@ TEST_F("require that active bucket is not moved until de-activated", Maintenance // de-activate bucket 1 f._ready.setBucketState(readyDocs.getBucket(1), false); f.notifyBucketStateChanged(readyDocs.getBucket(1), BucketInfo::NOT_ACTIVE); - ASSERT_TRUE(f._executor.waitIdle(TIMEOUT_SEC)); + ASSERT_TRUE(f._executor.waitIdle(TIMEOUT)); EXPECT_EQUAL(2u, f._ready.getNumUsedLids()); EXPECT_EQUAL(2u, f._ready.getDocumentCount()); EXPECT_EQUAL(8u, f._notReady.getNumUsedLids()); @@ -1093,7 +1092,7 @@ TEST_F("require that active bucket is not moved until de-activated", Maintenance // re-activate bucket 1 f._ready.setBucketState(readyDocs.getBucket(1), true); f.notifyBucketStateChanged(readyDocs.getBucket(1), BucketInfo::ACTIVE); - ASSERT_TRUE(f._executor.waitIdle(TIMEOUT_SEC)); + ASSERT_TRUE(f._executor.waitIdle(TIMEOUT)); EXPECT_EQUAL(5u, f._ready.getNumUsedLids()); EXPECT_EQUAL(5u, f._ready.getDocumentCount()); EXPECT_EQUAL(5u, f._notReady.getNumUsedLids()); @@ -1102,7 +1101,7 @@ TEST_F("require that active bucket is not moved until de-activated", Maintenance // de-activate bucket 1 f._ready.setBucketState(readyDocs.getBucket(1), false); f.notifyBucketStateChanged(readyDocs.getBucket(1), BucketInfo::NOT_ACTIVE); - ASSERT_TRUE(f._executor.waitIdle(TIMEOUT_SEC)); + ASSERT_TRUE(f._executor.waitIdle(TIMEOUT)); EXPECT_EQUAL(2u, f._ready.getNumUsedLids()); EXPECT_EQUAL(2u, f._ready.getDocumentCount()); EXPECT_EQUAL(8u, f._notReady.getNumUsedLids()); @@ -1111,7 +1110,7 @@ TEST_F("require that active bucket is not moved until de-activated", Maintenance // re-activate bucket 1 f._ready.setBucketState(readyDocs.getBucket(1), true); f.notifyBucketStateChanged(readyDocs.getBucket(1), BucketInfo::ACTIVE); - ASSERT_TRUE(f._executor.waitIdle(TIMEOUT_SEC)); + ASSERT_TRUE(f._executor.waitIdle(TIMEOUT)); EXPECT_EQUAL(5u, f._ready.getNumUsedLids()); EXPECT_EQUAL(5u, f._ready.getDocumentCount()); EXPECT_EQUAL(5u, f._notReady.getNumUsedLids()); @@ -1125,19 +1124,19 @@ TEST_F("require that a simple maintenance job is executed", MaintenanceControlle f._mc.registerJobInMasterThread(std::move(job)); f._injectDefaultJobs = false; f.startMaintenance(); - bool done = myJob._latch.await(TIMEOUT_MS); + bool done = myJob._latch.await(TIMEOUT); EXPECT_TRUE(done); EXPECT_EQUAL(0u, myJob._latch.getCount()); } TEST_F("require that a split maintenance job is executed", MaintenanceControllerFixture) { - auto job = std::make_unique<MySplitJob>(200ms, TIMEOUT_SEC * 2, 3); + auto job = std::make_unique<MySplitJob>(200ms, TIMEOUT * 2, 3); MySplitJob &myJob = *job; f._mc.registerJobInMasterThread(std::move(job)); f._injectDefaultJobs = false; f.startMaintenance(); - bool done = myJob._latch.await(TIMEOUT_MS); + bool done = myJob._latch.await(TIMEOUT); EXPECT_TRUE(done); EXPECT_EQUAL(0u, myJob._latch.getCount()); } @@ -1145,9 +1144,9 @@ TEST_F("require that a split maintenance job is executed", MaintenanceController TEST_F("require that a blocked job is unblocked and executed after thaw bucket", MaintenanceControllerFixture) { - auto job1 = std::make_unique<MySimpleJob>(TIMEOUT_SEC * 2, TIMEOUT_SEC * 2, 1); + auto job1 = std::make_unique<MySimpleJob>(TIMEOUT * 2, TIMEOUT * 2, 1); MySimpleJob &myJob1 = *job1; - auto job2 = std::make_unique< MySimpleJob>(TIMEOUT_SEC * 2, TIMEOUT_SEC * 2, 0); + auto job2 = std::make_unique< MySimpleJob>(TIMEOUT * 2, TIMEOUT * 2, 0); MySimpleJob &myJob2 = *job2; f._mc.registerJobInMasterThread(std::move(job1)); f._mc.registerJobInMasterThread(std::move(job2)); @@ -1169,7 +1168,7 @@ TEST_F("require that a blocked job is unblocked and executed after thaw bucket", f._executor.sync(); EXPECT_FALSE(myJob1.isBlocked()); EXPECT_FALSE(myJob2.isBlocked()); - bool done1 = myJob1._latch.await(TIMEOUT_MS); + bool done1 = myJob1._latch.await(TIMEOUT); EXPECT_TRUE(done1); std::this_thread::sleep_for(2s); EXPECT_EQUAL(0u, myJob2._runCnt); @@ -1190,14 +1189,14 @@ TEST_F("require that blocked jobs are not executed", MaintenanceControllerFixtur TEST_F("require that maintenance controller state list jobs", MaintenanceControllerFixture) { { - IMaintenanceJob::UP job1(new MySimpleJob(TIMEOUT_SEC * 2, TIMEOUT_SEC * 2, 0)); - IMaintenanceJob::UP job2(new MyLongRunningJob(200ms, 200ms)); + auto job1 = std::make_unique<MySimpleJob>(TIMEOUT * 2, TIMEOUT * 2, 0); + auto job2 = std::make_unique<MyLongRunningJob>(200ms, 200ms); auto &longRunningJob = dynamic_cast<MyLongRunningJob &>(*job2); f._mc.registerJobInMasterThread(std::move(job1)); f._mc.registerJobInMasterThread(std::move(job2)); f._injectDefaultJobs = false; f.startMaintenance(); - longRunningJob._firstRun.await(TIMEOUT_MS); + longRunningJob._firstRun.await(TIMEOUT); } MaintenanceControllerExplorer explorer(f._mc.getJobList()); diff --git a/searchcore/src/tests/proton/flushengine/flushengine_test.cpp b/searchcore/src/tests/proton/flushengine/flushengine_test.cpp index a675a45aa54..d5823a8e055 100644 --- a/searchcore/src/tests/proton/flushengine/flushengine_test.cpp +++ b/searchcore/src/tests/proton/flushengine/flushengine_test.cpp @@ -14,7 +14,7 @@ #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/testkit/testapp.h> #include <mutex> -#include <chrono> +#include <thread> #include <vespa/log/log.h> LOG_SETUP("flushengine_test"); @@ -31,9 +31,9 @@ using searchcorespi::IFlushTarget; using searchcorespi::FlushTask; using vespalib::Slime; -const long LONG_TIMEOUT = 66666; -const long SHORT_TIMEOUT = 1; -const uint32_t IINTERVAL = 1000; +constexpr vespalib::duration LONG_TIMEOUT = 66666ms; +constexpr vespalib::duration SHORT_TIMEOUT = 1ms; +constexpr vespalib::duration IINTERVAL = 1s; class SimpleExecutor : public vespalib::Executor { public: @@ -42,8 +42,7 @@ public: public: SimpleExecutor() : _done() - { - } + { } Task::UP execute(Task::UP task) override @@ -83,8 +82,7 @@ public: SimpleHandler &handler) : _task(std::move(task)), _handler(handler) - { - } + { } search::SerialNum getFlushSerial() const override { return _task->getFlushSerial(); @@ -95,19 +93,15 @@ class WrappedFlushTarget : public FlushTargetProxy { SimpleHandler &_handler; public: - WrappedFlushTarget(const IFlushTarget::SP &target, - SimpleHandler &handler) + WrappedFlushTarget(const IFlushTarget::SP &target, SimpleHandler &handler) : FlushTargetProxy(target), _handler(handler) - { - } + { } - Task::UP initFlush(SerialNum currentSerial, std::shared_ptr<search::IFlushToken> flush_token) override - { + Task::UP initFlush(SerialNum currentSerial, std::shared_ptr<search::IFlushToken> flush_token) override { Task::UP task(_target->initFlush(currentSerial, std::move(flush_token))); if (task) { - return std::make_unique<WrappedFlushTask>(std::move(task), - _handler); + return std::make_unique<WrappedFlushTask>(std::move(task), _handler); } return task; } @@ -140,33 +134,25 @@ public: _lock(), _done(targets.size()), _flushDoneHistory() - { - } + { } - search::SerialNum - getCurrentSerialNumber() const override - { - LOG(info, "SimpleHandler(%s)::getCurrentSerialNumber()", - getName().c_str()); + search::SerialNum getCurrentSerialNumber() const override { + LOG(info, "SimpleHandler(%s)::getCurrentSerialNumber()", getName().c_str()); return _currentSerial; } std::vector<IFlushTarget::SP> - getFlushTargets() override - { - LOG(info, "SimpleHandler(%s)::getFlushTargets()", - getName().c_str()); + getFlushTargets() override { + LOG(info, "SimpleHandler(%s)::getFlushTargets()", getName().c_str()); std::vector<IFlushTarget::SP> wrappedTargets; for (const auto &target : _targets) { - wrappedTargets.push_back(std::make_shared<WrappedFlushTarget> - (target, *this)); + wrappedTargets.push_back(std::make_shared<WrappedFlushTarget>(target, *this)); } return wrappedTargets; } // Called once by flush engine thread for each task done - void taskDone() - { + void taskDone() { std::lock_guard<std::mutex> guard(_lock); ++_pendingDone; } @@ -174,12 +160,9 @@ public: // Called by flush engine master thread after flush handler is // added to flush engine and when one or more flush tasks related // to flush handler have completed. - void - flushDone(search::SerialNum oldestSerial) override - { + void flushDone(search::SerialNum oldestSerial) override { std::lock_guard<std::mutex> guard(_lock); - LOG(info, "SimpleHandler(%s)::flushDone(%" PRIu64 ")", - getName().c_str(), oldestSerial); + LOG(info, "SimpleHandler(%s)::flushDone(%" PRIu64 ")", getName().c_str(), oldestSerial); _oldestSerial = std::max(_oldestSerial, oldestSerial); _flushDoneHistory.push_back(oldestSerial); while (_pendingDone > 0) { @@ -188,8 +171,7 @@ public: } } - FlushDoneHistory getFlushDoneHistory() - { + FlushDoneHistory getFlushDoneHistory() { std::lock_guard<std::mutex> guard(_lock); return _flushDoneHistory; } @@ -217,12 +199,11 @@ public: search::SerialNum ¤tSerial) : _flushedSerial(flushedSerial), _currentSerial(currentSerial), _start(start), _done(done), _proceed(proceed) - { - } + { } void run() override { _start.countDown(); - if (_proceed != NULL) { + if (_proceed != nullptr) { _proceed->await(); } _flushedSerial = _currentSerial; @@ -270,8 +251,7 @@ public: _taskStart(), _taskDone(), _task(std::move(task)) - { - } + { } SimpleTarget(search::SerialNum flushedSerial = 0, bool proceedImmediately = true) : SimpleTarget("anon", flushedSerial, proceedImmediately) @@ -316,8 +296,7 @@ public: : SimpleTarget("anon"), _mgain(false), _serial(false) - { - } + { } MemoryGain getApproxMemoryGain() const override { LOG_ASSERT(_mgain == false); @@ -389,8 +368,7 @@ public: class NoFlushStrategy : public SimpleStrategy { - FlushContext::List getFlushTargets(const FlushContext::List &, - const flushengine::TlsStatsMap &) const override { + FlushContext::List getFlushTargets(const FlushContext::List &, const flushengine::TlsStatsMap &) const override { return FlushContext::List(); } }; @@ -426,17 +404,15 @@ struct Fixture SimpleStrategy::SP strategy; FlushEngine engine; - Fixture(uint32_t numThreads, uint32_t idleIntervalMS, SimpleStrategy::SP strategy_) + Fixture(uint32_t numThreads, vespalib::duration idleInterval, SimpleStrategy::SP strategy_) : tlsStatsFactory(std::make_shared<SimpleTlsStatsFactory>()), strategy(strategy_), - engine(tlsStatsFactory, strategy, numThreads, idleIntervalMS) - { - } + engine(tlsStatsFactory, strategy, numThreads, idleInterval) + { } - Fixture(uint32_t numThreads, uint32_t idleIntervalMS) - : Fixture(numThreads, idleIntervalMS, std::make_shared<SimpleStrategy>()) - { - } + Fixture(uint32_t numThreads, vespalib::duration idleInterval) + : Fixture(numThreads, idleInterval, std::make_shared<SimpleStrategy>()) + { } void putFlushHandler(const vespalib::string &docTypeName, IFlushHandler::SP handler) { engine.putFlushHandler(DocTypeName(docTypeName), handler); @@ -446,17 +422,14 @@ struct Fixture strategy->_targets.push_back(std::move(target)); } - std::shared_ptr<SimpleHandler> - addSimpleHandler(Targets targets) - { + std::shared_ptr<SimpleHandler> addSimpleHandler(Targets targets) { auto handler = std::make_shared<SimpleHandler>(targets, "handler", 20); engine.putFlushHandler(DocTypeName("handler"), handler); engine.start(); return handler; } - void assertOldestSerial(SimpleHandler &handler, search::SerialNum expOldestSerial) - { + void assertOldestSerial(SimpleHandler &handler, search::SerialNum expOldestSerial) { using namespace std::chrono_literals; for (int pass = 0; pass < 600; ++pass) { std::this_thread::sleep_for(100ms); @@ -488,12 +461,12 @@ TEST_F("require that strategy controls flush target", Fixture(1, IINTERVAL)) EXPECT_EQUAL("bar", order[1]); } -TEST_F("require that zero handlers does not core", Fixture(2, 50)) +TEST_F("require that zero handlers does not core", Fixture(2, 50ms)) { f.engine.start(); } -TEST_F("require that zero targets does not core", Fixture(2, 50)) +TEST_F("require that zero targets does not core", Fixture(2, 50ms)) { f.putFlushHandler("foo", std::make_shared<SimpleHandler>(Targets(), "foo")); f.putFlushHandler("bar", std::make_shared<SimpleHandler>(Targets(), "bar")); @@ -593,8 +566,7 @@ TEST_F("require that target can refuse flush", Fixture(2, IINTERVAL)) EXPECT_TRUE(!handler->_done.await(SHORT_TIMEOUT)); } -TEST_F("require that targets are flushed when nothing new to flush", - Fixture(2, IINTERVAL)) +TEST_F("require that targets are flushed when nothing new to flush", Fixture(2, IINTERVAL)) { auto target = std::make_shared<SimpleTarget>("anon", 5); // oldest unflushed serial num = 5 auto handler = std::make_shared<SimpleHandler>(Targets({target}), "anon", 4); // current serial num = 4 @@ -640,7 +612,7 @@ TEST("require that threaded target works") auto target = std::make_shared<ThreadedFlushTarget>(executor, getSerialNum, std::make_shared<SimpleTarget>()); EXPECT_FALSE(executor._done.await(SHORT_TIMEOUT)); - EXPECT_TRUE(target->initFlush(0, std::make_shared<search::FlushToken>()).get() != NULL); + EXPECT_TRUE(target->initFlush(0, std::make_shared<search::FlushToken>())); EXPECT_TRUE(executor._done.await(LONG_TIMEOUT)); } @@ -692,7 +664,7 @@ assertThatHandlersInCurrentSet(FlushEngine & engine, const std::vector<const cha } } -TEST_F("require that concurrency works", Fixture(2, 1)) +TEST_F("require that concurrency works", Fixture(2, 1ms)) { auto target1 = std::make_shared<SimpleTarget>("target1", 1, false); auto target2 = std::make_shared<SimpleTarget>("target2", 2, false); @@ -713,7 +685,31 @@ TEST_F("require that concurrency works", Fixture(2, 1)) target2->_proceed.countDown(); } -TEST_F("require that state explorer can list flush targets", Fixture(1, 1)) +TEST_F("require that concurrency works with triggerFlush", Fixture(2, 1ms)) +{ + auto target1 = std::make_shared<SimpleTarget>("target1", 1, false); + auto target2 = std::make_shared<SimpleTarget>("target2", 2, false); + auto target3 = std::make_shared<SimpleTarget>("target3", 3, false); + auto handler = std::make_shared<SimpleHandler>(Targets({target1, target2, target3}), "handler", 9); + f.putFlushHandler("handler", handler); + std::thread thread([this]() { f.engine.triggerFlush(); }); + std::this_thread::sleep_for(1s); + f.engine.start(); + + EXPECT_TRUE(target1->_initDone.await(LONG_TIMEOUT)); + EXPECT_TRUE(target2->_initDone.await(LONG_TIMEOUT)); + EXPECT_TRUE(!target3->_initDone.await(SHORT_TIMEOUT)); + assertThatHandlersInCurrentSet(f.engine, {"handler.target1", "handler.target2"}); + EXPECT_TRUE(!target3->_initDone.await(SHORT_TIMEOUT)); + target1->_proceed.countDown(); + EXPECT_TRUE(target1->_taskDone.await(LONG_TIMEOUT)); + assertThatHandlersInCurrentSet(f.engine, {"handler.target2", "handler.target3"}); + target3->_proceed.countDown(); + target2->_proceed.countDown(); + thread.join(); +} + +TEST_F("require that state explorer can list flush targets", Fixture(1, 1ms)) { auto target = std::make_shared<SimpleTarget>("target1", 100, false); f.putFlushHandler("handler", @@ -744,7 +740,7 @@ TEST_F("require that state explorer can list flush targets", Fixture(1, 1)) target->_taskDone.await(LONG_TIMEOUT); } -TEST_F("require that oldest serial is updated when closing engine", Fixture(1, 100)) +TEST_F("require that oldest serial is updated when closing engine", Fixture(1, 100ms)) { auto target1 = std::make_shared<SimpleTarget>("target1", 10, false); auto handler = f.addSimpleHandler({ target1 }); @@ -754,7 +750,7 @@ TEST_F("require that oldest serial is updated when closing engine", Fixture(1, 1 EXPECT_EQUAL(20u, handler->_oldestSerial); } -TEST_F("require that oldest serial is updated when finishing priority flush strategy", Fixture(1, 100, std::make_shared<NoFlushStrategy>())) +TEST_F("require that oldest serial is updated when finishing priority flush strategy", Fixture(1, 100ms, std::make_shared<NoFlushStrategy>())) { auto target1 = std::make_shared<SimpleTarget>("target1", 10, true); auto handler = f.addSimpleHandler({ target1 }); diff --git a/searchcore/src/tests/proton/matching/docid_range_scheduler/docid_range_scheduler_test.cpp b/searchcore/src/tests/proton/matching/docid_range_scheduler/docid_range_scheduler_test.cpp index 4814ab4cb49..510269f2195 100644 --- a/searchcore/src/tests/proton/matching/docid_range_scheduler/docid_range_scheduler_test.cpp +++ b/searchcore/src/tests/proton/matching/docid_range_scheduler/docid_range_scheduler_test.cpp @@ -1,9 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/time_bomb.h> #include <vespa/searchcore/proton/matching/docid_range_scheduler.h> -#include <chrono> -#include <thread> using namespace proton::matching; using vespalib::TimeBomb; diff --git a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp index c108e98cc2a..1b9c60b379e 100644 --- a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp +++ b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/documentdb_job_trackers_test.cpp @@ -4,6 +4,7 @@ #include <vespa/searchcore/proton/metrics/job_tracked_flush_target.h> #include <vespa/searchcore/proton/test/dummy_flush_target.h> #include <vespa/vespalib/testkit/testapp.h> +#include <thread> #include <vespa/log/log.h> LOG_SETUP("documentdb_job_trackers_test"); diff --git a/searchcore/src/tests/proton/metrics/job_tracked_flush/job_tracked_flush_test.cpp b/searchcore/src/tests/proton/metrics/job_tracked_flush/job_tracked_flush_test.cpp index 7c22aab2285..20d75972c8b 100644 --- a/searchcore/src/tests/proton/metrics/job_tracked_flush/job_tracked_flush_test.cpp +++ b/searchcore/src/tests/proton/metrics/job_tracked_flush/job_tracked_flush_test.cpp @@ -27,8 +27,8 @@ struct MyFlushTask : public searchcorespi::FlushTask MyFlushTask(Gate &execGate) : _execGate(execGate) {} // Implements searchcorespi::FlushTask - virtual void run() override { - _execGate.await(5000); + void run() override { + _execGate.await(5s); } virtual search::SerialNum getFlushSerial() const override { return 5; } }; @@ -39,7 +39,7 @@ struct MyFlushTarget : public test::DummyFlushTarget SerialNum _initFlushSerial; Gate _execGate; Gate _initGate; - MyFlushTarget() + MyFlushTarget() noexcept : test::DummyFlushTarget("mytarget", Type::FLUSH, Component::OTHER), _initFlushSerial(0), _execGate(), @@ -50,8 +50,8 @@ struct MyFlushTarget : public test::DummyFlushTarget FlushTask::UP initFlush(SerialNum currentSerial, std::shared_ptr<search::IFlushToken>) override { if (currentSerial > 0) { _initFlushSerial = currentSerial; - _initGate.await(5000); - return FlushTask::UP(new MyFlushTask(_execGate)); + _initGate.await(5s); + return std::make_unique<MyFlushTask>(_execGate); } return FlushTask::UP(); } @@ -66,8 +66,8 @@ struct Fixture Gate _taskGate; ThreadStackExecutor _exec; Fixture(uint32_t numJobTrackings = 1) - : _tracker(new SimpleJobTracker(numJobTrackings)), - _target(new MyFlushTarget()), + : _tracker(std::make_shared<SimpleJobTracker>(numJobTrackings)), + _target(std::make_shared<MyFlushTarget>()), _trackedFlush(_tracker, _target), _task(), _taskGate(), @@ -95,12 +95,12 @@ TEST_F("require that flush task init is tracked", Fixture) EXPECT_EQUAL(1u, f._tracker->_ended.getCount()); f._exec.execute(makeLambdaTask([&]() {f.initFlush(FLUSH_SERIAL); })); - f._tracker->_started.await(5000); + f._tracker->_started.await(5s); EXPECT_EQUAL(0u, f._tracker->_started.getCount()); EXPECT_EQUAL(1u, f._tracker->_ended.getCount()); f._target->_initGate.countDown(); - f._taskGate.await(5000); + f._taskGate.await(5s); EXPECT_EQUAL(0u, f._tracker->_ended.getCount()); { JobTrackedFlushTask *trackedTask = dynamic_cast<JobTrackedFlushTask *>(f._task.get()); @@ -114,18 +114,18 @@ TEST_F("require that flush task execution is tracked", Fixture(2)) { f._exec.execute(makeLambdaTask([&]() { f.initFlush(FLUSH_SERIAL); })); f._target->_initGate.countDown(); - f._taskGate.await(5000); + f._taskGate.await(5s); EXPECT_EQUAL(1u, f._tracker->_started.getCount()); EXPECT_EQUAL(1u, f._tracker->_ended.getCount()); f._exec.execute(std::move(f._task)); - f._tracker->_started.await(5000); + f._tracker->_started.await(5s); EXPECT_EQUAL(0u, f._tracker->_started.getCount()); EXPECT_EQUAL(1u, f._tracker->_ended.getCount()); f._target->_execGate.countDown(); - f._tracker->_ended.await(5000); + f._tracker->_ended.await(5s); EXPECT_EQUAL(0u, f._tracker->_ended.getCount()); } diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp index 93fe2f0ae24..2e53b97a878 100644 --- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp +++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp @@ -6,7 +6,9 @@ #include <vespa/searchcore/proton/server/proton_config_fetcher.h> #include <vespa/searchcore/proton/server/proton_config_snapshot.h> #include <vespa/searchcore/proton/server/i_proton_configurer.h> +#include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/common/hw_info.h> +#include <vespa/searchcore/proton/common/subdbtype.h> #include <vespa/searchcore/config/config-ranking-constants.h> #include <vespa/searchcore/config/config-onnx-models.h> #include <vespa/searchsummary/config/config-juniperrc.h> @@ -41,6 +43,8 @@ using document::DocumenttypesConfigBuilder; using search::TuneFileDocumentDB; using std::map; using vespalib::VarHolder; +using search::GrowStrategy; +using search::CompactionStrategy; struct DoctypeFixture { using UP = std::unique_ptr<DoctypeFixture>; @@ -389,6 +393,32 @@ TEST_FF("require that docstore config computes cachesize automatically if unset" EXPECT_EQUAL(500000ul, config->getStoreConfig().getMaxCacheBytes()); } +TEST_FF("require that allocation config is propagated", + ConfigTestFixture("test"), + DocumentDBConfigManager(f1.configId + "/test", "test")) +{ + f1.protonBuilder.distribution.redundancy = 5; + f1.protonBuilder.distribution.searchablecopies = 2; + f1.addDocType("test"); + { + auto& allocation = f1.protonBuilder.documentdb.back().allocation; + allocation.initialnumdocs = 10000000; + allocation.growfactor = 0.1; + allocation.growbias = 1; + allocation.amortizecount = 10000; + allocation.multivaluegrowfactor = 0.15; + allocation.maxDeadBytesRatio = 0.25; + allocation.maxDeadAddressSpaceRatio = 0.3; + } + auto config = getDocumentDBConfig(f1, f2); + { + auto& alloc_config = config->get_alloc_config(); + EXPECT_EQUAL(AllocStrategy(GrowStrategy(20000000, 0.1, 1, 0.15), CompactionStrategy(0.25, 0.3), 10000), alloc_config.make_alloc_strategy(SubDbType::READY)); + EXPECT_EQUAL(AllocStrategy(GrowStrategy(100000, 0.1, 1, 0.15), CompactionStrategy(0.25, 0.3), 10000), alloc_config.make_alloc_strategy(SubDbType::REMOVED)); + EXPECT_EQUAL(AllocStrategy(GrowStrategy(30000000, 0.1, 1, 0.15), CompactionStrategy(0.25, 0.3), 10000), alloc_config.make_alloc_strategy(SubDbType::NOTREADY)); + } +} + TEST("test HwInfo equality") { EXPECT_TRUE(HwInfo::Cpu(1) == HwInfo::Cpu(1)); EXPECT_FALSE(HwInfo::Cpu(1) == HwInfo::Cpu(2)); diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp index 5a0cfc18c78..4f8e8e8aa8c 100644 --- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp +++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp @@ -9,6 +9,7 @@ #include <vespa/config-summarymap.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/fileacquirer/config-filedistributorrpc.h> +#include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/server/bootstrapconfig.h> #include <vespa/searchcore/proton/server/bootstrapconfigmanager.h> #include <vespa/searchcore/proton/server/documentdbconfigmanager.h> @@ -103,6 +104,7 @@ struct DBConfigFixture { std::make_shared<DocumentDBMaintenanceConfig>(), search::LogDocumentStore::Config(), std::make_shared<const ThreadingServiceConfig>(ThreadingServiceConfig::make(1)), + std::make_shared<const AllocConfig>(), configId, docTypeName); } diff --git a/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp b/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp index f028de23be6..63c861a60d9 100644 --- a/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp +++ b/searchcore/src/tests/proton/reference/gid_to_lid_change_listener/gid_to_lid_change_listener_test.cpp @@ -7,6 +7,7 @@ #include <vespa/searchcore/proton/reference/gid_to_lid_change_listener.h> #include <vespa/searchlib/common/i_gid_to_lid_mapper_factory.h> #include <vespa/vespalib/util/destructor_callbacks.h> +#include <vespa/vespalib/util/gate.h> #include <vespa/searchlib/test/mock_gid_to_lid_mapping.h> #include <map> #include <vespa/log/log.h> diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index 17d0ef8ad37..fb845982d74 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -306,6 +306,12 @@ documentdb[].allocation.amortizecount int default=10000 ## used in multi-value attribute vectors to store underlying values. documentdb[].allocation.multivaluegrowfactor double default=0.2 +## The ratio of used bytes that can be dead before attempting to perform compaction. +documentdb[].allocation.max_dead_bytes_ratio double default=0.2 + +## The ratio of used address space that can be dead before attempting to perform compaction. +documentdb[].allocation.max_dead_address_space_ratio double default=0.2 + ## The interval of when periodic tasks should be run periodic.interval double default=3600.0 diff --git a/searchcore/src/vespa/searchcore/proton/attribute/address_space_usage_stats.cpp b/searchcore/src/vespa/searchcore/proton/attribute/address_space_usage_stats.cpp index 3390447e26a..1acfa64285c 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/address_space_usage_stats.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/address_space_usage_stats.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "address_space_usage_stats.h" +#include <ostream> namespace proton { @@ -25,4 +26,13 @@ AddressSpaceUsageStats::merge(const vespalib::AddressSpace &usage, } } +std::ostream& +operator<<(std::ostream& out, const AddressSpaceUsageStats& rhs) +{ + out << "{usage=" << rhs.getUsage() << + ", attribute_name=" << rhs.getAttributeName() << + ", subdb_name=" << rhs.getSubDbName() << "}"; + return out; +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/address_space_usage_stats.h b/searchcore/src/vespa/searchcore/proton/attribute/address_space_usage_stats.h index 400c7cde03f..9ed68693ec1 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/address_space_usage_stats.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/address_space_usage_stats.h @@ -7,8 +7,8 @@ namespace proton { -/* - * class representing usage of a single address space (enum store or +/** + * Class representing usage of a single address space (enum store or * multi value) and the most largest attribute in that respect, relative * to the limit. */ @@ -28,6 +28,14 @@ public: const vespalib::AddressSpace &getUsage() const { return _usage; } const vespalib::string &getAttributeName() const { return _attributeName; } const vespalib::string &getSubDbName() const { return _subDbName; } + + bool operator==(const AddressSpaceUsageStats& rhs) const { + return (_usage == rhs._usage) && + (_attributeName == rhs._attributeName) && + (_subDbName == rhs._subDbName); + } }; +std::ostream& operator<<(std::ostream &out, const AddressSpaceUsageStats& rhs); + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp index 23701bfdd5d..4189688ea81 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.cpp @@ -9,11 +9,9 @@ using search::GrowStrategy; namespace proton { AttributeCollectionSpecFactory::AttributeCollectionSpecFactory( - const search::GrowStrategy &growStrategy, - size_t growNumDocs, + const AllocStrategy &alloc_strategy, bool fastAccessOnly) - : _growStrategy(growStrategy), - _growNumDocs(growNumDocs), + : _alloc_strategy(alloc_strategy), _fastAccessOnly(fastAccessOnly) { } @@ -25,8 +23,8 @@ AttributeCollectionSpecFactory::create(const AttributesConfig &attrCfg, { AttributeCollectionSpec::AttributeList attrs; // Amortize memory spike cost over N docs - const size_t skew = _growNumDocs/(attrCfg.attribute.size()+1); - GrowStrategy grow = _growStrategy; + const size_t skew = _alloc_strategy.get_amortize_count()/(attrCfg.attribute.size()+1); + GrowStrategy grow = _alloc_strategy.get_grow_strategy(); grow.setDocsInitialCapacity(std::max(grow.getDocsInitialCapacity(),docIdLimit)); for (const auto &attr : attrCfg.attribute) { search::attribute::Config cfg = ConfigConverter::convert(attr); @@ -35,6 +33,7 @@ AttributeCollectionSpecFactory::create(const AttributesConfig &attrCfg, } grow.setDocsGrowDelta(grow.getDocsGrowDelta() + skew); cfg.setGrowStrategy(grow); + cfg.setCompactionStrategy(_alloc_strategy.get_compaction_strategy()); attrs.push_back(AttributeSpec(attr.name, cfg)); } return std::make_unique<AttributeCollectionSpec>(attrs, docIdLimit, serialNum); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h index bfa6681d6f2..074c56448f3 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h @@ -4,7 +4,7 @@ #include "attribute_collection_spec.h" #include <vespa/searchcommon/attribute/config.h> -#include <vespa/searchcommon/common/growstrategy.h> +#include <vespa/searchcore/proton/common/alloc_strategy.h> #include <vespa/searchlib/common/serialnum.h> #include <vespa/config-attributes.h> @@ -19,13 +19,11 @@ class AttributeCollectionSpecFactory private: typedef vespa::config::search::AttributesConfig AttributesConfig; - const search::GrowStrategy _growStrategy; - const size_t _growNumDocs; + const AllocStrategy _alloc_strategy; const bool _fastAccessOnly; public: - AttributeCollectionSpecFactory(const search::GrowStrategy &growStrategy, - size_t growNumDocs, + AttributeCollectionSpecFactory(const AllocStrategy& alloc_strategy, bool fastAccessOnly); AttributeCollectionSpec::UP create(const AttributesConfig &attrCfg, diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp index 368e4ce12f2..372bfe5d631 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp @@ -199,7 +199,7 @@ AttributeInitializer::loadAttribute(const AttributeVectorSP &attr, return false; } else { attr->commit(serialNum, serialNum); - EventLogger::loadAttributeComplete(_documentSubDbName, attr->getName(), vespalib::count_ms(timer.elapsed())); + EventLogger::loadAttributeComplete(_documentSubDbName, attr->getName(), timer.elapsed()); } return true; } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp index 9198cfdafab..109a4bb2192 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp @@ -7,6 +7,7 @@ #include <future> using search::AttributeVector; +using search::CompactionStrategy; using search::GrowStrategy; using search::SerialNum; using vespa::config::search::AttributesConfig; @@ -135,7 +136,7 @@ AttributeCollectionSpec::UP AttributeManagerInitializer::createAttributeSpec() const { uint32_t docIdLimit = 1; // The real docIdLimit is used after attributes are loaded to pad them - AttributeCollectionSpecFactory factory(_attributeGrow, _attributeGrowNumDocs, _fastAccessAttributesOnly); + AttributeCollectionSpecFactory factory(_alloc_strategy, _fastAccessAttributesOnly); return factory.create(_attrCfg, docIdLimit, _configSerialNum); } @@ -144,8 +145,7 @@ AttributeManagerInitializer::AttributeManagerInitializer(SerialNum configSerialN DocumentMetaStore::SP documentMetaStore, AttributeManager::SP baseAttrMgr, const AttributesConfig &attrCfg, - const GrowStrategy &attributeGrow, - size_t attributeGrowNumDocs, + const AllocStrategy& alloc_strategy, bool fastAccessAttributesOnly, searchcorespi::index::IThreadService &master, std::shared_ptr<AttributeManager::SP> attrMgrResult) @@ -153,8 +153,7 @@ AttributeManagerInitializer::AttributeManagerInitializer(SerialNum configSerialN _documentMetaStore(documentMetaStore), _attrMgr(), _attrCfg(attrCfg), - _attributeGrow(attributeGrow), - _attributeGrowNumDocs(attributeGrowNumDocs), + _alloc_strategy(alloc_strategy), _fastAccessAttributesOnly(fastAccessAttributesOnly), _master(master), _attributesResult(), diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h index d56f4c25ace..f74a0f1519d 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.h @@ -4,9 +4,9 @@ #include "attributemanager.h" #include "initialized_attributes_result.h" -#include <vespa/searchcommon/common/growstrategy.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> #include <vespa/searchcore/proton/initializer/initializer_task.h> +#include <vespa/searchcore/proton/common/alloc_strategy.h> #include <vespa/searchlib/common/serialnum.h> #include <vespa/config-attributes.h> @@ -24,8 +24,7 @@ private: DocumentMetaStore::SP _documentMetaStore; AttributeManager::SP _attrMgr; vespa::config::search::AttributesConfig _attrCfg; - search::GrowStrategy _attributeGrow; - size_t _attributeGrowNumDocs; + AllocStrategy _alloc_strategy; bool _fastAccessAttributesOnly; searchcorespi::index::IThreadService &_master; InitializedAttributesResult _attributesResult; @@ -39,8 +38,7 @@ public: DocumentMetaStore::SP documentMetaStore, AttributeManager::SP baseAttrMgr, const vespa::config::search::AttributesConfig &attrCfg, - const search::GrowStrategy &attributeGrow, - size_t attributeGrowNumDocs, + const AllocStrategy& alloc_strategy, bool fastAccessAttributesOnly, searchcorespi::index::IThreadService &master, std::shared_ptr<AttributeManager::SP> attrMgrResult); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.cpp index 2852497cf3f..e19e7976227 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "attribute_usage_filter.h" +#include "i_attribute_usage_listener.h" #include <sstream> namespace proton { @@ -85,7 +86,8 @@ AttributeUsageFilter::AttributeUsageFilter() _attributeStats(), _config(), _state(), - _acceptWrite(true) + _acceptWrite(true), + _listener() { } @@ -97,6 +99,9 @@ AttributeUsageFilter::setAttributeStats(AttributeUsageStats attributeStats_in) Guard guard(_lock); _attributeStats = attributeStats_in; recalcState(guard); + if (_listener) { + _listener->notify_attribute_usage(_attributeStats); + } } AttributeUsageStats @@ -114,6 +119,13 @@ AttributeUsageFilter::setConfig(Config config_in) recalcState(guard); } +void +AttributeUsageFilter::set_listener(std::unique_ptr<IAttributeUsageListener> listener) +{ + Guard guard(_lock); + _listener = std::move(listener); +} + double AttributeUsageFilter::getEnumStoreUsedRatio() const { diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.h index ecc95d4ee02..cb9687e31a1 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_filter.h @@ -10,7 +10,9 @@ namespace proton { -/* +class IAttributeUsageListener; + +/** * Class to filter write operations based on sampled information about * attribute resource usage (e.g. enum store and multivalue mapping). * If resource limit is reached then further writes are denied in @@ -29,6 +31,7 @@ private: Config _config; State _state; std::atomic<bool> _acceptWrite; + std::unique_ptr<IAttributeUsageListener> _listener; void recalcState(const Guard &guard); // called with _lock held public: @@ -37,6 +40,7 @@ public: void setAttributeStats(AttributeUsageStats attributeStats_in); AttributeUsageStats getAttributeUsageStats() const; void setConfig(Config config); + void set_listener(std::unique_ptr<IAttributeUsageListener> listener); double getEnumStoreUsedRatio() const; double getMultiValueUsedRatio() const; bool acceptWriteOperation() const override; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp index 4d59c9081e0..f3da5486d3e 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "attribute_usage_stats.h" +#include <iostream> namespace proton { @@ -19,5 +20,12 @@ AttributeUsageStats::merge(const search::AddressSpaceUsage &usage, _multiValueUsage.merge(usage.multiValueUsage(), attributeName, subDbName); } +std::ostream& +operator<<(std::ostream& out, const AttributeUsageStats& rhs) +{ + out << "{enum_store=" << rhs.enumStoreUsage() << + ", multi_value=" << rhs.multiValueUsage() << "}"; + return out; +} } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.h index f1805809e80..1eb6a9cc6be 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_stats.h @@ -7,7 +7,7 @@ namespace proton { -/* +/** * Class representing aggregated attribute usage, with info about * the most bloated attributes with regards to enum store and * multivalue mapping. @@ -23,10 +23,15 @@ public: const vespalib::string &attributeName, const vespalib::string &subDbName); - const AddressSpaceUsageStats & - enumStoreUsage() const { return _enumStoreUsage; } - const AddressSpaceUsageStats & - multiValueUsage() const { return _multiValueUsage; } + const AddressSpaceUsageStats& enumStoreUsage() const { return _enumStoreUsage; } + const AddressSpaceUsageStats& multiValueUsage() const { return _multiValueUsage; } + + bool operator==(const AttributeUsageStats& rhs) const { + return (_enumStoreUsage == rhs._enumStoreUsage) && + (_multiValueUsage == rhs._multiValueUsage); + } }; +std::ostream& operator<<(std::ostream& out, const AttributeUsageStats& rhs); + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp index cc1af923fc0..f635ee34a04 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp @@ -177,10 +177,14 @@ AttributeManager::transferExistingAttributes(const AttributeManager &currMgr, auto shrinker = wrap->getShrinker(); assert(shrinker); addAttribute(AttributeWrap::normalAttribute(av), shrinker); + auto id = _attributeFieldWriter.getExecutorIdFromName(av->getNamePrefix()); + auto cfg = aspec.getConfig(); + _attributeFieldWriter.execute(id, [av, cfg]() { av->update_config(cfg); }); } else { toBeAdded.push_back(aspec); } } + _attributeFieldWriter.sync(); } void @@ -328,7 +332,7 @@ AttributeManager::flushAll(SerialNum currentSerial) for (const auto &ft : flushTargets) { vespalib::Executor::Task::UP task; task = ft->initFlush(currentSerial, std::make_shared<search::FlushToken>()); - if (task.get() != NULL) { + if (task) { task->run(); } } diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/bucketdb/CMakeLists.txt index 6619f9e419d..66a58ad66a3 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/CMakeLists.txt @@ -6,6 +6,7 @@ vespa_add_library(searchcore_bucketdb STATIC bucket_db_owner.cpp bucketdb.cpp bucketdbhandler.cpp + bucketscaniterator.cpp bucketsessionbase.cpp bucketstate.cpp checksumaggregators.cpp diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h index 05388931e20..e6848d095df 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h @@ -31,7 +31,7 @@ private: void checkEmpty() const; public: BucketDB(); - virtual ~BucketDB(); + ~BucketDB(); const BucketState & add(const GlobalId &gid, const BucketId &bucketId, const Timestamp ×tamp, uint32_t docSize, diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketscaniterator.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketscaniterator.cpp new file mode 100644 index 00000000000..46cbb4fc37f --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketscaniterator.cpp @@ -0,0 +1,29 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "bucketscaniterator.h" + +using document::BucketId; +using storage::spi::BucketInfo; + +namespace proton::bucketdb { + +ScanIterator::ScanIterator(BucketDBOwner::Guard db, Pass pass, BucketId lastBucket, BucketId endBucket) + : _db(std::move(db)), + _itr(lastBucket.isSet() ? _db->upperBound(lastBucket) : _db->begin()), + _end(pass == Pass::SECOND && endBucket.isSet() ? + _db->upperBound(endBucket) : _db->end()) +{ } + +ScanIterator::ScanIterator(BucketDBOwner::Guard db, BucketId bucket) + : _db(std::move(db)), + _itr(_db->lowerBound(bucket)), + _end(_db->end()) +{ } + +ScanIterator::ScanIterator(ScanIterator &&rhs) + : _db(std::move(rhs._db)), + _itr(rhs._itr), + _end(rhs._end) +{ } + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketscaniterator.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketscaniterator.h new file mode 100644 index 00000000000..a437230ed0f --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketscaniterator.h @@ -0,0 +1,49 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "bucket_db_owner.h" +#include "bucketdb.h" + +namespace proton::bucketdb { + +struct ScanPosition { + document::BucketId _lastBucket; + + ScanPosition() : _lastBucket() { } + bool validBucket() const { return _lastBucket.isSet(); } +}; + + +class ScanIterator { +private: + using BucketId = document::BucketId; + using BucketIterator = BucketDB::ConstMapIterator; + BucketDBOwner::Guard _db; + BucketIterator _itr; + BucketIterator _end; + +public: + enum class Pass {FIRST, SECOND}; + ScanIterator(BucketDBOwner::Guard db, Pass pass, BucketId lastBucket, BucketId endBucket); + + ScanIterator(BucketDBOwner::Guard db, BucketId bucket); + + ScanIterator(const ScanIterator &) = delete; + ScanIterator(ScanIterator &&rhs); + ScanIterator &operator=(const ScanIterator &) = delete; + ScanIterator &operator=(ScanIterator &&rhs) = delete; + + bool valid() const { return _itr != _end; } + bool isActive() const { return _itr->second.isActive(); } + BucketId getBucket() const { return _itr->first; } + bool hasReadyBucketDocs() const { return _itr->second.getReadyCount() != 0; } + bool hasNotReadyBucketDocs() const { return _itr->second.getNotReadyCount() != 0; } + + ScanIterator & operator++() { + ++_itr; + return *this; + } +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt index d59a4075dce..a91c35f0485 100644 --- a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt @@ -1,6 +1,8 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchcore_pcommon STATIC SOURCES + alloc_config.cpp + alloc_strategy.cpp attribute_updater.cpp attributefieldvaluenode.cpp cachedselect.cpp diff --git a/searchcore/src/vespa/searchcore/proton/common/alloc_config.cpp b/searchcore/src/vespa/searchcore/proton/common/alloc_config.cpp new file mode 100644 index 00000000000..1611d00fb0f --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/common/alloc_config.cpp @@ -0,0 +1,56 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "alloc_config.h" +#include <vespa/searchcore/proton/common/subdbtype.h> +#include <algorithm> + +using search::CompactionStrategy; +using search::GrowStrategy; + +namespace proton { + +AllocConfig::AllocConfig(const AllocStrategy& alloc_strategy, + uint32_t redundancy, uint32_t searchable_copies) + : _alloc_strategy(alloc_strategy), + _redundancy(redundancy), + _searchable_copies(searchable_copies) +{ +} + +AllocConfig::AllocConfig() + : AllocConfig(AllocStrategy(), 1, 1) +{ +} + +AllocConfig::~AllocConfig() = default; + +bool +AllocConfig::operator==(const AllocConfig &rhs) const noexcept +{ + return ((_alloc_strategy == rhs._alloc_strategy) && + (_redundancy == rhs._redundancy) && + (_searchable_copies == rhs._searchable_copies)); +} + +AllocStrategy +AllocConfig::make_alloc_strategy(SubDbType sub_db_type) const +{ + auto &baseline_grow_strategy = _alloc_strategy.get_grow_strategy(); + size_t initial_capacity = baseline_grow_strategy.getDocsInitialCapacity(); + switch (sub_db_type) { + case SubDbType::READY: + initial_capacity *= _searchable_copies; + break; + case SubDbType::NOTREADY: + initial_capacity *= (_redundancy - _searchable_copies); + break; + case SubDbType::REMOVED: + default: + initial_capacity = std::max(1024ul, initial_capacity / 100); + break; + } + GrowStrategy grow_strategy(initial_capacity, baseline_grow_strategy.getDocsGrowFactor(), baseline_grow_strategy.getDocsGrowDelta(), baseline_grow_strategy.getMultiValueAllocGrowFactor()); + return AllocStrategy(grow_strategy, _alloc_strategy.get_compaction_strategy(), _alloc_strategy.get_amortize_count()); +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/common/alloc_config.h b/searchcore/src/vespa/searchcore/proton/common/alloc_config.h new file mode 100644 index 00000000000..25b953b9871 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/common/alloc_config.h @@ -0,0 +1,34 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "alloc_strategy.h" + +namespace proton { + +enum class SubDbType; + +/* + * Class representing allocation config for proton which can be used + * to make an allocation strategy for large data structures owned by a + * document sub db (e.g. attribute vectors, document meta store). + */ +class AllocConfig +{ + AllocStrategy _alloc_strategy; // baseline before adjusting for redundancy / searchable copies + const uint32_t _redundancy; + const uint32_t _searchable_copies; + +public: + AllocConfig(const AllocStrategy& alloc_strategy, uint32_t redundancy, uint32_t searchable_copies); + AllocConfig(); + ~AllocConfig(); + + bool operator==(const AllocConfig &rhs) const noexcept; + bool operator!=(const AllocConfig &rhs) const noexcept { + return !operator==(rhs); + } + AllocStrategy make_alloc_strategy(SubDbType sub_db_type) const; +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/common/alloc_strategy.cpp b/searchcore/src/vespa/searchcore/proton/common/alloc_strategy.cpp new file mode 100644 index 00000000000..3af72757ccb --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/common/alloc_strategy.cpp @@ -0,0 +1,41 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "alloc_strategy.h" +#include <iostream> + +using search::CompactionStrategy; +using search::GrowStrategy; + +namespace proton { + +AllocStrategy::AllocStrategy(const GrowStrategy& grow_strategy, + const CompactionStrategy& compaction_strategy, + uint32_t amortize_count) + : _grow_strategy(grow_strategy), + _compaction_strategy(compaction_strategy), + _amortize_count(amortize_count) +{ +} + +AllocStrategy::AllocStrategy() + : AllocStrategy(GrowStrategy(), CompactionStrategy(), 10000) +{ +} + +AllocStrategy::~AllocStrategy() = default; + +bool +AllocStrategy::operator==(const AllocStrategy &rhs) const noexcept +{ + return ((_grow_strategy == rhs._grow_strategy) && + (_compaction_strategy == rhs._compaction_strategy) && + (_amortize_count == rhs._amortize_count)); +} + +std::ostream& operator<<(std::ostream& os, const AllocStrategy&alloc_strategy) +{ + os << "{ grow_strategy=" << alloc_strategy.get_grow_strategy() << ", compaction_strategy=" << alloc_strategy.get_compaction_strategy() << ", amortize_count=" << alloc_strategy.get_amortize_count() << "}"; + return os; +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/common/alloc_strategy.h b/searchcore/src/vespa/searchcore/proton/common/alloc_strategy.h new file mode 100644 index 00000000000..74bcc1772ee --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/common/alloc_strategy.h @@ -0,0 +1,41 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchcommon/common/compaction_strategy.h> +#include <vespa/searchcommon/common/growstrategy.h> +#include <iosfwd> + +namespace proton { + +/* + * Class representing allocation strategy for large data structures + * owned by a document sub db (e.g. attribute vectors, document meta store). + */ +class AllocStrategy +{ +protected: + const search::GrowStrategy _grow_strategy; + const search::CompactionStrategy _compaction_strategy; + const uint32_t _amortize_count; + +public: + AllocStrategy(const search::GrowStrategy& grow_strategy, + const search::CompactionStrategy& compaction_strategy, + uint32_t amortize_count); + + AllocStrategy(); + ~AllocStrategy(); + + bool operator==(const AllocStrategy &rhs) const noexcept; + bool operator!=(const AllocStrategy &rhs) const noexcept { + return !operator==(rhs); + } + const search::GrowStrategy& get_grow_strategy() const noexcept { return _grow_strategy; } + const search::CompactionStrategy& get_compaction_strategy() const noexcept { return _compaction_strategy; } + uint32_t get_amortize_count() const noexcept { return _amortize_count; } +}; + +std::ostream& operator<<(std::ostream& os, const AllocStrategy&alloc_strategy); + +} diff --git a/searchcore/src/vespa/searchcore/proton/common/eventlogger.cpp b/searchcore/src/vespa/searchcore/proton/common/eventlogger.cpp index 5966589d635..20b87f4bc63 100644 --- a/searchcore/src/vespa/searchcore/proton/common/eventlogger.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/eventlogger.cpp @@ -10,6 +10,7 @@ LOG_SETUP(".proton.common.eventlogger"); using search::util::LogUtil; using vespalib::JSONStringer; using vespalib::make_string; +using vespalib::count_ms; namespace { @@ -32,12 +33,12 @@ doTransactionLogReplayStart(const string &domainName, SerialNum first, SerialNum } void -doTransactionLogReplayComplete(const string &domainName, int64_t elapsedTimeMs, const string &eventName) +doTransactionLogReplayComplete(const string &domainName, vespalib::duration elapsedTime, const string &eventName) { JSONStringer jstr; jstr.beginObject(); jstr.appendKey("domain").appendString(domainName); - jstr.appendKey("time.elapsed.ms").appendInt64(elapsedTimeMs); + jstr.appendKey("time.elapsed.ms").appendInt64(count_ms(elapsedTime)); jstr.endObject(); EV_STATE(eventName.c_str(), jstr.toString().data()); } @@ -71,9 +72,9 @@ EventLogger::transactionLogReplayProgress(const string &domainName, float progre } void -EventLogger::transactionLogReplayComplete(const string &domainName, int64_t elapsedTimeMs) +EventLogger::transactionLogReplayComplete(const string &domainName, vespalib::duration elapsedTime) { - doTransactionLogReplayComplete(domainName, elapsedTimeMs, "transactionlog.replay.complete"); + doTransactionLogReplayComplete(domainName, elapsedTime, "transactionlog.replay.complete"); } void @@ -109,13 +110,13 @@ EventLogger::flushStart(const string &name, int64_t beforeMemory, int64_t afterM } void -EventLogger::flushComplete(const string &name, int64_t elapsedTimeMs, SerialNum flushed, +EventLogger::flushComplete(const string &name, vespalib::duration elapsedTime, SerialNum flushed, const string &outputPath, size_t outputPathElems) { JSONStringer jstr; jstr.beginObject(); jstr.appendKey("name").appendString(name); - jstr.appendKey("time.elapsed.ms").appendInt64(elapsedTimeMs); + jstr.appendKey("time.elapsed.ms").appendInt64(count_ms(elapsedTime)); jstr.appendKey("serialnum") .beginObject() .appendKey("flushed").appendInt64(flushed) @@ -237,13 +238,13 @@ EventLogger::reprocessDocumentsProgress(const string &subDb, double progress, do void -EventLogger::reprocessDocumentsComplete(const string &subDb, double visitCost, int64_t elapsedTimeMs) +EventLogger::reprocessDocumentsComplete(const string &subDb, double visitCost, vespalib::duration elapsedTime) { JSONStringer jstr; jstr.beginObject(); jstr.appendKey("documentsubdb").appendString(subDb); jstr.appendKey("visitcost").appendDouble(visitCost); - jstr.appendKey("time.elapsed.ms").appendInt64(elapsedTimeMs); + jstr.appendKey("time.elapsed.ms").appendInt64(count_ms(elapsedTime)); jstr.endObject(); EV_STATE("reprocess.documents.complete", jstr.toString().data()); } @@ -261,13 +262,13 @@ EventLogger::loadAttributeStart(const vespalib::string &subDbName, const vespali void EventLogger::loadAttributeComplete(const vespalib::string &subDbName, - const vespalib::string &attrName, int64_t elapsedTimeMs) + const vespalib::string &attrName, vespalib::duration elapsedTime) { JSONStringer jstr; jstr.beginObject(); jstr.appendKey("documentsubdb").appendString(subDbName); jstr.appendKey("name").appendString(attrName); - jstr.appendKey("time.elapsed.ms").appendInt64(elapsedTimeMs); + jstr.appendKey("time.elapsed.ms").appendInt64(count_ms(elapsedTime)); jstr.endObject(); EV_STATE("load.attribute.complete", jstr.toString().data()); } @@ -285,12 +286,12 @@ loadComponentStart(const vespalib::string &subDbName, const vespalib::string &co } void -loadComponentComplete(const vespalib::string &subDbName, const vespalib::string &componentName, int64_t elapsedTimeMs) +loadComponentComplete(const vespalib::string &subDbName, const vespalib::string &componentName, vespalib::duration elapsedTime) { JSONStringer jstr; jstr.beginObject(); jstr.appendKey("documentsubdb").appendString(subDbName); - jstr.appendKey("time.elapsed.ms").appendInt64(elapsedTimeMs); + jstr.appendKey("time.elapsed.ms").appendInt64(count_ms(elapsedTime)); jstr.endObject(); EV_STATE(make_string("load.%s.complete", componentName.c_str()).c_str(), jstr.toString().data()); } @@ -304,9 +305,9 @@ EventLogger::loadDocumentMetaStoreStart(const vespalib::string &subDbName) } void -EventLogger::loadDocumentMetaStoreComplete(const vespalib::string &subDbName, int64_t elapsedTimeMs) +EventLogger::loadDocumentMetaStoreComplete(const vespalib::string &subDbName, vespalib::duration elapsedTime) { - loadComponentComplete(subDbName, "documentmetastore", elapsedTimeMs); + loadComponentComplete(subDbName, "documentmetastore", elapsedTime); } void @@ -316,9 +317,9 @@ EventLogger::loadDocumentStoreStart(const vespalib::string &subDbName) } void -EventLogger::loadDocumentStoreComplete(const vespalib::string &subDbName, int64_t elapsedTimeMs) +EventLogger::loadDocumentStoreComplete(const vespalib::string &subDbName, vespalib::duration elapsedTime) { - loadComponentComplete(subDbName, "documentstore", elapsedTimeMs); + loadComponentComplete(subDbName, "documentstore", elapsedTime); } void diff --git a/searchcore/src/vespa/searchcore/proton/common/eventlogger.h b/searchcore/src/vespa/searchcore/proton/common/eventlogger.h index 574e650732a..7d32c777f98 100644 --- a/searchcore/src/vespa/searchcore/proton/common/eventlogger.h +++ b/searchcore/src/vespa/searchcore/proton/common/eventlogger.h @@ -3,6 +3,7 @@ #include <vespa/searchlib/common/serialnum.h> #include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/time.h> #include <vector> namespace proton { @@ -15,7 +16,7 @@ private: typedef search::SerialNum SerialNum; typedef vespalib::string string; public: - static void transactionLogReplayComplete(const string &domainName, int64_t elapsedTimeMs); + static void transactionLogReplayComplete(const string &domainName, vespalib::duration elapsedTime); static void populateAttributeStart(const std::vector<string> &names); static void populateAttributeComplete(const std::vector<string> &names, int64_t documentsVisisted); static void populateDocumentFieldStart(const string &fieldName); @@ -23,7 +24,7 @@ public: static void lidSpaceCompactionComplete(const string &subDbName, uint32_t lidLimit); static void reprocessDocumentsStart(const string &subDb, double visitCost); static void reprocessDocumentsProgress(const string &subDb, double progress, double visitCost); - static void reprocessDocumentsComplete(const string &subDb, double visitCost, int64_t elapsedTimeMs); + static void reprocessDocumentsComplete(const string &subDb, double visitCost, vespalib::duration elapsedTime); static void transactionLogReplayStart(const string &domainName, SerialNum first, SerialNum last); @@ -40,18 +41,18 @@ public: SerialNum unflushed, SerialNum current); static void flushComplete(const string &name, - int64_t elapsedTimeMs, + vespalib::duration elapsedTime, SerialNum flushed, const string &outputPath, size_t outputPathElems); static void flushPrune(const string &name, SerialNum oldestFlushed); static void loadAttributeStart(const vespalib::string &subDbName, const vespalib::string &attrName); static void loadAttributeComplete(const vespalib::string &subDbName, - const vespalib::string &attrName, int64_t elapsedTimeMs); + const vespalib::string &attrName, vespalib::duration elapsedTime); static void loadDocumentMetaStoreStart(const vespalib::string &subDbName); - static void loadDocumentMetaStoreComplete(const vespalib::string &subDbName, int64_t elapsedTimeMs); + static void loadDocumentMetaStoreComplete(const vespalib::string &subDbName, vespalib::duration elapsedTime); static void loadDocumentStoreStart(const vespalib::string &subDbName); - static void loadDocumentStoreComplete(const vespalib::string &subDbName, int64_t elapsedTimeMs); + static void loadDocumentStoreComplete(const vespalib::string &subDbName, vespalib::duration elapsedTime); static void transactionLogPruneComplete(const string &domainName, SerialNum prunedSerial); }; diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.cpp index 29346903aef..85c92f0da6b 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanagerinitializer.cpp @@ -43,7 +43,7 @@ SummaryManagerInitializer::run() *_result = std::make_shared<SummaryManager> (_summaryExecutor, _storeCfg, _grow, _baseDir, _docTypeName, _tuneFile, _fileHeaderContext, _tlSyncer, _bucketizer); - EventLogger::loadDocumentStoreComplete(_subDbName, vespalib::count_ms(timer.elapsed())); + EventLogger::loadDocumentStoreComplete(_subDbName, timer.elapsed()); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp index b563cee1d0c..5e6a9f354f9 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp @@ -39,9 +39,7 @@ private: bool saveDocumentMetaStore(); // not updating snap info. public: Flusher(DocumentMetaStoreFlushTarget &dmsft, uint64_t syncToken, AttributeDirectory::Writer &writer); - ~Flusher(); - uint64_t getSyncToken() const { return _syncToken; } - bool saveSnapInfo(); + ~Flusher() override; bool flush(AttributeDirectory::Writer &writer); void updateStats(); bool cleanUp(AttributeDirectory::Writer &writer); @@ -66,29 +64,23 @@ Flusher(DocumentMetaStoreFlushTarget &dmsft, assert(_saver); } -DocumentMetaStoreFlushTarget::Flusher::~Flusher() -{ - // empty -} +DocumentMetaStoreFlushTarget::Flusher::~Flusher() = default; bool DocumentMetaStoreFlushTarget::Flusher::saveDocumentMetaStore() { vespalib::mkdir(_flushDir, false); - SerialNumFileHeaderContext fileHeaderContext(_dmsft._fileHeaderContext, - _syncToken); + SerialNumFileHeaderContext fileHeaderContext(_dmsft._fileHeaderContext, _syncToken); bool saveSuccess = false; if (_dmsft._hwInfo.disk().slow()) { search::AttributeMemorySaveTarget memorySaveTarget; saveSuccess = _saver->save(memorySaveTarget); _saver.reset(); if (saveSuccess) { - saveSuccess = memorySaveTarget.writeToFile(_dmsft._tuneFileAttributes, - fileHeaderContext); + saveSuccess = memorySaveTarget.writeToFile(_dmsft._tuneFileAttributes, fileHeaderContext); } } else { - search::AttributeFileSaveTarget saveTarget(_dmsft._tuneFileAttributes, - fileHeaderContext); + search::AttributeFileSaveTarget saveTarget(_dmsft._tuneFileAttributes, fileHeaderContext); saveSuccess = _saver->save(saveTarget); _saver.reset(); } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreinitializer.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreinitializer.cpp index a1a8297d5c0..2bf6a1c8432 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreinitializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreinitializer.cpp @@ -51,7 +51,7 @@ DocumentMetaStoreInitializer::run() } else { _dms->commit(snap.syncToken, snap.syncToken); } - EventLogger::loadDocumentMetaStoreComplete(_subDbName, vespalib::count_ms(stopWatch.elapsed())); + EventLogger::loadDocumentMetaStoreComplete(_subDbName, stopWatch.elapsed()); } } else { vespalib::mkdir(_baseDir, false); diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp index 94bba1bf64f..9a4054d02ca 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp @@ -83,7 +83,7 @@ DocumentMetaStoreSaver(vespalib::GenerationHandler::Guard &&guard, } -DocumentMetaStoreSaver::~DocumentMetaStoreSaver() { } +DocumentMetaStoreSaver::~DocumentMetaStoreSaver() = default; bool diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.h index 2f745929667..a11f60af4b6 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.h @@ -32,14 +32,14 @@ private: const MetaDataStore &_metaDataStore; bool _writeDocSize; - virtual bool onSave(search::IAttributeSaveTarget &saveTarget) override; + bool onSave(search::IAttributeSaveTarget &saveTarget) override; public: DocumentMetaStoreSaver(vespalib::GenerationHandler::Guard &&guard, const search::attribute::AttributeHeader &header, const GidIterator &gidIterator, const MetaDataStore &metaDataStore); - virtual ~DocumentMetaStoreSaver(); + ~DocumentMetaStoreSaver() override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/cachedflushtarget.h b/searchcore/src/vespa/searchcore/proton/flushengine/cachedflushtarget.h index ecd66f9cbfb..f2b3514fcf9 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/cachedflushtarget.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/cachedflushtarget.h @@ -42,16 +42,18 @@ public: const IFlushTarget::SP & getFlushTarget() { return _target; } // Implements IFlushTarget. - virtual MemoryGain getApproxMemoryGain() const override { return _memoryGain; } - virtual DiskGain getApproxDiskGain() const override { return _diskGain; } - virtual SerialNum getFlushedSerialNum() const override { return _flushedSerialNum; } - virtual Time getLastFlushTime() const override { return _lastFlushTime; } - virtual bool needUrgentFlush() const override { return _needUrgentFlush; } - - virtual Task::UP initFlush(SerialNum currentSerial, std::shared_ptr<search::IFlushToken> flush_token) override { return _target->initFlush(currentSerial, std::move(flush_token)); } - virtual FlushStats getLastFlushStats() const override { return _target->getLastFlushStats(); } - - virtual uint64_t getApproxBytesToWriteToDisk() const override; + MemoryGain getApproxMemoryGain() const override { return _memoryGain; } + DiskGain getApproxDiskGain() const override { return _diskGain; } + SerialNum getFlushedSerialNum() const override { return _flushedSerialNum; } + Time getLastFlushTime() const override { return _lastFlushTime; } + bool needUrgentFlush() const override { return _needUrgentFlush; } + + Task::UP initFlush(SerialNum currentSerial, std::shared_ptr<search::IFlushToken> flush_token) override { + return _target->initFlush(currentSerial, std::move(flush_token)); + } + FlushStats getLastFlushStats() const override { return _target->getLastFlushStats(); } + + uint64_t getApproxBytesToWriteToDisk() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flush_engine_explorer.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flush_engine_explorer.cpp index 2a447957334..1f5ad8ae3a3 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flush_engine_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flush_engine_explorer.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "flush_engine_explorer.h" - +#include "flushengine.h" #include <vespa/vespalib/data/slime/cursor.h> #include <vespa/vespalib/data/slime/inserter.h> diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flush_engine_explorer.h b/searchcore/src/vespa/searchcore/proton/flushengine/flush_engine_explorer.h index a674b39bc26..c2a90597ca2 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flush_engine_explorer.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flush_engine_explorer.h @@ -2,11 +2,12 @@ #pragma once -#include "flushengine.h" #include <vespa/vespalib/net/state_explorer.h> namespace proton { +class FlushEngine; + /** * Class used to explore the state of a flush engine and its flush targets. */ diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.cpp index b569e499876..ed31317cb69 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushcontext.cpp @@ -16,20 +16,17 @@ FlushContext::FlushContext( _target(target), _task(), _lastSerial(lastSerial) -{ - // empty -} +{ } -vespalib::string FlushContext::createName(const IFlushHandler & handler, const IFlushTarget & target) -{ +vespalib::string +FlushContext::createName(const IFlushHandler & handler, const IFlushTarget & target) { return (handler.getName() + "." + target.getName()); } FlushContext::~FlushContext() { - if (_task.get() != NULL) { - LOG(warning, "Unexecuted flush task for '%s' destroyed.", - _name.c_str()); + if (_task) { + LOG(warning, "Unexecuted flush task for '%s' destroyed.", _name.c_str()); } } @@ -38,11 +35,11 @@ FlushContext::initFlush(std::shared_ptr<search::IFlushToken> flush_token) { LOG(debug, "Attempting to flush '%s'.", _name.c_str()); _task = _target->initFlush(std::max(_handler->getCurrentSerialNumber(), _lastSerial), std::move(flush_token)); - if (_task.get() == NULL) { + if ( ! _task ) { LOG(debug, "Target refused to init flush."); return false; } return true; } -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp index ab5b1ac5937..50162936835 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp @@ -77,10 +77,10 @@ FlushEngine::FlushInfo::FlushInfo(uint32_t taskId, const IFlushTarget::SP &targe } FlushEngine::FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory> tlsStatsFactory, - IFlushStrategy::SP strategy, uint32_t numThreads, uint32_t idleIntervalMS) + IFlushStrategy::SP strategy, uint32_t numThreads, vespalib::duration idleInterval) : _closed(false), _maxConcurrent(numThreads), - _idleIntervalMS(idleIntervalMS), + _idleInterval(idleInterval), _taskId(0), _threadPool(128 * 1024), _strategy(std::move(strategy)), @@ -151,13 +151,13 @@ FlushEngine::canFlushMore(const std::unique_lock<std::mutex> &guard) const } bool -FlushEngine::wait(size_t minimumWaitTimeIfReady) +FlushEngine::wait(vespalib::duration minimumWaitTimeIfReady, bool ignorePendingPrune) { std::unique_lock<std::mutex> guard(_lock); - if ( (minimumWaitTimeIfReady > 0) && canFlushMore(guard) && _pendingPrune.empty()) { - _cond.wait_for(guard, std::chrono::milliseconds(minimumWaitTimeIfReady)); + if ( (minimumWaitTimeIfReady != vespalib::duration::zero()) && canFlushMore(guard) && _pendingPrune.empty()) { + _cond.wait_for(guard, minimumWaitTimeIfReady); } - while ( ! canFlushMore(guard) && _pendingPrune.empty()) { + while ( ! canFlushMore(guard) && ( ignorePendingPrune || _pendingPrune.empty())) { _cond.wait_for(guard, 1s); // broadcast when flush done } return !_closed; @@ -168,7 +168,7 @@ FlushEngine::Run(FastOS_ThreadInterface *, void *) { bool shouldIdle = false; vespalib::string prevFlushName; - while (wait(shouldIdle ? _idleIntervalMS : 0)) { + while (wait(shouldIdle ? _idleInterval : vespalib::duration::zero(), false)) { shouldIdle = false; if (prune()) { continue; // Prune attempted on one or more handlers @@ -181,8 +181,8 @@ FlushEngine::Run(FastOS_ThreadInterface *, void *) } else { shouldIdle = true; } - LOG(debug, "Making another wait(idle=%s, timeMS=%d) last was '%s'", - shouldIdle ? "true" : "false", shouldIdle ? _idleIntervalMS : 0, prevFlushName.c_str()); + LOG(debug, "Making another wait(idle=%s, timeS=%1.3f) last was '%s'", + shouldIdle ? "true" : "false", shouldIdle ? vespalib::to_s(_idleInterval) : 0, prevFlushName.c_str()); } _executor.sync(); prune(); @@ -220,7 +220,8 @@ FlushEngine::prune() return true; } -bool FlushEngine::isFlushing(const std::lock_guard<std::mutex> & guard, const vespalib::string & name) const +bool +FlushEngine::isFlushing(const std::lock_guard<std::mutex> & guard, const vespalib::string & name) const { (void) guard; for(const auto & it : _flushing) { @@ -306,7 +307,7 @@ FlushEngine::flushAll(const FlushContext::List &lst) { LOG(debug, "%ld targets to flush.", lst.size()); for (const FlushContext::SP & ctx : lst) { - if (wait(0)) { + if (wait(vespalib::duration::zero(), true)) { if (ctx->initFlush(get_flush_token(*ctx))) { logTarget("initiated", *ctx); _executor.execute(std::make_unique<FlushTask>(initFlush(*ctx), *this, ctx)); @@ -371,7 +372,7 @@ FlushEngine::flushDone(const FlushContext &ctx, uint32_t taskId) } if (LOG_WOULD_LOG(event)) { FlushStats stats = ctx.getTarget()->getLastFlushStats(); - EventLogger::flushComplete(ctx.getName(), vespalib::count_ms(duration), ctx.getTarget()->getFlushedSerialNum(), + EventLogger::flushComplete(ctx.getName(), duration, ctx.getTarget()->getFlushedSerialNum(), stats.getPath(), stats.getPathElementsToLog()); } LOG(debug, "FlushEngine::flushDone(taskId='%d') took '%f' secs", taskId, vespalib::to_s(duration)); diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h index f51e93f0fbd..8aea546bd14 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h @@ -50,7 +50,7 @@ private: typedef HandlerMap<IFlushHandler> FlushHandlerMap; bool _closed; const uint32_t _maxConcurrent; - const uint32_t _idleIntervalMS; + const vespalib::duration _idleInterval; uint32_t _taskId; FastOS_ThreadPool _threadPool; IFlushStrategy::SP _strategy; @@ -79,7 +79,7 @@ private: uint32_t initFlush(const IFlushHandler::SP &handler, const IFlushTarget::SP &target); void flushDone(const FlushContext &ctx, uint32_t taskId); bool canFlushMore(const std::unique_lock<std::mutex> &guard) const; - bool wait(size_t minimumWaitTimeIfReady); + bool wait(vespalib::duration minimumWaitTimeIfReady, bool ignorePendingPrune); bool isFlushing(const std::lock_guard<std::mutex> &guard, const vespalib::string & name) const; friend class FlushTask; @@ -102,7 +102,7 @@ public: * @param idleInterval The interval between when flushes are checked whne there are no one progressing. */ FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory> tlsStatsFactory, - IFlushStrategy::SP strategy, uint32_t numThreads, uint32_t idleIntervalMS); + IFlushStrategy::SP strategy, uint32_t numThreads, vespalib::duration idleInterval); /** * Destructor. Waits for all pending tasks to complete. diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushtargetproxy.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushtargetproxy.h index 84bf91a2f9a..2d0be231adc 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushtargetproxy.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushtargetproxy.h @@ -31,8 +31,7 @@ public: * @param target The target to decorate. * @param prefix The prefix to prepend to the target */ - FlushTargetProxy(const IFlushTarget::SP &target, - const vespalib::string & prefix); + FlushTargetProxy(const IFlushTarget::SP &target, const vespalib::string & prefix); /** * Returns the decorated flush target. This should not be used for anything * but testing, as invoking a method on the returned target beats the @@ -40,34 +39,16 @@ public: * * @return The decorated flush target. */ - const IFlushTarget::SP & - getFlushTarget() const - { - return _target; - } + const IFlushTarget::SP & getFlushTarget() const { return _target; } // Implements IFlushTarget. - virtual MemoryGain - getApproxMemoryGain() const override; - - virtual DiskGain - getApproxDiskGain() const override; - - virtual SerialNum - getFlushedSerialNum() const override; - - virtual Time - getLastFlushTime() const override; - - virtual bool - needUrgentFlush() const override; - - virtual Task::UP - initFlush(SerialNum currentSerial, std::shared_ptr<search::IFlushToken> flush_token) override; - - virtual searchcorespi::FlushStats - getLastFlushStats() const override; - - virtual uint64_t getApproxBytesToWriteToDisk() const override; + MemoryGain getApproxMemoryGain() const override; + DiskGain getApproxDiskGain() const override; + SerialNum getFlushedSerialNum() const override; + Time getLastFlushTime() const override; + bool needUrgentFlush() const override; + Task::UP initFlush(SerialNum currentSerial, std::shared_ptr<search::IFlushToken> flush_token) override; + searchcorespi::FlushStats getLastFlushStats() const override; + uint64_t getApproxBytesToWriteToDisk() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.cpp index 94476d93b15..2b1bd3b423d 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.cpp @@ -1,17 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "flushtask.h" +#include "flushengine.h" namespace proton { FlushTask::FlushTask(uint32_t taskId, FlushEngine &engine, - const FlushContext::SP &ctx) + std::shared_ptr<FlushContext> ctx) : _taskId(taskId), _engine(engine), - _context(ctx) + _context(std::move(ctx)) { - assert(_context.get() != NULL); + assert(_context); } FlushTask::~FlushTask() diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.h index 7860acaab41..fb3474e2a90 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushtask.h @@ -1,10 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/searchcore/proton/flushengine/flushengine.h> +#include <memory> +#include <vespa/vespalib/util/executor.h> namespace proton { +class FlushEngine; +class FlushContext; /** * This class decorates a task returned by initFlush() in IFlushTarget so that * the appropriate callback is invoked on the running FlushEngine. @@ -14,7 +17,7 @@ class FlushTask : public vespalib::Executor::Task private: uint32_t _taskId; FlushEngine &_engine; - FlushContext::SP _context; + std::shared_ptr<FlushContext> _context; public: FlushTask(const FlushTask &) = delete; @@ -26,15 +29,14 @@ public: * @param engine The running flush engine. * @param ctx The context of the flush to perform. */ - FlushTask(uint32_t taskId, FlushEngine &engine, const FlushContext::SP &ctx); + FlushTask(uint32_t taskId, FlushEngine &engine, std::shared_ptr<FlushContext> ctx); /** * Destructor. Notifies the engine that the flush is done to prevent the * engine from locking targets because of a glitch. */ - ~FlushTask(); + ~FlushTask() override; - // Implements Executor::Task. void run() override; }; diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h b/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h index ca23f27ed49..66ca730bfbc 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/iflushhandler.h @@ -2,7 +2,6 @@ #pragma once #include <vespa/searchcorespi/flush/iflushtarget.h> -#include <vector> namespace proton { @@ -35,7 +34,7 @@ public: /** * Virtual destructor required for inheritance. */ - virtual ~IFlushHandler() { } + virtual ~IFlushHandler() = default; /** * Returns the unique name of this handler. diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/iflushstrategy.h b/searchcore/src/vespa/searchcore/proton/flushengine/iflushstrategy.h index 67f6771650a..8c6a31ff270 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/iflushstrategy.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/iflushstrategy.h @@ -14,17 +14,12 @@ namespace flushengine { class TlsStatsMap; } */ class IFlushStrategy { public: - /** - * Convenience typedefs. - */ typedef std::shared_ptr<IFlushStrategy> SP; IFlushStrategy(const IFlushStrategy &) = delete; IFlushStrategy & operator = (const IFlushStrategy &) = delete; - /** - * Virtual destructor required for inheritance. - */ - virtual ~IFlushStrategy() { } + + virtual ~IFlushStrategy() = default; /** * Takes an input of targets that are candidates for flush and returns @@ -34,11 +29,10 @@ public: * @return A prioritized list of targets to flush. */ virtual FlushContext::List getFlushTargets(const FlushContext::List & targetList, - const flushengine::TlsStatsMap & - tlsStatsMap) const = 0; + const flushengine::TlsStatsMap & tlsStatsMap) const = 0; protected: IFlushStrategy() = default; }; -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/threadedflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/threadedflushtarget.cpp index 59a5504b57b..3db341ecb70 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/threadedflushtarget.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/threadedflushtarget.cpp @@ -49,11 +49,9 @@ ThreadedFlushTarget::initFlush(SerialNum currentSerial, std::shared_ptr<search:: { std::promise<Task::UP> promise; std::future<Task::UP> future = promise.get_future(); - _executor.execute(makeLambdaTask([&]() - { promise.set_value(callInitFlush(_target.get(), - currentSerial, - &_getSerialNum, - flush_token)); })); + _executor.execute(makeLambdaTask([&]() { + promise.set_value(callInitFlush(_target.get(), currentSerial, &_getSerialNum, flush_token)); + })); return future.get(); } diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/threadedflushtarget.h b/searchcore/src/vespa/searchcore/proton/flushengine/threadedflushtarget.h index cd2ee11265f..d949ffa9c50 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/threadedflushtarget.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/threadedflushtarget.h @@ -18,7 +18,7 @@ class ThreadedFlushTarget : public FlushTargetProxy { private: using IFlushTarget = searchcorespi::IFlushTarget; - vespalib::Executor &_executor; + vespalib::Executor &_executor; const IGetSerialNum &_getSerialNum; public: @@ -48,9 +48,7 @@ public: const IFlushTarget::SP &target, const vespalib::string & prefix); - // Implements IFlushTarget. - virtual Task::UP - initFlush(SerialNum currentSerial, std::shared_ptr<search::IFlushToken> flush_token) override; + Task::UP initFlush(SerialNum currentSerial, std::shared_ptr<search::IFlushToken> flush_token) override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/reprocessing/reprocess_documents_task.cpp b/searchcore/src/vespa/searchcore/proton/reprocessing/reprocess_documents_task.cpp index d8e0fcc7843..d5cbb4615f4 100644 --- a/searchcore/src/vespa/searchcore/proton/reprocessing/reprocess_documents_task.cpp +++ b/searchcore/src/vespa/searchcore/proton/reprocessing/reprocess_documents_task.cpp @@ -44,8 +44,7 @@ ReprocessDocumentsTask::run() docstore.accept(_handler, *this, *_docTypeRepo); } _handler.done(); - EventLogger::reprocessDocumentsComplete(_subDbName, _visitorCost, - duration_cast<milliseconds>(clock::now() - _start).count()); + EventLogger::reprocessDocumentsComplete(_subDbName, _visitorCost,(clock::now() - _start)); } } @@ -55,8 +54,8 @@ ReprocessDocumentsTask::updateProgress(double progress) _visitorProgress = progress; double deltaProgress = progress - _loggedProgress; if (deltaProgress >= 0.01) { - auto secondsSinceLastLog = duration_cast<seconds>(clock::now() - _lastLogTime); - if (secondsSinceLastLog >= 60s || deltaProgress >= 0.10) { + auto timeSinceLastLog = clock::now() - _lastLogTime; + if (timeSinceLastLog >= 60s || deltaProgress >= 0.10) { EventLogger::reprocessDocumentsProgress(_subDbName, progress, _visitorCost); _lastLogTime = clock::now(); _loggedProgress = progress; diff --git a/searchcore/src/vespa/searchcore/proton/server/blockable_maintenance_job.cpp b/searchcore/src/vespa/searchcore/proton/server/blockable_maintenance_job.cpp index d8b5aa7b129..ce3bd3b8e9b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/blockable_maintenance_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/blockable_maintenance_job.cpp @@ -4,6 +4,7 @@ #include "disk_mem_usage_state.h" #include "imaintenancejobrunner.h" #include "document_db_maintenance_config.h" +#include "move_operation_limiter.h" namespace proton { @@ -49,7 +50,7 @@ BlockableMaintenanceJob::BlockableMaintenanceJob(const vespalib::string &name, BlockableMaintenanceJob::~BlockableMaintenanceJob() { - _moveOpsLimiter->clearJob(); + dynamic_cast<MoveOperationLimiter &>(*_moveOpsLimiter).clearJob(); } bool diff --git a/searchcore/src/vespa/searchcore/proton/server/blockable_maintenance_job.h b/searchcore/src/vespa/searchcore/proton/server/blockable_maintenance_job.h index db7b8d05ca2..f20511bc417 100644 --- a/searchcore/src/vespa/searchcore/proton/server/blockable_maintenance_job.h +++ b/searchcore/src/vespa/searchcore/proton/server/blockable_maintenance_job.h @@ -2,7 +2,7 @@ #pragma once #include "i_blockable_maintenance_job.h" -#include "move_operation_limiter.h" +#include "i_move_operation_limiter.h" #include <mutex> #include <unordered_set> @@ -11,6 +11,7 @@ namespace proton { class BlockableMaintenanceJobConfig; class DiskMemUsageState; class IMaintenanceJobRunner; +struct IMoveOperationLimiter; /** * Implementation of a maintenance job that can be blocked and unblocked due to various external reasons. @@ -27,12 +28,10 @@ private: bool _blocked; IMaintenanceJobRunner *_runner; double _resourceLimitFactor; + std::shared_ptr<IMoveOperationLimiter> _moveOpsLimiter; void updateBlocked(const LockGuard &guard); - protected: - MoveOperationLimiter::SP _moveOpsLimiter; - void internalNotifyDiskMemUsage(const DiskMemUsageState &state); public: @@ -54,6 +53,7 @@ public: void unBlock(BlockedReason reason) override; bool isBlocked() const override; void registerRunner(IMaintenanceJobRunner *runner) override { _runner = runner; } + IMoveOperationLimiter & getLimiter() { return *_moveOpsLimiter; } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp index 8485cb6835e..1c642205c86 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp @@ -6,6 +6,8 @@ #include "iclusterstatechangednotifier.h" #include "maintenancedocumentsubdb.h" #include "i_disk_mem_usage_notifier.h" +#include "ibucketmodifiedhandler.h" +#include "move_operation_limiter.h" #include <vespa/searchcore/proton/bucketdb/i_bucket_create_notifier.h> #include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h> @@ -19,37 +21,10 @@ namespace proton { namespace { -const uint32_t FIRST_SCAN_PASS = 1; -const uint32_t SECOND_SCAN_PASS = 2; - const char * bool2str(bool v) { return (v ? "T" : "F"); } } -BucketMoveJob::ScanIterator:: -ScanIterator(BucketDBOwner::Guard db, uint32_t pass, BucketId lastBucket, BucketId endBucket) - : _db(std::move(db)), - _itr(lastBucket.isSet() ? _db->upperBound(lastBucket) : _db->begin()), - _end(pass == SECOND_SCAN_PASS && endBucket.isSet() ? - _db->upperBound(endBucket) : _db->end()) -{ -} - -BucketMoveJob::ScanIterator:: -ScanIterator(BucketDBOwner::Guard db, BucketId bucket) - : _db(std::move(db)), - _itr(_db->lowerBound(bucket)), - _end(_db->end()) -{ -} - -BucketMoveJob::ScanIterator::ScanIterator(ScanIterator &&rhs) - : _db(std::move(rhs._db)), - _itr(rhs._itr), - _end(rhs._end) -{ -} - void BucketMoveJob::checkBucket(const BucketId &bucket, ScanIterator &itr, @@ -176,17 +151,17 @@ BucketMoveJob(const IBucketStateCalculator::SP &calc, _modifiedHandler(modifiedHandler), _ready(ready), _notReady(notReady), - _mover(*_moveOpsLimiter), + _mover(getLimiter()), _doneScan(false), _scanPos(), - _scanPass(FIRST_SCAN_PASS), + _scanPass(ScanPass::FIRST), _endPos(), _bucketSpace(bucketSpace), _delayedBuckets(), _delayedBucketsFrozen(), _frozenBuckets(frozenBuckets), _bucketCreateNotifier(bucketCreateNotifier), - _delayedMover(*_moveOpsLimiter), + _delayedMover(getLimiter()), _clusterStateChangedNotifier(clusterStateChangedNotifier), _bucketStateChangedNotifier(bucketStateChangedNotifier), _diskMemUsageNotifier(diskMemUsageNotifier) @@ -277,18 +252,14 @@ BucketMoveJob::changedCalculator() _endPos = _scanPos; } _doneScan = false; - _scanPass = FIRST_SCAN_PASS; + _scanPass = ScanPass::FIRST; maybeCancelMover(_mover); maybeCancelMover(_delayedMover); } bool -BucketMoveJob::scanAndMove(size_t maxBucketsToScan, - size_t maxDocsToMove) +BucketMoveJob::scanAndMove(size_t maxBucketsToScan, size_t maxDocsToMove) { - if (done()) { - return true; - } IFrozenBucketHandler::ExclusiveBucketGuard::UP bucketGuard; // Look for delayed bucket to be processed now while (!_delayedBuckets.empty() && _delayedMover.bucketDone()) { @@ -306,14 +277,13 @@ BucketMoveJob::scanAndMove(size_t maxBucketsToScan, size_t bucketsScanned = 0; for (;;) { if (_mover.bucketDone()) { - ScanResult res = scanBuckets(maxBucketsToScan - - bucketsScanned, bucketGuard); + ScanResult res = scanBuckets(maxBucketsToScan - bucketsScanned, bucketGuard); bucketsScanned += res.first; if (res.second) { - if (_scanPass == FIRST_SCAN_PASS && + if (_scanPass == ScanPass::FIRST && _endPos.validBucket()) { _scanPos = ScanPosition(); - _scanPass = SECOND_SCAN_PASS; + _scanPass = ScanPass::SECOND; } else { _doneScan = true; break; @@ -334,7 +304,7 @@ BucketMoveJob::scanAndMove(size_t maxBucketsToScan, bool BucketMoveJob::run() { - if (isBlocked()) { + if (isBlocked() || done()) { return true; // indicate work is done, since node state is bad } /// Returning false here will immediately post the job back on the executor. This will give a busy loop, @@ -363,8 +333,7 @@ BucketMoveJob::notifyClusterStateChanged(const IBucketStateCalculator::SP &newCa } void -BucketMoveJob::notifyBucketStateChanged(const BucketId &bucketId, - BucketInfo::ActiveState newState) +BucketMoveJob::notifyBucketStateChanged(const BucketId &bucketId, BucketInfo::ActiveState newState) { // Called by master write thread if (newState == BucketInfo::NOT_ACTIVE) { diff --git a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h index e462b8ac7c6..8a84a10199c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h +++ b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h @@ -6,22 +6,21 @@ #include "documentbucketmover.h" #include "i_disk_mem_usage_listener.h" #include "ibucketfreezelistener.h" -#include "ibucketmodifiedhandler.h" -#include "ibucketstatecalculator.h" #include "ibucketstatechangedhandler.h" #include "iclusterstatechangedhandler.h" #include "ifrozenbuckethandler.h" -#include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h> +#include <vespa/searchcore/proton/bucketdb/bucketscaniterator.h> #include <vespa/searchcore/proton/bucketdb/i_bucket_create_listener.h> #include <set> -namespace proton -{ +namespace proton { class BlockableMaintenanceJobConfig; class IBucketStateChangedNotifier; class IClusterStateChangedNotifier; class IDiskMemUsageNotifier; +class IBucketModifiedHandler; + namespace bucketdb { class IBucketCreateNotifier; } /** @@ -35,65 +34,24 @@ class BucketMoveJob : public BlockableMaintenanceJob, public IBucketStateChangedHandler, public IDiskMemUsageListener { -public: - struct ScanPosition - { - document::BucketId _lastBucket; - - ScanPosition() : _lastBucket() { } - ScanPosition(document::BucketId lastBucket) : _lastBucket(lastBucket) { } - bool validBucket() const { return _lastBucket.isSet(); } - }; - - typedef BucketDB::ConstMapIterator BucketIterator; - - class ScanIterator - { - private: - BucketDBOwner::Guard _db; - BucketIterator _itr; - BucketIterator _end; - - public: - ScanIterator(BucketDBOwner::Guard db, - uint32_t pass, - document::BucketId lastBucket, - document::BucketId endBucket); - - ScanIterator(BucketDBOwner::Guard db, document::BucketId bucket); - - ScanIterator(const ScanIterator &) = delete; - ScanIterator(ScanIterator &&rhs); - ScanIterator &operator=(const ScanIterator &) = delete; - ScanIterator &operator=(ScanIterator &&rhs) = delete; - - bool valid() const { return _itr != _end; } - bool isActive() const { return _itr->second.isActive(); } - document::BucketId getBucket() const { return _itr->first; } - bool hasReadyBucketDocs() const { return _itr->second.getReadyCount() != 0; } - bool hasNotReadyBucketDocs() const { return _itr->second.getNotReadyCount() != 0; } - - ScanIterator & operator++() { - ++_itr; - return *this; - } - }; - private: - typedef std::pair<size_t, bool> ScanResult; - IBucketStateCalculator::SP _calc; - IDocumentMoveHandler &_moveHandler; - IBucketModifiedHandler &_modifiedHandler; - const MaintenanceDocumentSubDB &_ready; - const MaintenanceDocumentSubDB &_notReady; - DocumentBucketMover _mover; - bool _doneScan; - ScanPosition _scanPos; - uint32_t _scanPass; - ScanPosition _endPos; - document::BucketSpace _bucketSpace; - - typedef std::set<document::BucketId> DelayedBucketSet; + using ScanPosition = bucketdb::ScanPosition; + using ScanIterator = bucketdb::ScanIterator; + using ScanPass = ScanIterator::Pass; + using ScanResult = std::pair<size_t, bool>; + std::shared_ptr<IBucketStateCalculator> _calc; + IDocumentMoveHandler &_moveHandler; + IBucketModifiedHandler &_modifiedHandler; + const MaintenanceDocumentSubDB &_ready; + const MaintenanceDocumentSubDB &_notReady; + DocumentBucketMover _mover; + bool _doneScan; + ScanPosition _scanPos; + ScanPass _scanPass; + ScanPosition _endPos; + document::BucketSpace _bucketSpace; + + using DelayedBucketSet = std::set<document::BucketId>; // Delayed buckets that are no longer frozen or active that can be considered for moving. DelayedBucketSet _delayedBuckets; @@ -185,4 +143,3 @@ public: }; } // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.cpp b/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.cpp index 6951125f408..1f2b0cf1a75 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.cpp @@ -12,7 +12,6 @@ using document::BucketId; using document::Document; using document::GlobalId; -using search::DocumentIdT; using storage::spi::Timestamp; namespace proton { @@ -20,9 +19,7 @@ namespace proton { typedef IDocumentMetaStore::Iterator Iterator; bool -DocumentBucketMover::moveDocument(DocumentIdT lid, - const document::GlobalId &gid, - Timestamp timestamp) +DocumentBucketMover::moveDocument(uint32_t lid, const document::GlobalId &gid, Timestamp timestamp) { if ( _source->lidNeedsCommit(lid) ) { return false; @@ -80,13 +77,11 @@ namespace { class MoveKey { public: - DocumentIdT _lid; + uint32_t _lid; document::GlobalId _gid; Timestamp _timestamp; - MoveKey(DocumentIdT lid, - const document::GlobalId &gid, - Timestamp timestamp) + MoveKey(uint32_t lid, const document::GlobalId &gid, Timestamp timestamp) : _lid(lid), _gid(gid), _timestamp(timestamp) @@ -115,7 +110,7 @@ DocumentBucketMover::moveDocuments(size_t maxDocsToMove) typedef std::vector<MoveKey> MoveVec; MoveVec toMove; for (; itr != end && docsMoved < maxDocsToMove; ++itr) { - DocumentIdT lid = itr.getKey().get_lid(); + uint32_t lid = itr.getKey().get_lid(); const RawDocumentMetaData &metaData = _source->meta_store()->getRawMetaData(lid); if (metaData.getBucketUsedBits() != _bucket.getUsedBits()) { ++docsSkipped; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.h b/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.h index eded3a456d8..80e1811268b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.h @@ -4,7 +4,6 @@ #include <vespa/document/bucket/bucketid.h> #include <vespa/document/base/globalid.h> -#include <vespa/searchlib/query/base.h> #include <persistence/spi/types.h> namespace proton { @@ -32,7 +31,7 @@ private: document::GlobalId _lastGid; bool _lastGidValid; - bool moveDocument(search::DocumentIdT lid, + bool moveDocument(uint32_t lid, const document::GlobalId &gid, storage::spi::Timestamp timestamp); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index cd517fe7c60..166d1e928bb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -6,16 +6,17 @@ #include "document_subdb_collection_explorer.h" #include "documentdb.h" #include "documentdbconfigscout.h" +#include "feedhandler.h" #include "idocumentdbowner.h" #include "idocumentsubdb.h" #include "lid_space_compaction_handler.h" #include "maintenance_jobs_injector.h" #include "reconfig_params.h" -#include "feedhandler.h" -#include <vespa/searchcore/proton/persistenceengine/commit_and_wait_document_retriever.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/metrics/updatehook.h> #include <vespa/searchcore/proton/attribute/attribute_config_inspector.h> #include <vespa/searchcore/proton/attribute/attribute_writer.h> +#include <vespa/searchcore/proton/attribute/i_attribute_usage_listener.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/common/eventlogger.h> #include <vespa/searchcore/proton/common/statusreport.h> @@ -26,6 +27,7 @@ #include <vespa/searchcore/proton/initializer/task_runner.h> #include <vespa/searchcore/proton/metrics/executor_threading_service_stats.h> #include <vespa/searchcore/proton/metrics/metricswireservice.h> +#include <vespa/searchcore/proton/persistenceengine/commit_and_wait_document_retriever.h> #include <vespa/searchcore/proton/reference/document_db_reference_resolver.h> #include <vespa/searchcore/proton/reference/i_document_db_reference_registry.h> #include <vespa/searchlib/attribute/attributefactory.h> @@ -34,7 +36,6 @@ #include <vespa/searchlib/engine/searchreply.h> #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/exceptions.h> -#include <vespa/metrics/updatehook.h> #include <vespa/log/log.h> #include <vespa/searchcorespi/index/warmupconfig.h> @@ -69,39 +70,11 @@ namespace proton { namespace { constexpr uint32_t indexing_thread_stack_size = 128 * 1024; -using Allocation = ProtonConfig::Documentdb::Allocation; -GrowStrategy -makeGrowStrategy(uint32_t docsInitialCapacity, const Allocation &allocCfg) -{ - return GrowStrategy(docsInitialCapacity, allocCfg.growfactor, allocCfg.growbias, allocCfg.multivaluegrowfactor); -} - -DocumentSubDBCollection::Config -makeSubDBConfig(const ProtonConfig::Distribution & distCfg, const Allocation & allocCfg, size_t numSearcherThreads) { - size_t initialNumDocs(allocCfg.initialnumdocs); - GrowStrategy searchableGrowth = makeGrowStrategy(initialNumDocs * distCfg.searchablecopies, allocCfg); - GrowStrategy removedGrowth = makeGrowStrategy(std::max(1024ul, initialNumDocs/100), allocCfg); - GrowStrategy notReadyGrowth = makeGrowStrategy(initialNumDocs * (distCfg.redundancy - distCfg.searchablecopies), allocCfg); - return DocumentSubDBCollection::Config(searchableGrowth, notReadyGrowth, removedGrowth, allocCfg.amortizecount, numSearcherThreads); -} - index::IndexConfig makeIndexConfig(const ProtonConfig::Index & cfg) { return index::IndexConfig(WarmupConfig(vespalib::from_s(cfg.warmup.time), cfg.warmup.unpack), cfg.maxflushed, cfg.cache.size); } -ProtonConfig::Documentdb _G_defaultProtonDocumentDBConfig; - -const ProtonConfig::Documentdb * -findDocumentDB(const ProtonConfig::DocumentdbVector & documentDBs, const vespalib::string & docType) { - for (const auto & dbCfg : documentDBs) { - if (dbCfg.inputdoctypename == docType) { - return & dbCfg; - } - } - return &_G_defaultProtonDocumentDBConfig; -} - class MetricsUpdateHook : public metrics::UpdateHook { DocumentDB &_db; public: @@ -182,9 +155,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _feedHandler(std::make_unique<FeedHandler>(_writeService, tlsSpec, docTypeName, *this, _writeFilter, *this, tlsWriterFactory)), _subDBs(*this, *this, *_feedHandler, _docTypeName, _writeService, warmupExecutor, fileHeaderContext, metricsWireService, getMetrics(), queryLimiter, clock, _configMutex, _baseDir, - makeSubDBConfig(protonCfg.distribution, - findDocumentDB(protonCfg.documentdb, docTypeName.getName())->allocation, - protonCfg.numsearcherthreads), + DocumentSubDBCollection::Config(protonCfg.numsearcherthreads), hwInfo), _maintenanceController(_writeService.master(), sharedExecutor, _docTypeName), _lidSpaceCompactionHandlers(), @@ -1113,4 +1084,10 @@ DocumentDB::transient_memory_usage_provider() return _transient_memory_usage_provider; } +void +DocumentDB::set_attribute_usage_listener(std::unique_ptr<IAttributeUsageListener> listener) +{ + _writeFilter.set_listener(std::move(listener)); +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index 9c2facdddb0..b5d975884a3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -416,6 +416,8 @@ public: IDiskMemUsageListener *diskMemUsageListener() { return &_dmUsageForwarder; } std::shared_ptr<const ITransientMemoryUsageProvider> transient_memory_usage_provider(); ExecutorThreadingService & getWriteService() { return _writeService; } + + void set_attribute_usage_listener(std::unique_ptr<IAttributeUsageListener> listener); }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp index baa6c8eb450..7c487043b5b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp @@ -14,6 +14,7 @@ #include <vespa/searchcore/config/config-ranking-constants.h> #include <vespa/searchcore/config/config-onnx-models.h> #include <vespa/searchcore/proton/attribute/attribute_aspect_delayer.h> +#include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/common/document_type_inspector.h> #include <vespa/searchcore/proton/common/indexschema_inspector.h> @@ -49,7 +50,8 @@ DocumentDBConfig::ComparisonResult::ComparisonResult() storeChanged(false), visibilityDelayChanged(false), flushChanged(false), - threading_service_config_changed(false) + threading_service_config_changed(false), + alloc_config_changed(false) { } DocumentDBConfig::DocumentDBConfig( @@ -70,6 +72,7 @@ DocumentDBConfig::DocumentDBConfig( const DocumentDBMaintenanceConfig::SP &maintenance, const search::LogDocumentStore::Config & storeConfig, std::shared_ptr<const ThreadingServiceConfig> threading_service_config, + std::shared_ptr<const AllocConfig> alloc_config, const vespalib::string &configId, const vespalib::string &docTypeName) noexcept : _configId(configId), @@ -91,6 +94,7 @@ DocumentDBConfig::DocumentDBConfig( _maintenance(maintenance), _storeConfig(storeConfig), _threading_service_config(std::move(threading_service_config)), + _alloc_config(std::move(alloc_config)), _orig(), _delayedAttributeAspects(false) { } @@ -117,6 +121,7 @@ DocumentDBConfig(const DocumentDBConfig &cfg) _maintenance(cfg._maintenance), _storeConfig(cfg._storeConfig), _threading_service_config(cfg._threading_service_config), + _alloc_config(cfg._alloc_config), _orig(cfg._orig), _delayedAttributeAspects(false) { } @@ -141,7 +146,8 @@ DocumentDBConfig::operator==(const DocumentDBConfig & rhs) const equals<Schema>(_schema.get(), rhs._schema.get()) && equals<DocumentDBMaintenanceConfig>(_maintenance.get(), rhs._maintenance.get()) && _storeConfig == rhs._storeConfig && - equals<ThreadingServiceConfig>(_threading_service_config.get(), rhs._threading_service_config.get()); + equals<ThreadingServiceConfig>(_threading_service_config.get(), rhs._threading_service_config.get()) && + equals<AllocConfig>(_alloc_config.get(), rhs._alloc_config.get()); } @@ -167,6 +173,7 @@ DocumentDBConfig::compare(const DocumentDBConfig &rhs) const retval.visibilityDelayChanged = (_maintenance->getVisibilityDelay() != rhs._maintenance->getVisibilityDelay()); retval.flushChanged = !equals<DocumentDBMaintenanceConfig>(_maintenance.get(), rhs._maintenance.get(), [](const auto &l, const auto &r) { return l.getFlushConfig() == r.getFlushConfig(); }); retval.threading_service_config_changed = !equals<ThreadingServiceConfig>(_threading_service_config.get(), rhs._threading_service_config.get()); + retval.alloc_config_changed = !equals<AllocConfig>(_alloc_config.get(), rhs._alloc_config.get()); return retval; } @@ -188,7 +195,8 @@ DocumentDBConfig::valid() const _tuneFileDocumentDB && _schema && _maintenance && - _threading_service_config; + _threading_service_config && + _alloc_config; } namespace @@ -232,6 +240,7 @@ DocumentDBConfig::makeReplayConfig(const SP & orig) o._maintenance, o._storeConfig, o._threading_service_config, + o._alloc_config, o._configId, o._docTypeName); ret->_orig = orig; @@ -274,6 +283,7 @@ DocumentDBConfig::newFromAttributesConfig(const AttributesConfigSP &attributes) _maintenance, _storeConfig, _threading_service_config, + _alloc_config, _configId, _docTypeName); } @@ -311,6 +321,7 @@ DocumentDBConfig::makeDelayedAttributeAspectConfig(const SP &newCfg, const Docum n._maintenance, n._storeConfig, n._threading_service_config, + n._alloc_config, n._configId, n._docTypeName); result->_delayedAttributeAspects = true; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h index dc163e91ade..8e24ed8e96a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h @@ -30,6 +30,7 @@ namespace document::internal { class InternalDocumenttypesType; } namespace proton { class ThreadingServiceConfig; +class AllocConfig; class DocumentDBConfig { @@ -55,6 +56,7 @@ public: bool visibilityDelayChanged; bool flushChanged; bool threading_service_config_changed; + bool alloc_config_changed; ComparisonResult(); ComparisonResult &setRankProfilesChanged(bool val) { rankProfilesChanged = val; return *this; } @@ -88,6 +90,7 @@ public: return *this; } ComparisonResult &set_threading_service_config_changed(bool val) { threading_service_config_changed = val; return *this; } + ComparisonResult &set_alloc_config_changed(bool val) { alloc_config_changed = val; return *this; } }; using SP = std::shared_ptr<DocumentDBConfig>; @@ -131,6 +134,7 @@ private: MaintenanceConfigSP _maintenance; search::LogDocumentStore::Config _storeConfig; std::shared_ptr<const ThreadingServiceConfig> _threading_service_config; + std::shared_ptr<const AllocConfig> _alloc_config; SP _orig; bool _delayedAttributeAspects; @@ -169,6 +173,7 @@ public: const DocumentDBMaintenanceConfig::SP &maintenance, const search::LogDocumentStore::Config & storeConfig, std::shared_ptr<const ThreadingServiceConfig> threading_service_config, + std::shared_ptr<const AllocConfig> alloc_config, const vespalib::string &configId, const vespalib::string &docTypeName) noexcept; @@ -210,6 +215,8 @@ public: bool getDelayedAttributeAspects() const { return _delayedAttributeAspects; } const ThreadingServiceConfig& get_threading_service_config() const { return *_threading_service_config; } const std::shared_ptr<const ThreadingServiceConfig>& get_threading_service_config_shared_ptr() const { return _threading_service_config; } + const AllocConfig& get_alloc_config() const { return *_alloc_config; } + const std::shared_ptr<const AllocConfig>& get_alloc_config_shared_ptr() const { return _alloc_config; } bool operator==(const DocumentDBConfig &rhs) const; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index 115a49fe997..f5a594d2f36 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -3,6 +3,7 @@ #include "documentdbconfigmanager.h" #include "bootstrapconfig.h" #include "threading_service_config.h" +#include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/config/config-ranking-constants.h> #include <vespa/searchcore/config/config-onnx-models.h> @@ -252,6 +253,19 @@ build_threading_service_config(const ProtonConfig &proton_config, hw_info.cpu())); } +std::shared_ptr<const AllocConfig> +build_alloc_config(const ProtonConfig& proton_config, const vespalib::string& doc_type_name) +{ + auto& document_db_config_entry = find_document_db_config_entry(proton_config.documentdb, doc_type_name); + auto& alloc_config = document_db_config_entry.allocation; + auto& distribution_config = proton_config.distribution; + search::GrowStrategy grow_strategy(alloc_config.initialnumdocs, alloc_config.growfactor, alloc_config.growbias, alloc_config.multivaluegrowfactor); + search::CompactionStrategy compaction_strategy(alloc_config.maxDeadBytesRatio, alloc_config.maxDeadAddressSpaceRatio); + return std::make_shared<const AllocConfig> + (AllocStrategy(grow_strategy, compaction_strategy, alloc_config.amortizecount), + distribution_config.redundancy, distribution_config.searchablecopies); +} + } void @@ -275,6 +289,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) MaintenanceConfigSP oldMaintenanceConfig; MaintenanceConfigSP newMaintenanceConfig; std::shared_ptr<const ThreadingServiceConfig> old_threading_service_config; + std::shared_ptr<const AllocConfig> old_alloc_config; if (!_ignoreForwardedConfig) { if (!(_bootstrapConfig->getDocumenttypesConfigSP() && @@ -299,6 +314,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) newIndexschemaConfig = current->getIndexschemaConfigSP(); oldMaintenanceConfig = current->getMaintenanceConfigSP(); old_threading_service_config = current->get_threading_service_config_shared_ptr(); + old_alloc_config = current->get_alloc_config_shared_ptr(); currentGeneration = current->getGeneration(); } @@ -382,6 +398,10 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) (*new_threading_service_config == *old_threading_service_config)) { new_threading_service_config = old_threading_service_config; } + auto new_alloc_config = build_alloc_config(_bootstrapConfig->getProtonConfig(), _docTypeName); + if (new_alloc_config && old_alloc_config &&(*new_alloc_config == *old_alloc_config)) { + new_alloc_config = old_alloc_config; + } auto newSnapshot = std::make_shared<DocumentDBConfig>(generation, newRankProfilesConfig, newRankingConstants, @@ -399,6 +419,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) newMaintenanceConfig, storeConfig, new_threading_service_config, + new_alloc_config, _configId, _docTypeName); assert(newSnapshot->valid()); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp index 61cf45a81d7..6ce0b896a50 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp @@ -19,13 +19,8 @@ using vespalib::makeLambdaTask; namespace proton { -DocumentSubDBCollection::Config::Config(GrowStrategy ready, GrowStrategy notReady, GrowStrategy removed, - size_t fixedAttributeTotalSkew, size_t numSearchThreads) - : _readyGrowth(ready), - _notReadyGrowth(notReady), - _removedGrowth(removed), - _fixedAttributeTotalSkew(fixedAttributeTotalSkew), - _numSearchThreads(numSearchThreads) +DocumentSubDBCollection::Config::Config(size_t numSearchThreads) + : _numSearchThreads(numSearchThreads) { } DocumentSubDBCollection::DocumentSubDBCollection( @@ -66,7 +61,6 @@ DocumentSubDBCollection::DocumentSubDBCollection( SearchableDocSubDB::Config( FastAccessDocSubDB::Config( StoreOnlyDocSubDB::Config(docTypeName, "0.ready", baseDir, - cfg.getReadyGrowth(), cfg.getFixedAttributeTotalSkew(), _readySubDbId, SubDbType::READY), true, true, false), cfg.getNumSearchThreads()), @@ -76,15 +70,14 @@ DocumentSubDBCollection::DocumentSubDBCollection( _subDBs.push_back (new StoreOnlyDocSubDB( - StoreOnlyDocSubDB::Config(docTypeName, "1.removed", baseDir, cfg.getRemovedGrowth(), - cfg.getFixedAttributeTotalSkew(), _remSubDbId, SubDbType::REMOVED), + StoreOnlyDocSubDB::Config(docTypeName, "1.removed", baseDir, + _remSubDbId, SubDbType::REMOVED), context)); _subDBs.push_back (new FastAccessDocSubDB( FastAccessDocSubDB::Config( StoreOnlyDocSubDB::Config(docTypeName, "2.notready", baseDir, - cfg.getNotReadyGrowth(), cfg.getFixedAttributeTotalSkew(), _notReadySubDbId, SubDbType::NOTREADY), true, true, true), FastAccessDocSubDB::Context(context, metrics.notReady.attributes, metricsWireService))); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h index 317ec191d60..c86370c942b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h @@ -5,7 +5,6 @@ #include <vespa/searchcore/proton/bucketdb/bucketdbhandler.h> #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/persistenceengine/i_document_retriever.h> -#include <vespa/searchcommon/common/growstrategy.h> #include <vespa/searchlib/common/serialnum.h> #include <vespa/vespalib/util/varholder.h> #include <mutex> @@ -61,19 +60,9 @@ public: using SerialNum = search::SerialNum; class Config { public: - using GrowStrategy = search::GrowStrategy; - Config(GrowStrategy ready, GrowStrategy notReady, GrowStrategy removed, - size_t fixedAttributeTotalSkew, size_t numSearchThreads); - GrowStrategy getReadyGrowth() const { return _readyGrowth; } - GrowStrategy getNotReadyGrowth() const { return _notReadyGrowth; } - GrowStrategy getRemovedGrowth() const { return _removedGrowth; } - size_t getNumSearchThreads() const { return _numSearchThreads; } - size_t getFixedAttributeTotalSkew() const { return _fixedAttributeTotalSkew; } + Config(size_t numSearchThreads); + size_t getNumSearchThreads() const noexcept { return _numSearchThreads; } private: - const GrowStrategy _readyGrowth; - const GrowStrategy _notReadyGrowth; - const GrowStrategy _removedGrowth; - const size_t _fixedAttributeTotalSkew; const size_t _numSearchThreads; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp index f92f3e74e31..ca2a98d43e0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp @@ -13,6 +13,7 @@ #include <vespa/searchcore/proton/attribute/attribute_populator.h> #include <vespa/searchcore/proton/attribute/filter_attribute_manager.h> #include <vespa/searchcore/proton/attribute/sequential_attributes_initializer.h> +#include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/matching/sessionmanager.h> #include <vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h> #include <vespa/searchcore/proton/reprocessing/document_reprocessing_handler.h> @@ -62,6 +63,7 @@ FastAccessDocSubDB::createAttributeManagerInitializer(const DocumentDBConfig &co DocumentMetaStore::SP documentMetaStore, std::shared_ptr<AttributeManager::SP> attrMgrResult) const { + AllocStrategy alloc_strategy = configSnapshot.get_alloc_config().make_alloc_strategy(_subDbType); IAttributeFactory::SP attrFactory = std::make_shared<AttributeFactory>(); AttributeManager::SP baseAttrMgr = std::make_shared<AttributeManager>(_baseDir + "/attribute", @@ -77,8 +79,7 @@ FastAccessDocSubDB::createAttributeManagerInitializer(const DocumentDBConfig &co documentMetaStore, baseAttrMgr, (_hasAttributes ? configSnapshot.getAttributesConfig() : AttributesConfig()), - _attributeGrow, - _attributeGrowNumDocs, + alloc_strategy, _fastAccessAttributesOnly, _writeService.master(), attrMgrResult); @@ -101,11 +102,10 @@ FastAccessDocSubDB::setupAttributeManager(AttributeManager::SP attrMgrResult) AttributeCollectionSpec::UP -FastAccessDocSubDB::createAttributeSpec(const AttributesConfig &attrCfg, SerialNum serialNum) const +FastAccessDocSubDB::createAttributeSpec(const AttributesConfig &attrCfg, const AllocStrategy& alloc_strategy, SerialNum serialNum) const { uint32_t docIdLimit(_dms->getCommittedDocIdLimit()); - AttributeCollectionSpecFactory factory(_attributeGrow, - _attributeGrowNumDocs, _fastAccessAttributesOnly); + AttributeCollectionSpecFactory factory(alloc_strategy, _fastAccessAttributesOnly); return factory.create(attrCfg, docIdLimit, serialNum); } @@ -246,7 +246,8 @@ FastAccessDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const { (void) resolver; - reconfigure(newConfigSnapshot.getStoreConfig()); + AllocStrategy alloc_strategy = newConfigSnapshot.get_alloc_config().make_alloc_strategy(_subDbType); + reconfigure(newConfigSnapshot.getStoreConfig(), alloc_strategy); IReprocessingTask::List tasks; /* * If attribute manager should change then document retriever @@ -262,7 +263,7 @@ FastAccessDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const std::make_unique<AttributeWriterFactory>(), getSubDbName()); proton::IAttributeManager::SP oldMgr = extractAttributeManager(_fastAccessFeedView.get()); AttributeCollectionSpec::UP attrSpec = - createAttributeSpec(newConfigSnapshot.getAttributesConfig(), serialNum); + createAttributeSpec(newConfigSnapshot.getAttributesConfig(), alloc_strategy, serialNum); IReprocessingInitializer::UP initializer = configurer.reconfigure(newConfigSnapshot, oldConfigSnapshot, *attrSpec); if (initializer->hasReprocessors()) { diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h index 2bf574aba10..7e2ac3a67cc 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h @@ -84,7 +84,7 @@ protected: MetricsWireService &_metricsWireService; DocIdLimit _docIdLimit; - AttributeCollectionSpec::UP createAttributeSpec(const AttributesConfig &attrCfg, SerialNum serialNum) const; + AttributeCollectionSpec::UP createAttributeSpec(const AttributesConfig &attrCfg, const AllocStrategy& alloc_strategy, SerialNum serialNum) const; AttributeManager::SP getAndResetInitAttributeManager(); virtual IFlushTargetList getFlushTargetsInternal() override; void reconfigureAttributeMetrics(const IAttributeManager &newMgr, const IAttributeManager &oldMgr); diff --git a/searchcore/src/vespa/searchcore/proton/server/i_move_operation_limiter.h b/searchcore/src/vespa/searchcore/proton/server/i_move_operation_limiter.h index d9fc426b08a..cc91413826c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_move_operation_limiter.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_move_operation_limiter.h @@ -11,7 +11,7 @@ namespace proton { * Interface used to limit the number of outstanding move operations a blockable maintenance job can have. */ struct IMoveOperationLimiter { - virtual ~IMoveOperationLimiter() {} + virtual ~IMoveOperationLimiter() = default; virtual std::shared_ptr<vespalib::IDestructorCallback> beginOperation() = 0; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/ibucketmodifiedhandler.h b/searchcore/src/vespa/searchcore/proton/server/ibucketmodifiedhandler.h index 652c303283f..1c23c35e082 100644 --- a/searchcore/src/vespa/searchcore/proton/server/ibucketmodifiedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/ibucketmodifiedhandler.h @@ -10,7 +10,7 @@ class IBucketModifiedHandler { public: virtual void notifyBucketModified(const document::BucketId &bucket) = 0; - virtual ~IBucketModifiedHandler() {} + virtual ~IBucketModifiedHandler() = default; }; } diff --git a/searchcore/src/vespa/searchcore/proton/server/imaintenancejobrunner.h b/searchcore/src/vespa/searchcore/proton/server/imaintenancejobrunner.h index 617aa69631f..f733821a47b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/imaintenancejobrunner.h +++ b/searchcore/src/vespa/searchcore/proton/server/imaintenancejobrunner.h @@ -14,7 +14,7 @@ public: * Schedule job to be run in the future. */ virtual void run() = 0; - virtual ~IMaintenanceJobRunner() { } + virtual ~IMaintenanceJobRunner() = default; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp index 78efdaebe16..095169b84ce 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp @@ -30,7 +30,7 @@ LidSpaceCompactionJob::scanDocuments(const LidUsageStats &stats) if ( ! op ) { return false; } - vespalib::IDestructorCallback::SP context = _moveOpsLimiter->beginOperation(); + vespalib::IDestructorCallback::SP context = getLimiter().beginOperation(); _opStorer.appendOperation(*op, context); _handler->handleMove(*op, std::move(context)); if (isBlocked(BlockedReason::OUTSTANDING_OPS)) { diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_take2.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_take2.cpp index 6e1b8ed79b5..cbf3de20b1f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_take2.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_take2.cpp @@ -27,7 +27,7 @@ CompactionJob::scanDocuments(const LidUsageStats &stats) DocumentMetaData document = getNextDocument(stats, false); if (document.valid()) { Bucket metaBucket(document::Bucket(_bucketSpace, document.bucketId)); - IDestructorCallback::SP context = _moveOpsLimiter->beginOperation(); + IDestructorCallback::SP context = getLimiter().beginOperation(); auto failed = _bucketExecutor.execute(metaBucket, makeBucketTask([this, meta=document, opsTracker=std::move(context)] (const Bucket & bucket, std::shared_ptr<IDestructorCallback> onDone) { assert(bucket.getBucketId() == meta.bucketId); using DoneContext = vespalib::KeepAlive<std::pair<IDestructorCallback::SP, IDestructorCallback::SP>>; diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_take2.h b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_take2.h index 35e02246255..053950ebd2f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_take2.h +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_take2.h @@ -6,7 +6,8 @@ #include <vespa/document/bucket/bucketspace.h> namespace storage::spi { struct BucketExecutor;} -namespace searchcorespi::index { class IThreadService; } +namespace searchcorespi::index { struct IThreadService; } +namespace vespalib { class IDestructorCallback; } namespace proton { class IDiskMemUsageNotifier; class IClusterStateChangedNotifier; diff --git a/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.h b/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.h index 01737336a58..b5c8b1b9998 100644 --- a/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.h +++ b/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.h @@ -35,7 +35,7 @@ private: public: using SP = std::shared_ptr<MoveOperationLimiter>; MoveOperationLimiter(IBlockableMaintenanceJob *job, uint32_t maxOutstandingOps); - ~MoveOperationLimiter(); + ~MoveOperationLimiter() override; void clearJob(); bool isAboveLimit() const; bool hasPending() const; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 6706581f52c..1b615d765f0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -18,14 +18,16 @@ #include <vespa/document/base/exceptions.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/metrics/updatehook.h> +#include <vespa/searchcore/proton/attribute/i_attribute_usage_listener.h> #include <vespa/searchcore/proton/flushengine/flush_engine_explorer.h> #include <vespa/searchcore/proton/flushengine/flushengine.h> #include <vespa/searchcore/proton/flushengine/tls_stats_factory.h> #include <vespa/searchcore/proton/matchengine/matchengine.h> +#include <vespa/searchcore/proton/persistenceengine/persistenceengine.h> #include <vespa/searchcore/proton/reference/document_db_reference_registry.h> #include <vespa/searchcore/proton/summaryengine/docsum_by_slime.h> #include <vespa/searchcore/proton/summaryengine/summaryengine.h> -#include <vespa/searchcore/proton/persistenceengine/persistenceengine.h> #include <vespa/searchlib/common/packets.h> #include <vespa/searchlib/transactionlog/trans_log_server_explorer.h> #include <vespa/searchlib/transactionlog/translogserverapp.h> @@ -36,7 +38,6 @@ #include <vespa/vespalib/util/host_name.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/random.h> -#include <vespa/metrics/updatehook.h> #include <vespa/searchlib/aggregation/forcelink.hpp> #include <vespa/searchlib/expression/forcelink.hpp> @@ -303,7 +304,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) vespalib::chdir(protonConfig.basedir); _tls->start(); _flushEngine = std::make_unique<FlushEngine>(std::make_shared<flushengine::TlsStatsFactory>(_tls->getTransLogServer()), - strategy, flush.maxconcurrent, flush.idleinterval*1000); + strategy, flush.maxconcurrent, vespalib::from_s(flush.idleinterval)); _metricsEngine->addExternalMetrics(_summaryEngine->getMetrics()); char tmp[1024]; @@ -630,6 +631,8 @@ Proton::addDocumentDB(const document::DocumentType &docType, } // TODO: Fix race with new cluster state setting. _persistenceEngine->putHandler(persistenceWGuard, bucketSpace, docTypeName, persistenceHandler); + ret->set_attribute_usage_listener( + _persistenceEngine->get_resource_usage_tracker().make_attribute_usage_listener(docTypeName.getName())); } auto searchHandler = std::make_shared<SearchHandlerProxy>(ret); _summaryEngine->putSearchHandler(docTypeName, searchHandler); diff --git a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp index 4fc241571ac..bb6c5423175 100644 --- a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp @@ -27,7 +27,8 @@ ReconfigParams::configHasChanged() const _res.tuneFileDocumentDBChanged || _res.schemaChanged || _res.maintenanceChanged || - _res.storeChanged; + _res.storeChanged || + _res.alloc_config_changed; } bool @@ -51,7 +52,7 @@ ReconfigParams::shouldIndexManagerChange() const bool ReconfigParams::shouldAttributeManagerChange() const { - return _res.attributesChanged || _res.importedFieldsChanged || _res.visibilityDelayChanged; + return _res.attributesChanged || _res.importedFieldsChanged || _res.visibilityDelayChanged || _res.alloc_config_changed; } bool diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp index 2768c7ea337..51e6f8e45df 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp @@ -7,6 +7,7 @@ #include "i_document_subdb_owner.h" #include "ibucketstatecalculator.h" #include <vespa/searchcore/proton/attribute/attribute_writer.h> +#include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/flushengine/threadedflushtarget.h> #include <vespa/searchcore/proton/index/index_manager_initializer.h> #include <vespa/searchcore/proton/index/index_writer.h> @@ -143,7 +144,8 @@ IReprocessingTask::List SearchableDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot, SerialNum serialNum, const ReconfigParams ¶ms, IDocumentDBReferenceResolver &resolver) { - StoreOnlyDocSubDB::reconfigure(newConfigSnapshot.getStoreConfig()); + AllocStrategy alloc_strategy = newConfigSnapshot.get_alloc_config().make_alloc_strategy(_subDbType); + StoreOnlyDocSubDB::reconfigure(newConfigSnapshot.getStoreConfig(), alloc_strategy); IReprocessingTask::List tasks; applyFlushConfig(newConfigSnapshot.getMaintenanceConfigSP()->getFlushConfig()); if (params.shouldMatchersChange() && _addMetrics) { @@ -152,7 +154,7 @@ SearchableDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const if (params.shouldAttributeManagerChange()) { proton::IAttributeManager::SP oldMgr = getAttributeManager(); AttributeCollectionSpec::UP attrSpec = - createAttributeSpec(newConfigSnapshot.getAttributesConfig(), serialNum); + createAttributeSpec(newConfigSnapshot.getAttributesConfig(), alloc_strategy, serialNum); IReprocessingInitializer::UP initializer = _configurer.reconfigure(newConfigSnapshot, oldConfigSnapshot, *attrSpec, params, resolver); if (initializer && initializer->hasReprocessors()) { diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp index 2aeece204fb..9b504c74635 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp @@ -10,6 +10,7 @@ #include "storeonlydocsubdb.h" #include <vespa/searchcore/proton/attribute/attribute_writer.h> #include <vespa/searchcore/proton/bucketdb/ibucketdbhandlerinitializer.h> +#include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/docsummary/summaryflushtarget.h> #include <vespa/searchcore/proton/docsummary/summarymanagerinitializer.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastoreinitializer.h> @@ -28,6 +29,7 @@ #include <vespa/log/log.h> LOG_SETUP(".proton.server.storeonlydocsubdb"); +using search::CompactionStrategy; using search::GrowStrategy; using search::AttributeGuard; using search::AttributeVector; @@ -56,13 +58,10 @@ IIndexWriter::SP nullIndexWriter; StoreOnlyDocSubDB::Config::Config(const DocTypeName &docTypeName, const vespalib::string &subName, const vespalib::string &baseDir, - const search::GrowStrategy &attributeGrow, size_t attributeGrowNumDocs, uint32_t subDbId, SubDbType subDbType) : _docTypeName(docTypeName), _subName(subName), _baseDir(baseDir + "/" + subName), - _attributeGrow(attributeGrow), - _attributeGrowNumDocs(attributeGrowNumDocs), _subDbId(subDbId), _subDbType(subDbType) { } @@ -99,8 +98,6 @@ StoreOnlyDocSubDB::StoreOnlyDocSubDB(const Config &cfg, const Context &ctx) _bucketDB(ctx._bucketDB), _bucketDBHandlerInitializer(ctx._bucketDBHandlerInitializer), _metaStoreCtx(), - _attributeGrow(cfg._attributeGrow), - _attributeGrowNumDocs(cfg._attributeGrowNumDocs), _flushedDocumentMetaStoreSerialNum(0u), _flushedDocumentStoreSerialNum(0u), _dms(), @@ -222,11 +219,12 @@ StoreOnlyDocSubDB::getNewestFlushedSerial() initializer::InitializerTask::SP StoreOnlyDocSubDB:: createSummaryManagerInitializer(const search::LogDocumentStore::Config & storeCfg, + const AllocStrategy& alloc_strategy, const search::TuneFileSummary &tuneFile, search::IBucketizer::SP bucketizer, std::shared_ptr<SummaryManager::SP> result) const { - GrowStrategy grow = _attributeGrow; + GrowStrategy grow = alloc_strategy.get_grow_strategy(); vespalib::string baseDir(_baseDir + "/summary"); return std::make_shared<SummaryManagerInitializer> (grow, baseDir, getSubDbName(), _docTypeName, _writeService.shared(), @@ -245,12 +243,13 @@ StoreOnlyDocSubDB::setupSummaryManager(SummaryManager::SP summaryManager) InitializerTask::SP StoreOnlyDocSubDB:: -createDocumentMetaStoreInitializer(const search::TuneFileAttributes &tuneFile, +createDocumentMetaStoreInitializer(const AllocStrategy& alloc_strategy, + const search::TuneFileAttributes &tuneFile, std::shared_ptr<DocumentMetaStoreInitializerResult::SP> result) const { - GrowStrategy grow = _attributeGrow; + GrowStrategy grow = alloc_strategy.get_grow_strategy(); // Amortize memory spike cost over N docs - grow.setDocsGrowDelta(grow.getDocsGrowDelta() + _attributeGrowNumDocs); + grow.setDocsGrowDelta(grow.getDocsGrowDelta() + alloc_strategy.get_amortize_count()); vespalib::string baseDir(_baseDir + "/documentmetastore"); vespalib::string name = DocumentMetaStore::getFixedName(); vespalib::string attrFileName = baseDir + "/" + name; // XXX: Wrong @@ -292,10 +291,13 @@ StoreOnlyDocSubDB::createInitializer(const DocumentDBConfig &configSnapshot, Ser { auto result = std::make_unique<DocumentSubDbInitializer>(const_cast<StoreOnlyDocSubDB &>(*this), _writeService.master()); - auto dmsInitTask = createDocumentMetaStoreInitializer(configSnapshot.getTuneFileDocumentDBSP()->_attr, + AllocStrategy alloc_strategy = configSnapshot.get_alloc_config().make_alloc_strategy(_subDbType); + auto dmsInitTask = createDocumentMetaStoreInitializer(alloc_strategy, + configSnapshot.getTuneFileDocumentDBSP()->_attr, result->writableResult().writableDocumentMetaStore()); result->addDocumentMetaStoreInitTask(dmsInitTask); auto summaryTask = createSummaryManagerInitializer(configSnapshot.getStoreConfig(), + alloc_strategy, configSnapshot.getTuneFileDocumentDBSP()->_summary, result->result().documentMetaStore()->documentMetaStore(), result->writableResult().writableSummaryManager()); @@ -409,14 +411,22 @@ StoreOnlyDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const (void) params; (void) resolver; assert(_writeService.master().isCurrentThread()); - reconfigure(newConfigSnapshot.getStoreConfig()); + AllocStrategy alloc_strategy = newConfigSnapshot.get_alloc_config().make_alloc_strategy(_subDbType); + reconfigure(newConfigSnapshot.getStoreConfig(), alloc_strategy); initFeedView(newConfigSnapshot); return IReprocessingTask::List(); } void -StoreOnlyDocSubDB::reconfigure(const search::LogDocumentStore::Config & config) +StoreOnlyDocSubDB::reconfigure(const search::LogDocumentStore::Config & config, const AllocStrategy& alloc_strategy) { + auto cfg = _dms->getConfig(); + GrowStrategy grow = alloc_strategy.get_grow_strategy(); + // Amortize memory spike cost over N docs + grow.setDocsGrowDelta(grow.getDocsGrowDelta() + alloc_strategy.get_amortize_count()); + cfg.setGrowStrategy(grow); + cfg.setCompactionStrategy(alloc_strategy.get_compaction_strategy()); + _dms->update_config(cfg); // Update grow and compaction config _rSummaryMgr->reconfigure(config); } diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h index 7c3f7c82eb0..37229bd551c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h @@ -20,6 +20,7 @@ namespace proton { +class AllocStrategy; struct DocumentDBTaggedMetrics; class DocumentMetaStoreInitializerResult; class FeedHandler; @@ -85,14 +86,12 @@ public: const DocTypeName _docTypeName; const vespalib::string _subName; const vespalib::string _baseDir; - const search::GrowStrategy _attributeGrow; - const size_t _attributeGrowNumDocs; const uint32_t _subDbId; const SubDbType _subDbType; Config(const DocTypeName &docTypeName, const vespalib::string &subName, - const vespalib::string &baseDir, const search::GrowStrategy &attributeGrow, - size_t attributeGrowNumDocs, uint32_t subDbId, SubDbType subDbType); + const vespalib::string &baseDir, + uint32_t subDbId, SubDbType subDbType); ~Config(); }; @@ -130,8 +129,6 @@ protected: BucketDBOwner::SP _bucketDB; bucketdb::IBucketDBHandlerInitializer &_bucketDBHandlerInitializer; IDocumentMetaStoreContext::SP _metaStoreCtx; - const search::GrowStrategy _attributeGrow; - const size_t _attributeGrowNumDocs; // The following two serial numbers reflect state at program startup // and are used by replay logic. SerialNum _flushedDocumentMetaStoreSerialNum; @@ -164,6 +161,7 @@ protected: std::shared_ptr<initializer::InitializerTask> createSummaryManagerInitializer(const search::LogDocumentStore::Config & protonSummaryCfg, + const AllocStrategy& alloc_strategy, const search::TuneFileSummary &tuneFile, search::IBucketizer::SP bucketizer, std::shared_ptr<SummaryManager::SP> result) const; @@ -171,7 +169,8 @@ protected: void setupSummaryManager(SummaryManager::SP summaryManager); std::shared_ptr<initializer::InitializerTask> - createDocumentMetaStoreInitializer(const search::TuneFileAttributes &tuneFile, + createDocumentMetaStoreInitializer(const AllocStrategy& alloc_strategy, + const search::TuneFileAttributes &tuneFile, std::shared_ptr<std::shared_ptr<DocumentMetaStoreInitializerResult>> result) const; void setupDocumentMetaStore(std::shared_ptr<DocumentMetaStoreInitializerResult> dmsResult); @@ -181,7 +180,8 @@ protected: StoreOnlyFeedView::PersistentParams getFeedViewPersistentParams(); vespalib::string getSubDbName() const; - void reconfigure(const search::LogDocumentStore::Config & protonConfig); + void reconfigure(const search::LogDocumentStore::Config & protonConfig, + const AllocStrategy& alloc_strategy); public: StoreOnlyDocSubDB(const Config &cfg, const Context &ctx); ~StoreOnlyDocSubDB() override; diff --git a/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.cpp index 9c928f49ca8..b776e32e0db 100644 --- a/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/transactionlogmanager.cpp @@ -20,7 +20,7 @@ void TransactionLogManager::doLogReplayComplete(const vespalib::string &domainName, vespalib::duration elapsedTime) const { - EventLogger::transactionLogReplayComplete(domainName, vespalib::count_ms(elapsedTime)); + EventLogger::transactionLogReplayComplete(domainName, elapsedTime); } diff --git a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp index 335c7aca24c..17cb91ac4ce 100644 --- a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp +++ b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp @@ -10,6 +10,7 @@ #include <vespa/searchsummary/config/config-juniperrc.h> #include <vespa/document/config/config-documenttypes.h> #include <vespa/config-imported-fields.h> +#include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/server/threading_service_config.h> using document::DocumenttypesConfig; @@ -47,6 +48,7 @@ DocumentDBConfigBuilder::DocumentDBConfigBuilder(int64_t generation, _maintenance(std::make_shared<DocumentDBMaintenanceConfig>()), _store(), _threading_service_config(std::make_shared<const ThreadingServiceConfig>(ThreadingServiceConfig::make(1))), + _alloc_config(std::make_shared<const AllocConfig>()), _configId(configId), _docTypeName(docTypeName) { } @@ -70,6 +72,7 @@ DocumentDBConfigBuilder::DocumentDBConfigBuilder(const DocumentDBConfig &cfg) _maintenance(cfg.getMaintenanceConfigSP()), _store(cfg.getStoreConfig()), _threading_service_config(cfg.get_threading_service_config_shared_ptr()), + _alloc_config(cfg.get_alloc_config_shared_ptr()), _configId(cfg.getConfigId()), _docTypeName(cfg.getDocTypeName()) {} @@ -97,6 +100,7 @@ DocumentDBConfigBuilder::build() _maintenance, _store, _threading_service_config, + _alloc_config, _configId, _docTypeName); } diff --git a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h index 218a7c56fa9..706e14e73db 100644 --- a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h +++ b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h @@ -28,6 +28,7 @@ private: DocumentDBConfig::MaintenanceConfigSP _maintenance; search::LogDocumentStore::Config _store; std::shared_ptr<const ThreadingServiceConfig> _threading_service_config; + std::shared_ptr<const AllocConfig> _alloc_config; vespalib::string _configId; vespalib::string _docTypeName; diff --git a/searchcore/src/vespa/searchcore/proton/test/simple_job_tracker.h b/searchcore/src/vespa/searchcore/proton/test/simple_job_tracker.h index cf4decf0f4a..9048fb7a356 100644 --- a/searchcore/src/vespa/searchcore/proton/test/simple_job_tracker.h +++ b/searchcore/src/vespa/searchcore/proton/test/simple_job_tracker.h @@ -11,7 +11,7 @@ struct SimpleJobTracker : public IJobTracker typedef std::shared_ptr<SimpleJobTracker> SP; vespalib::CountDownLatch _started; vespalib::CountDownLatch _ended; - SimpleJobTracker(uint32_t numJobTrackings) + SimpleJobTracker(uint32_t numJobTrackings) noexcept : _started(numJobTrackings), _ended(numJobTrackings) {} diff --git a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp index dde6b14121d..36a697eaa12 100644 --- a/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp +++ b/searchlib/src/tests/attribute/compaction/attribute_compaction_test.cpp @@ -226,4 +226,24 @@ TEST_F("Compaction is not executed when free lists are used", EXPECT_EQUAL(1001u, afterSpace.dead()); } +TEST_F("Compaction is peformed when compaction strategy is changed to enable compaction", + Fixture(compactAddressSpaceAttributeConfig(false))) +{ + populate_and_hammer(f, true); + AddressSpace after1 = f.getMultiValueAddressSpaceUsage("after1"); + // 100 * 1000 dead arrays due to new values for docids + // 1 reserved array accounted as dead + EXPECT_EQUAL(100001u, after1.dead()); + f._v->update_config(compactAddressSpaceAttributeConfig(true)); + auto old_dead = after1.dead(); + AddressSpace after2 = f.getMultiValueAddressSpaceUsage("after2"); + while (after2.dead() < old_dead) { + old_dead = after2.dead(); + f._v->commit(); // new commit might trigger further compaction + after2 = f.getMultiValueAddressSpaceUsage("after2"); + } + // DEAD_ARRAYS_SLACK in multi value mapping is is 64k + EXPECT_GREATER(65536u, after2.dead()); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/transactionlog/translogclient_test.cpp b/searchlib/src/tests/transactionlog/translogclient_test.cpp index e097eebd42c..fd598eb2d31 100644 --- a/searchlib/src/tests/transactionlog/translogclient_test.cpp +++ b/searchlib/src/tests/transactionlog/translogclient_test.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/document/util/bytebuffer.h> #include <vespa/fastos/file.h> +#include <thread> #include <vespa/log/log.h> LOG_SETUP("translogclient_test"); diff --git a/searchlib/src/vespa/searchlib/attribute/attributesaver.h b/searchlib/src/vespa/searchlib/attribute/attributesaver.h index f90f9492487..c1bf18e775c 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributesaver.h +++ b/searchlib/src/vespa/searchlib/attribute/attributesaver.h @@ -2,11 +2,10 @@ #pragma once -#include <vespa/vespalib/util/generationhandler.h> #include "attribute_header.h" +#include <vespa/vespalib/util/generationhandler.h> -namespace search -{ +namespace search { class IAttributeSaveTarget; diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index 2168bbe4276..d2574bd32a2 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -21,6 +21,7 @@ #include <vespa/vespalib/util/exceptions.h> #include <vespa/searchlib/util/logutil.h> #include <vespa/searchcommon/attribute/attribute_utils.h> +#include <thread> #include <vespa/log/log.h> LOG_SETUP(".searchlib.attribute.attributevector"); @@ -769,6 +770,31 @@ AttributeVector::logEnumStoreEvent(const char *reason, const char *stage) EV_STATE(eventName.c_str(), jstr.toString().data()); } +void +AttributeVector::drain_hold(uint64_t hold_limit) +{ + incGeneration(); + for (int retry = 0; ; ++retry) { + removeAllOldGenerations(); + updateStat(true); + if (_status.getOnHold() <= hold_limit) { + return; + } + std::this_thread::sleep_for(retry < 20 ? 20ms : 100ms); + } +} + +void +AttributeVector::update_config(const Config& cfg) +{ + commit(true); + drain_hold(1024 * 1024); // Wait until 1MiB or less on hold + _config.setGrowStrategy(cfg.getGrowStrategy()); + _config.setCompactionStrategy(cfg.getCompactionStrategy()); + commit(); // might trigger compaction if compaction strategy changed + drain_hold(1024 * 1024); // Wait until 1MiB or less on hold +} + template bool AttributeVector::append<StringChangeData>(ChangeVectorT< ChangeTemplate<StringChangeData> > &changes, uint32_t , const StringChangeData &, int32_t, bool); template bool AttributeVector::update<StringChangeData>(ChangeVectorT< ChangeTemplate<StringChangeData> > &changes, uint32_t , const StringChangeData &); template bool AttributeVector::remove<StringChangeData>(ChangeVectorT< ChangeTemplate<StringChangeData> > &changes, uint32_t , const StringChangeData &, int32_t); diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index 4fc6589850e..f308ee8d024 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -393,6 +393,7 @@ public: /** Return the fixed length of the attribute. If 0 then you must inquire each document. */ size_t getFixedWidth() const override { return _config.basicType().fixedSize(); } const Config &getConfig() const { return _config; } + void update_config(const Config& cfg); BasicType getInternalBasicType() const { return _config.basicType(); } CollectionType getInternalCollectionType() const { return _config.collectionType(); } const BaseName & getBaseFileName() const { return _baseFileName; } @@ -667,6 +668,8 @@ public: static bool isEnumerated(const vespalib::GenericHeader &header); virtual vespalib::MemoryUsage getChangeVectorMemoryUsage() const; + + void drain_hold(uint64_t hold_limit); }; } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java index def3e49be4d..d2a42d21973 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/DefaultTlsContext.java @@ -43,10 +43,6 @@ public class DefaultTlsContext implements TlsContext { this(sslContext, TlsContext.ALLOWED_CIPHER_SUITES, peerAuthentication); } - public DefaultTlsContext(SSLContext sslContext) { - this(sslContext, TlsContext.ALLOWED_CIPHER_SUITES, PeerAuthentication.NEED); - } - DefaultTlsContext(SSLContext sslContext, Set<String> acceptedCiphers, PeerAuthentication peerAuthentication) { this.sslContext = sslContext; this.peerAuthentication = peerAuthentication; diff --git a/slobrok/src/tests/configure/configure.cpp b/slobrok/src/tests/configure/configure.cpp index 0e5b68053c8..98c716b3a48 100644 --- a/slobrok/src/tests/configure/configure.cpp +++ b/slobrok/src/tests/configure/configure.cpp @@ -12,6 +12,7 @@ #include <vespa/vespalib/util/host_name.h> #include <algorithm> #include <iostream> +#include <thread> #include <vespa/log/log.h> LOG_SETUP("configure_test"); diff --git a/slobrok/src/tests/mirrorapi/mirrorapi.cpp b/slobrok/src/tests/mirrorapi/mirrorapi.cpp index 4126f34716f..ae309f8c661 100644 --- a/slobrok/src/tests/mirrorapi/mirrorapi.cpp +++ b/slobrok/src/tests/mirrorapi/mirrorapi.cpp @@ -8,6 +8,7 @@ #include <vespa/fnet/frt/supervisor.h> #include <vespa/fnet/frt/target.h> #include <vespa/fnet/transport.h> +#include <thread> #include <vespa/log/log.h> LOG_SETUP("mirrorapi_test"); diff --git a/slobrok/src/tests/registerapi/registerapi.cpp b/slobrok/src/tests/registerapi/registerapi.cpp index 92f08ee41cb..59bc4690985 100644 --- a/slobrok/src/tests/registerapi/registerapi.cpp +++ b/slobrok/src/tests/registerapi/registerapi.cpp @@ -8,6 +8,7 @@ #include <vespa/fnet/frt/supervisor.h> #include <sstream> #include <algorithm> +#include <thread> #include <vespa/log/log.h> LOG_SETUP("registerapi_test"); diff --git a/slobrok/src/tests/standalone/standalone.cpp b/slobrok/src/tests/standalone/standalone.cpp index 65553c57530..883aeed1494 100644 --- a/slobrok/src/tests/standalone/standalone.cpp +++ b/slobrok/src/tests/standalone/standalone.cpp @@ -3,6 +3,7 @@ #include <vespa/slobrok/server/slobrokserver.h> #include <vespa/fnet/frt/supervisor.h> #include <vespa/fnet/frt/target.h> +#include <thread> //----------------------------------------------------------------------------- diff --git a/staging_vespalib/src/tests/clock/clock_test.cpp b/staging_vespalib/src/tests/clock/clock_test.cpp index 06eee21d2b9..b1572d11dd7 100644 --- a/staging_vespalib/src/tests/clock/clock_test.cpp +++ b/staging_vespalib/src/tests/clock/clock_test.cpp @@ -3,6 +3,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/clock.h> #include <vespa/fastos/thread.h> +#include <thread> using vespalib::Clock; using vespalib::duration; diff --git a/staging_vespalib/src/tests/shutdownguard/shutdownguard_test.cpp b/staging_vespalib/src/tests/shutdownguard/shutdownguard_test.cpp index 04a811494c9..24607add153 100644 --- a/staging_vespalib/src/tests/shutdownguard/shutdownguard_test.cpp +++ b/staging_vespalib/src/tests/shutdownguard/shutdownguard_test.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/shutdownguard.h> +#include <thread> #include <unistd.h> #include <sys/wait.h> #include <cstdlib> diff --git a/staging_vespalib/src/tests/timer/timer_test.cpp b/staging_vespalib/src/tests/timer/timer_test.cpp index 04b541e75e9..2031e3cdcbe 100644 --- a/staging_vespalib/src/tests/timer/timer_test.cpp +++ b/staging_vespalib/src/tests/timer/timer_test.cpp @@ -37,10 +37,10 @@ void Test::testScheduling() vespalib::CountDownLatch latch1(3); vespalib::CountDownLatch latch2(2); ScheduledExecutor timer; - timer.scheduleAtFixedRate(Task::UP(new TestTask(latch1)), 100ms, 200ms); - timer.scheduleAtFixedRate(Task::UP(new TestTask(latch2)), 500ms, 500ms); - EXPECT_TRUE(latch1.await(60000)); - EXPECT_TRUE(latch2.await(60000)); + timer.scheduleAtFixedRate(std::make_unique<TestTask>(latch1), 100ms, 200ms); + timer.scheduleAtFixedRate(std::make_unique<TestTask>(latch2), 500ms, 500ms); + EXPECT_TRUE(latch1.await(60s)); + EXPECT_TRUE(latch2.await(60s)); } void Test::testReset() @@ -49,9 +49,9 @@ void Test::testReset() ScheduledExecutor timer; timer.scheduleAtFixedRate(std::make_unique<TestTask>(latch1), 2s, 3s); timer.reset(); - EXPECT_TRUE(!latch1.await(3000)); + EXPECT_TRUE(!latch1.await(3s)); timer.scheduleAtFixedRate(std::make_unique<TestTask>(latch1), 200ms, 300ms); - EXPECT_TRUE(latch1.await(60000)); + EXPECT_TRUE(latch1.await(60s)); } TEST_APPHOOK(Test) diff --git a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h b/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h index ed2209d130a..4e0388caf8a 100644 --- a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h +++ b/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h @@ -6,7 +6,6 @@ #include <vespa/vespalib/util/arrayqueue.hpp> #include <vespa/vespalib/util/gate.h> #include <vespa/vespalib/util/eventbarrier.hpp> -#include <vespa/vespalib/util/gate.h> #include <vespa/fastos/thread.h> #include <mutex> #include <condition_variable> diff --git a/storage/src/tests/persistence/filestorage/service_layer_host_info_reporter_test.cpp b/storage/src/tests/persistence/filestorage/service_layer_host_info_reporter_test.cpp index 86595fb44e8..a54c5c8ccf9 100644 --- a/storage/src/tests/persistence/filestorage/service_layer_host_info_reporter_test.cpp +++ b/storage/src/tests/persistence/filestorage/service_layer_host_info_reporter_test.cpp @@ -102,6 +102,28 @@ TEST_F(ServiceLayerHostInfoReporterTest, request_almost_immediate_node_state_as_ EXPECT_EQ(ResourceUsage(0.8, 0.7, {0.1, attr_es_name}, {0.2, attr_mv_name}), get_usage()); } +TEST_F(ServiceLayerHostInfoReporterTest, + first_valid_attribute_enum_store_sample_triggers_immediate_node_state_when_below_slack_diff) +{ + // TODO: Assert this is below slack diff when that becomes configurable. + constexpr double usage_below_slack_diff = 0.00001; + notify(0.0, 0.0, {usage_below_slack_diff, attr_es_name}, {}); + EXPECT_EQ(1, requested_almost_immediate_replies()); + EXPECT_EQ(ResourceUsage(0.0, 0.0, {usage_below_slack_diff, attr_es_name}, {}), get_old_usage()); + EXPECT_EQ(ResourceUsage(0.0, 0.0, {usage_below_slack_diff, attr_es_name}, {}), get_usage()); +} + +TEST_F(ServiceLayerHostInfoReporterTest, + first_valid_attribute_multi_value_sample_triggers_immediate_node_state_when_below_slack_diff) +{ + // TODO: Assert this is below slack diff when that becomes configurable. + constexpr double usage_below_slack_diff = 0.00001; + notify(0.0, 0.0, {}, {usage_below_slack_diff, attr_mv_name}); + EXPECT_EQ(1, requested_almost_immediate_replies()); + EXPECT_EQ(ResourceUsage(0.0, 0.0, {}, {usage_below_slack_diff, attr_mv_name}), get_old_usage()); + EXPECT_EQ(ResourceUsage(0.0, 0.0, {}, {usage_below_slack_diff, attr_mv_name}), get_usage()); +} + TEST_F(ServiceLayerHostInfoReporterTest, json_report_generated) { EXPECT_EQ(ResourceUsage(0.0, 0.0), get_slime_usage()); diff --git a/storage/src/vespa/storage/bucketdb/bucketcopy.h b/storage/src/vespa/storage/bucketdb/bucketcopy.h index 94a1e63e53e..3e93e4594a6 100644 --- a/storage/src/vespa/storage/bucketdb/bucketcopy.h +++ b/storage/src/vespa/storage/bucketdb/bucketcopy.h @@ -28,9 +28,9 @@ public: { } - bool trusted() const { return _flags & TRUSTED; } + bool trusted() const noexcept { return _flags & TRUSTED; } - BucketCopy& setTrusted(bool val = true) { + BucketCopy& setTrusted(bool val = true) noexcept { if (!val) { clearTrusted(); } else { @@ -40,46 +40,44 @@ public: return *this; } - void clearTrusted() { _flags &= ~TRUSTED; } + void clearTrusted() noexcept { _flags &= ~TRUSTED; } - bool valid() const { return getBucketInfo().valid(); } - bool empty() const { return getBucketInfo().empty(); } - bool wasRecentlyCreated() const { + bool valid() const noexcept { return getBucketInfo().valid(); } + bool empty() const noexcept { return getBucketInfo().empty(); } + bool wasRecentlyCreated() const noexcept { return (getChecksum() == 1 && getDocumentCount() == 0 && getTotalDocumentSize() == 0); } - static BucketCopy recentlyCreatedCopy(uint64_t timestamp, uint16_t nodeIdx) - { + static BucketCopy recentlyCreatedCopy(uint64_t timestamp, uint16_t nodeIdx) noexcept { return BucketCopy(timestamp, nodeIdx, api::BucketInfo(1, 0, 0, 0, 0)); } - uint16_t getNode() const { return _node; } - uint64_t getTimestamp() const { return _timestamp; } + uint16_t getNode() const noexcept { return _node; } + uint64_t getTimestamp() const noexcept { return _timestamp; } - uint32_t getChecksum() const { return _info.getChecksum(); } - uint32_t getDocumentCount() const { return _info.getDocumentCount(); } - uint32_t getTotalDocumentSize() const - { return _info.getTotalDocumentSize(); } - uint32_t getMetaCount() const { return _info.getMetaCount(); } - uint32_t getUsedFileSize() const { return _info.getUsedFileSize(); } - bool active() const { return _info.isActive(); } - bool ready() const { return _info.isReady(); } + uint32_t getChecksum() const noexcept { return _info.getChecksum(); } + uint32_t getDocumentCount() const noexcept { return _info.getDocumentCount(); } + uint32_t getTotalDocumentSize() const noexcept { return _info.getTotalDocumentSize(); } + uint32_t getMetaCount() const noexcept { return _info.getMetaCount(); } + uint32_t getUsedFileSize() const noexcept { return _info.getUsedFileSize(); } + bool active() const noexcept { return _info.isActive(); } + bool ready() const noexcept { return _info.isReady(); } - const api::BucketInfo& getBucketInfo() const { return _info; } + const api::BucketInfo& getBucketInfo() const noexcept { return _info; } - void setBucketInfo(uint64_t timestamp, const api::BucketInfo& bInfo) { + void setBucketInfo(uint64_t timestamp, const api::BucketInfo& bInfo) noexcept { _info = bInfo; _timestamp = timestamp; } - void setActive(bool setactive) { + void setActive(bool setactive) noexcept { _info.setActive(setactive); } bool consistentWith(const BucketCopy& other, - bool countInvalidAsConsistent = false) const + bool countInvalidAsConsistent = false) const noexcept { // If both are valid, check checksum and doc count. if (valid() && other.valid()) { @@ -94,7 +92,7 @@ public: std::string toString() const; - bool operator==(const BucketCopy& other) const { + bool operator==(const BucketCopy& other) const noexcept { return getBucketInfo() == other.getBucketInfo() && _flags == other._flags; diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp index 5be6f310c71..2d70ee8d3ba 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp @@ -166,17 +166,17 @@ namespace { uint64_t active; uint64_t ready; - Count() : docs(0), bytes(0), buckets(0), active(0), ready(0) {} + Count() noexcept : docs(0), bytes(0), buckets(0), active(0), ready(0) {} }; Count count; uint32_t lowestUsedBit; - MetricsUpdater() + MetricsUpdater() noexcept : count(), lowestUsedBit(58) {} void operator()(document::BucketId::Type bucketId, - const StorBucketDatabase::Entry& data) + const StorBucketDatabase::Entry& data) noexcept { document::BucketId bucket( document::BucketId::keyToBucketId(bucketId)); @@ -198,7 +198,7 @@ namespace { } }; - void add(const MetricsUpdater& rhs) { + void add(const MetricsUpdater& rhs) noexcept { auto& d = count; auto& s = rhs.count; d.buckets += s.buckets; diff --git a/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition_entry.h b/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition_entry.h index 124ee1bdf45..785419e78cf 100644 --- a/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition_entry.h +++ b/storage/src/vespa/storage/distributor/pending_bucket_space_db_transition_entry.h @@ -8,7 +8,7 @@ namespace storage::distributor::dbtransition { struct Entry { Entry(const document::BucketId& bid, - const BucketCopy& copy_) + const BucketCopy& copy_) noexcept : bucket_key(bid.toKey()), copy(copy_) {} diff --git a/storage/src/vespa/storage/distributor/pendingmessagetracker.h b/storage/src/vespa/storage/distributor/pendingmessagetracker.h index 51971a276b4..13d83157150 100644 --- a/storage/src/vespa/storage/distributor/pendingmessagetracker.h +++ b/storage/src/vespa/storage/distributor/pendingmessagetracker.h @@ -196,18 +196,6 @@ private: // to be present for that exact purpose. mutable std::mutex _lock; - /** - * Increment latency and operation count stats for the node the message - * was sent towards based on the registered send time and the current time. - * - * In the event that system time has moved backwards across sending a - * command and reciving its reply, the latency will not be recorded but - * the total number of messages will increase. - * - * _lock MUST be held upon invocation. - */ - void updateNodeStatsOnReply(const MessageEntry& entry); - void getStatusStartPage(std::ostream& out) const; void getStatusPerNode(std::ostream& out) const; void getStatusPerBucket(std::ostream& out) const; diff --git a/storage/src/vespa/storage/persistence/asynchandler.cpp b/storage/src/vespa/storage/persistence/asynchandler.cpp index b8ed6b8ec91..7f8da8e76e7 100644 --- a/storage/src/vespa/storage/persistence/asynchandler.cpp +++ b/storage/src/vespa/storage/persistence/asynchandler.cpp @@ -104,7 +104,7 @@ AsyncHandler::handleRunTask(RunTaskCommand& cmd, MessageTracker::UP tracker) con }); spi::Bucket bucket(cmd.getBucket()); auto onDone = std::make_unique<ResultTaskOperationDone>(_sequencedExecutor, cmd.getBucketId(), std::move(task)); - cmd.task().run(bucket, std::make_shared<vespalib::KeepAlive<decltype(onDone)>>(std::move(onDone))); + cmd.run(bucket, std::make_shared<vespalib::KeepAlive<decltype(onDone)>>(std::move(onDone))); return tracker; } diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index 67fa22ada03..c71a7fee424 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "filestorhandlerimpl.h" #include "filestormanager.h" +#include "filestorhandlerimpl.h" #include <vespa/storage/bucketdb/minimumusedbitstracker.h> #include <vespa/storage/common/bucketmessages.h> #include <vespa/storage/common/content_bucket_space_repo.h> @@ -76,9 +76,12 @@ FileStorManager(const config::ConfigUri & configUri, spi::PersistenceProvider& p _configFetcher(configUri.getContext()), _use_async_message_handling_on_schedule(false), _metrics(std::make_unique<FileStorMetrics>()), + _filestorHandler(), + _sequencedExecutor(), _closed(false), _lock(), - _host_info_reporter(_component.getStateUpdater()) + _host_info_reporter(_component.getStateUpdater()), + _resource_usage_listener_registration(provider.register_resource_usage_listener(_host_info_reporter)) { _configFetcher.subscribe(configUri.getConfigId(), this); _configFetcher.start(); @@ -809,6 +812,8 @@ FileStorManager::sendUp(const std::shared_ptr<api::StorageMessage>& msg) void FileStorManager::onClose() { LOG(debug, "Start closing"); + _bucketExecutorRegistration.reset(); + _resource_usage_listener_registration.reset(); // Avoid getting config during shutdown _configFetcher.close(); LOG(debug, "Closed _configFetcher."); @@ -977,7 +982,10 @@ FileStorManager::execute(const spi::Bucket &bucket, std::unique_ptr<spi::BucketT StorBucketDatabase::WrappedEntry entry(_component.getBucketDatabase(bucket.getBucketSpace()).get( bucket.getBucketId(), "FileStorManager::execute")); if (entry.exist()) { - _filestorHandler->schedule(std::make_shared<RunTaskCommand>(bucket, std::move(task))); + auto cmd = std::make_shared<RunTaskCommand>(bucket, std::move(task)); + if ( ! _filestorHandler->schedule(cmd) ) { + task = cmd->stealTask(); + } } return task; } diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h index ae298d70a29..6eaef45e9bd 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h @@ -77,6 +77,7 @@ class FileStorManager : public StorageLinkQueued, std::mutex _lock; std::unique_ptr<vespalib::IDestructorCallback> _bucketExecutorRegistration; ServiceLayerHostInfoReporter _host_info_reporter; + std::unique_ptr<vespalib::IDestructorCallback> _resource_usage_listener_registration; public: FileStorManager(const config::ConfigUri &, spi::PersistenceProvider&, diff --git a/storage/src/vespa/storage/persistence/filestorage/service_layer_host_info_reporter.cpp b/storage/src/vespa/storage/persistence/filestorage/service_layer_host_info_reporter.cpp index c78b24bee78..88cdcc2b42f 100644 --- a/storage/src/vespa/storage/persistence/filestorage/service_layer_host_info_reporter.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/service_layer_host_info_reporter.cpp @@ -1,8 +1,13 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "service_layer_host_info_reporter.h" +#include <vespa/persistence/spi/resource_usage.h> #include <vespa/storage/common/nodestateupdater.h> #include <cmath> +#include <sstream> + +#include <vespa/log/log.h> +LOG_SETUP(".persistence.filestor.service_layer_host_info_reporter"); namespace storage { @@ -33,13 +38,22 @@ void write_attribute_usage(vespalib::JsonStream& output, const vespalib::string output << End(); } -bool want_immediate_report(const spi::ResourceUsage& old_resource_usage, const spi::ResourceUsage& resource_usage) +bool want_immediate_report(const spi::ResourceUsage& old_usage, const spi::ResourceUsage& new_usage) { - auto disk_usage_diff = fabs(resource_usage.get_disk_usage() - old_resource_usage.get_disk_usage()); - auto memory_usage_diff = fabs(resource_usage.get_memory_usage() - old_resource_usage.get_memory_usage()); - auto attribute_enum_store_diff = fabs(resource_usage.get_attribute_enum_store_usage().get_usage() - old_resource_usage.get_attribute_enum_store_usage().get_usage()); - auto attribute_multivalue_diff = fabs(resource_usage.get_attribute_multivalue_usage().get_usage() - old_resource_usage.get_attribute_multivalue_usage().get_usage()); - return (disk_usage_diff > diff_slack || memory_usage_diff > diff_slack || attribute_enum_store_diff > diff_slack || attribute_multivalue_diff > diff_slack); + auto disk_usage_diff = fabs(new_usage.get_disk_usage() - old_usage.get_disk_usage()); + auto memory_usage_diff = fabs(new_usage.get_memory_usage() - old_usage.get_memory_usage()); + auto enum_store_diff = fabs(new_usage.get_attribute_enum_store_usage().get_usage() - old_usage.get_attribute_enum_store_usage().get_usage()); + auto multivalue_diff = fabs(new_usage.get_attribute_multivalue_usage().get_usage() - old_usage.get_attribute_multivalue_usage().get_usage()); + bool enum_store_got_valid = !old_usage.get_attribute_enum_store_usage().valid() && + new_usage.get_attribute_enum_store_usage().valid(); + bool multivalue_got_valid = !old_usage.get_attribute_multivalue_usage().valid() && + new_usage.get_attribute_multivalue_usage().valid(); + return ((disk_usage_diff > diff_slack) || + (memory_usage_diff > diff_slack) || + (enum_store_diff > diff_slack) || + (multivalue_diff > diff_slack) || + enum_store_got_valid || + multivalue_got_valid); } } @@ -58,10 +72,25 @@ ServiceLayerHostInfoReporter::~ServiceLayerHostInfoReporter() spi::ResourceUsageListener::reset(); // detach } +namespace { + +vespalib::string +to_string(const spi::ResourceUsage& usage) +{ + std::ostringstream oss; + oss << usage; + return oss.str(); +} + +} + void ServiceLayerHostInfoReporter::update_resource_usage(const spi::ResourceUsage& resource_usage) { bool immediate_report = want_immediate_report(_old_resource_usage, resource_usage); + LOG(debug, "update_resource_usage(): immediate_report=%s, old_usage=%s, new_usage=%s", + (immediate_report ? "true" : "false"), to_string(_old_resource_usage).c_str(), + to_string(resource_usage).c_str()); if (immediate_report) { _old_resource_usage = resource_usage; } @@ -82,6 +111,7 @@ ServiceLayerHostInfoReporter::report(vespalib::JsonStream& output) { std::lock_guard guard(_lock); auto& usage = get_usage(); + LOG(debug, "report(): usage=%s", to_string(usage).c_str()); write_usage(output, memory_label, usage.get_memory_usage()); write_usage(output, disk_label, usage.get_disk_usage()); write_attribute_usage(output, attribute_enum_store_label, usage.get_attribute_enum_store_usage()); diff --git a/storage/src/vespa/storage/persistence/filestorage/service_layer_host_info_reporter.h b/storage/src/vespa/storage/persistence/filestorage/service_layer_host_info_reporter.h index be0abc94987..b58e047d0af 100644 --- a/storage/src/vespa/storage/persistence/filestorage/service_layer_host_info_reporter.h +++ b/storage/src/vespa/storage/persistence/filestorage/service_layer_host_info_reporter.h @@ -25,7 +25,7 @@ public: ServiceLayerHostInfoReporter(const ServiceLayerHostInfoReporter&) = delete; ServiceLayerHostInfoReporter& operator=(const ServiceLayerHostInfoReporter&) = delete; - ~ServiceLayerHostInfoReporter(); + ~ServiceLayerHostInfoReporter() override; void report(vespalib::JsonStream& output) override; const spi::ResourceUsage &get_old_resource_usage() noexcept { return _old_resource_usage; } diff --git a/storage/src/vespa/storage/persistence/messages.cpp b/storage/src/vespa/storage/persistence/messages.cpp index 7ccb3ee895d..beae22a429b 100644 --- a/storage/src/vespa/storage/persistence/messages.cpp +++ b/storage/src/vespa/storage/persistence/messages.cpp @@ -187,9 +187,7 @@ RunTaskCommand::RunTaskCommand(const spi::Bucket &bucket, std::unique_ptr<spi::B : api::InternalCommand(ID), _task(std::move(task)), _bucket(bucket) -{ - assert(_task); -} +{ } RunTaskCommand::~RunTaskCommand() = default; @@ -203,6 +201,14 @@ RunTaskCommand::print(std::ostream& out, bool verbose, const std::string& indent } } +void +RunTaskCommand::run(const spi::Bucket & bucket, std::shared_ptr<vespalib::IDestructorCallback> onComplete) +{ + if (_task) { + _task->run(bucket, std::move(onComplete)); + } +} + RunTaskReply::RunTaskReply(const RunTaskCommand& cmd) : api::InternalReply(ID, cmd) {} diff --git a/storage/src/vespa/storage/persistence/messages.h b/storage/src/vespa/storage/persistence/messages.h index 043747d10d2..50834782f39 100644 --- a/storage/src/vespa/storage/persistence/messages.h +++ b/storage/src/vespa/storage/persistence/messages.h @@ -249,11 +249,10 @@ public: document::Bucket getBucket() const override { return _bucket.getBucket(); } std::unique_ptr<api::StorageReply> makeReply() override; - spi::BucketTask & task() & { - return *_task; - } + void run(const spi::Bucket & bucket, std::shared_ptr<vespalib::IDestructorCallback> onComplete); void print(std::ostream& out, bool verbose, const std::string& indent) const override; + std::unique_ptr<spi::BucketTask> stealTask() { return std::move(_task); } private: std::unique_ptr<spi::BucketTask> _task; spi::Bucket _bucket; diff --git a/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp b/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp index 9e4e48d67f4..4c3b290d4d7 100644 --- a/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp +++ b/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp @@ -19,8 +19,7 @@ BucketInfo::BucketInfo() noexcept _active(false) {} -BucketInfo::BucketInfo(uint32_t checksum, uint32_t docCount, - uint32_t totDocSize) noexcept +BucketInfo::BucketInfo(uint32_t checksum, uint32_t docCount, uint32_t totDocSize) noexcept : _lastModified(0), _checksum(checksum), _docCount(docCount), @@ -73,7 +72,7 @@ BucketInfo::BucketInfo(uint32_t checksum, uint32_t docCount, {} bool -BucketInfo::operator==(const BucketInfo& info) const +BucketInfo::operator==(const BucketInfo& info) const noexcept { return (_checksum == info._checksum && _docCount == info._docCount && diff --git a/storageapi/src/vespa/storageapi/buckets/bucketinfo.h b/storageapi/src/vespa/storageapi/buckets/bucketinfo.h index 5b2de4b4d61..7e6e7a2aed2 100644 --- a/storageapi/src/vespa/storageapi/buckets/bucketinfo.h +++ b/storageapi/src/vespa/storageapi/buckets/bucketinfo.h @@ -42,37 +42,37 @@ public: uint32_t metaCount, uint32_t usedFileSize, bool ready, bool active, Timestamp lastModified) noexcept; - Timestamp getLastModified() const { return _lastModified; } - uint32_t getChecksum() const { return _checksum; } - uint32_t getDocumentCount() const { return _docCount; } - uint32_t getTotalDocumentSize() const { return _totDocSize; } - uint32_t getMetaCount() const { return _metaCount; } - uint32_t getUsedFileSize() const { return _usedFileSize; } - bool isReady() const { return _ready; } - bool isActive() const { return _active; } + Timestamp getLastModified() const noexcept { return _lastModified; } + uint32_t getChecksum() const noexcept { return _checksum; } + uint32_t getDocumentCount() const noexcept { return _docCount; } + uint32_t getTotalDocumentSize() const noexcept { return _totDocSize; } + uint32_t getMetaCount() const noexcept { return _metaCount; } + uint32_t getUsedFileSize() const noexcept { return _usedFileSize; } + bool isReady() const noexcept { return _ready; } + bool isActive() const noexcept { return _active; } - void setChecksum(uint32_t crc) { _checksum = crc; } - void setDocumentCount(uint32_t count) { _docCount = count; } - void setTotalDocumentSize(uint32_t size) { _totDocSize = size; } - void setMetaCount(uint32_t count) { _metaCount = count; } - void setUsedFileSize(uint32_t size) { _usedFileSize = size; } - void setReady(bool ready = true) { _ready = ready; } - void setActive(bool active = true) { _active = active; } - void setLastModified(Timestamp lastModified) { _lastModified = lastModified; } + void setChecksum(uint32_t crc) noexcept { _checksum = crc; } + void setDocumentCount(uint32_t count) noexcept { _docCount = count; } + void setTotalDocumentSize(uint32_t size) noexcept { _totDocSize = size; } + void setMetaCount(uint32_t count) noexcept { _metaCount = count; } + void setUsedFileSize(uint32_t size) noexcept { _usedFileSize = size; } + void setReady(bool ready = true) noexcept { _ready = ready; } + void setActive(bool active = true) noexcept { _active = active; } + void setLastModified(Timestamp lastModified) noexcept { _lastModified = lastModified; } /** * Only compare checksum, total document count and document * size, not meta count or used file size. */ - bool equalDocumentInfo(const BucketInfo& other) const { + bool equalDocumentInfo(const BucketInfo& other) const noexcept { return (_checksum == other._checksum && _docCount == other._docCount && _totDocSize == other._totDocSize); } - bool operator==(const BucketInfo& info) const; - bool valid() const { return (_docCount > 0 || _totDocSize == 0); } - bool empty() const { + bool operator==(const BucketInfo& info) const noexcept; + bool valid() const noexcept { return (_docCount > 0 || _totDocSize == 0); } + bool empty() const noexcept { return _metaCount == 0 && _usedFileSize == 0 && _checksum == 0; } vespalib::string toString() const; diff --git a/vbench/src/apps/vbench/vbench.cpp b/vbench/src/apps/vbench/vbench.cpp index ab695f0b728..4b3419b2f50 100644 --- a/vbench/src/apps/vbench/vbench.cpp +++ b/vbench/src/apps/vbench/vbench.cpp @@ -45,8 +45,8 @@ int run(const std::string &cfg_name) { vespalib::RunnablePair runBoth(vbench, notify); vespalib::Thread thread(runBoth); thread.start(); - while (!SIG::INT.check() && !SIG::TERM.check() && !done.await(1000)) {} - if (!done.await(0)) { + while (!SIG::INT.check() && !SIG::TERM.check() && !done.await(1s)) {} + if (!done.await(vespalib::duration::zero())) { vbench.abort(); done.await(); } diff --git a/vbench/src/tests/time_queue/time_queue_test.cpp b/vbench/src/tests/time_queue/time_queue_test.cpp index 5ce4feab747..47c1afbfaa9 100644 --- a/vbench/src/tests/time_queue/time_queue_test.cpp +++ b/vbench/src/tests/time_queue/time_queue_test.cpp @@ -16,8 +16,8 @@ TEST_MT_FFF("time queue", 2, TimeQueue<int>(10.0, 5.0), vespalib::Gate(), vespal } else { double delay; std::vector<std::unique_ptr<int> > list; - EXPECT_TRUE(f2.await(20000)); - EXPECT_FALSE(f3.await(20)); + EXPECT_TRUE(f2.await(20s)); + EXPECT_FALSE(f3.await(20ms)); { f1.extract(1.5, list, delay); ASSERT_EQUAL(1u, list.size()); @@ -38,7 +38,7 @@ TEST_MT_FFF("time queue", 2, TimeQueue<int>(10.0, 5.0), vespalib::Gate(), vespal EXPECT_EQUAL(0u, list.size()); EXPECT_EQUAL(5.0, delay); } - EXPECT_TRUE(f3.await(20000)); + EXPECT_TRUE(f3.await(20s)); { f1.extract(99.25, list, delay); EXPECT_EQUAL(0u, list.size()); diff --git a/vespalib/src/tests/barrier/barrier_test.cpp b/vespalib/src/tests/barrier/barrier_test.cpp index f829b7fa5d8..2340476d4e0 100644 --- a/vespalib/src/tests/barrier/barrier_test.cpp +++ b/vespalib/src/tests/barrier/barrier_test.cpp @@ -13,9 +13,9 @@ struct Fixture { TEST_MT_F("require that barriers are satisfied by the appropriate number of threads", 3, Fixture(num_threads)) { if (thread_id == 0) { f1.latch.countDown(); - EXPECT_FALSE(f.latch.await(250)); + EXPECT_FALSE(f.latch.await(250ms)); EXPECT_TRUE(f.barrier.await()); - EXPECT_TRUE(f.latch.await(25000)); + EXPECT_TRUE(f.latch.await(25s)); } else { EXPECT_TRUE(f1.barrier.await()); f1.latch.countDown(); @@ -27,9 +27,9 @@ TEST_MT_F("require that barriers can be used multiple times", 3, Fixture(num_thr EXPECT_TRUE(f1.barrier.await()); if (thread_id == 0) { f1.latch.countDown(); - EXPECT_FALSE(f.latch.await(250)); + EXPECT_FALSE(f.latch.await(250ms)); EXPECT_TRUE(f.barrier.await()); - EXPECT_TRUE(f.latch.await(25000)); + EXPECT_TRUE(f.latch.await(25s)); } else { EXPECT_TRUE(f1.barrier.await()); f1.latch.countDown(); @@ -40,9 +40,9 @@ TEST_MT_F("require that barriers can be broken", 3, Fixture(num_threads)) { EXPECT_TRUE(f1.barrier.await()); if (thread_id == 0) { f1.latch.countDown(); - EXPECT_FALSE(f.latch.await(250)); + EXPECT_FALSE(f.latch.await(250ms)); f1.barrier.destroy(); - EXPECT_TRUE(f.latch.await(25000)); + EXPECT_TRUE(f.latch.await(25s)); } else { EXPECT_FALSE(f1.barrier.await()); f1.latch.countDown(); diff --git a/vespalib/src/tests/benchmark_timer/benchmark_timer_test.cpp b/vespalib/src/tests/benchmark_timer/benchmark_timer_test.cpp index 3f7187458d8..a1781abca81 100644 --- a/vespalib/src/tests/benchmark_timer/benchmark_timer_test.cpp +++ b/vespalib/src/tests/benchmark_timer/benchmark_timer_test.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/benchmark_timer.h> +#include <thread> using namespace vespalib; diff --git a/vespalib/src/tests/executor/blockingthreadstackexecutor_test.cpp b/vespalib/src/tests/executor/blockingthreadstackexecutor_test.cpp index 46db619516a..a0ef200ab83 100644 --- a/vespalib/src/tests/executor/blockingthreadstackexecutor_test.cpp +++ b/vespalib/src/tests/executor/blockingthreadstackexecutor_test.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> - +#include <vespa/vespalib/testkit/time_bomb.h> #include <vespa/vespalib/util/blockingthreadstackexecutor.h> #include <vespa/vespalib/util/executor.h> #include <vespa/vespalib/util/backtrace.h> @@ -8,7 +8,7 @@ using namespace vespalib; -constexpr int msWait = 30000; +constexpr vespalib::duration waitTime = 30s; class MyTask : public Executor::Task { @@ -21,8 +21,8 @@ public: : _entryGate(entryGate), _exitLatch(exitLatch) {} - virtual void run() override { - _entryGate.await(msWait); + void run() override { + _entryGate.await(waitTime); _exitLatch.countDown(); } static Task::UP create(Gate &entryGate, CountDownLatch &exitLatch) { @@ -64,14 +64,14 @@ struct Fixture workersEntryGate.countDown(); } void waitForWorkers() { - workersExitLatch.await(msWait); + workersExitLatch.await(waitTime); } void assertExecuteIsBlocked() { - blockedExecuteGate.await(10); + blockedExecuteGate.await(10ms); EXPECT_EQUAL(1u, blockedExecuteGate.getCount()); } void waitForExecuteIsFinished() { - blockedExecuteGate.await(msWait); + blockedExecuteGate.await(waitTime); EXPECT_EQUAL(0u, blockedExecuteGate.getCount()); } ThreadUP blockedExecuteThread() { diff --git a/vespalib/src/tests/executor/stress_test.cpp b/vespalib/src/tests/executor/stress_test.cpp index 615e7addfd5..01873fa0480 100644 --- a/vespalib/src/tests/executor/stress_test.cpp +++ b/vespalib/src/tests/executor/stress_test.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/util/executor.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/locale/c.h> +#include <thread> using namespace vespalib; using namespace std::literals; diff --git a/vespalib/src/tests/executor/threadstackexecutor_test.cpp b/vespalib/src/tests/executor/threadstackexecutor_test.cpp index b43a75f8244..e860a25c83c 100644 --- a/vespalib/src/tests/executor/threadstackexecutor_test.cpp +++ b/vespalib/src/tests/executor/threadstackexecutor_test.cpp @@ -147,9 +147,9 @@ TEST_MT_F("require that threads can wait for a specific task count", 7, WaitStat if (next_done < f1.block_task.size()) { f1.block_task[f1.block_task.size() - 1 - next_done].countDown(); } - EXPECT_TRUE(f1.wait_done[next_done].await(25000)); + EXPECT_TRUE(f1.wait_done[next_done].await(25s)); for (size_t i = 0; i < next_done; ++i) { - EXPECT_TRUE(!f1.wait_done[i].await(20)); + EXPECT_TRUE(!f1.wait_done[i].await(20ms)); } } } else { diff --git a/vespalib/src/tests/latch/latch_test.cpp b/vespalib/src/tests/latch/latch_test.cpp index f29b673e508..a0daf7ef7d5 100644 --- a/vespalib/src/tests/latch/latch_test.cpp +++ b/vespalib/src/tests/latch/latch_test.cpp @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/time_bomb.h> #include <vespa/vespalib/util/gate.h> #include <vespa/vespalib/util/latch.h> @@ -17,9 +18,9 @@ TEST("require that write then read works") { TEST_MT_FFF("require that read waits for write", 2, Latch<int>(), Gate(), TimeBomb(60)) { if (thread_id == 0) { - EXPECT_TRUE(!f2.await(10)); + EXPECT_TRUE(!f2.await(10ms)); f1.write(123); - EXPECT_TRUE(f2.await(60000)); + EXPECT_TRUE(f2.await(60s)); } else { EXPECT_EQUAL(f1.read(), 123); f2.countDown(); @@ -32,9 +33,9 @@ TEST_MT_FFF("require that write waits for read", 2, Latch<int>(), Gate(), TimeBo f1.write(456); f2.countDown(); } else { - EXPECT_TRUE(!f2.await(10)); + EXPECT_TRUE(!f2.await(10ms)); EXPECT_EQUAL(f1.read(), 123); - EXPECT_TRUE(f2.await(60000)); + EXPECT_TRUE(f2.await(60s)); EXPECT_EQUAL(f1.read(), 456); } } diff --git a/vespalib/src/tests/net/async_resolver/async_resolver_test.cpp b/vespalib/src/tests/net/async_resolver/async_resolver_test.cpp index f04fe549c09..8f3cad135ec 100644 --- a/vespalib/src/tests/net/async_resolver/async_resolver_test.cpp +++ b/vespalib/src/tests/net/async_resolver/async_resolver_test.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/time_bomb.h> #include <vespa/vespalib/net/async_resolver.h> #include <vespa/vespalib/net/socket_spec.h> #include <atomic> diff --git a/vespalib/src/tests/net/crypto_socket/crypto_socket_test.cpp b/vespalib/src/tests/net/crypto_socket/crypto_socket_test.cpp index e938e15f4e6..7230f97818f 100644 --- a/vespalib/src/tests/net/crypto_socket/crypto_socket_test.cpp +++ b/vespalib/src/tests/net/crypto_socket/crypto_socket_test.cpp @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/time_bomb.h> #include <vespa/vespalib/net/crypto_engine.h> #include <vespa/vespalib/net/tls/tls_crypto_engine.h> #include <vespa/vespalib/net/tls/maybe_tls_crypto_engine.h> diff --git a/vespalib/src/tests/net/selector/selector_test.cpp b/vespalib/src/tests/net/selector/selector_test.cpp index 5c91dfdc122..d13821ce2d0 100644 --- a/vespalib/src/tests/net/selector/selector_test.cpp +++ b/vespalib/src/tests/net/selector/selector_test.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/time_bomb.h> #include <vespa/vespalib/net/socket_address.h> #include <vespa/vespalib/net/selector.h> #include <vespa/vespalib/net/socket_utils.h> diff --git a/vespalib/src/tests/net/send_fd/send_fd_test.cpp b/vespalib/src/tests/net/send_fd/send_fd_test.cpp index e9921f75207..cc367996bd6 100644 --- a/vespalib/src/tests/net/send_fd/send_fd_test.cpp +++ b/vespalib/src/tests/net/send_fd/send_fd_test.cpp @@ -1,5 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/time_bomb.h> + #include <vespa/vespalib/net/selector.h> #include <vespa/vespalib/net/socket_spec.h> #include <vespa/vespalib/net/server_socket.h> diff --git a/vespalib/src/tests/net/socket/socket_test.cpp b/vespalib/src/tests/net/socket/socket_test.cpp index 08893c9273b..e3b13b69884 100644 --- a/vespalib/src/tests/net/socket/socket_test.cpp +++ b/vespalib/src/tests/net/socket/socket_test.cpp @@ -1,5 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/time_bomb.h> #include <vespa/vespalib/net/selector.h> #include <vespa/vespalib/net/socket_spec.h> #include <vespa/vespalib/net/server_socket.h> @@ -9,7 +10,6 @@ #include <vespa/vespalib/test/socket_options_verifier.h> #include <thread> #include <functional> -#include <chrono> #include <unistd.h> #include <sys/stat.h> diff --git a/vespalib/src/tests/net/sync_crypto_socket/sync_crypto_socket_test.cpp b/vespalib/src/tests/net/sync_crypto_socket/sync_crypto_socket_test.cpp index 56767051dad..8a4961b786b 100644 --- a/vespalib/src/tests/net/sync_crypto_socket/sync_crypto_socket_test.cpp +++ b/vespalib/src/tests/net/sync_crypto_socket/sync_crypto_socket_test.cpp @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/time_bomb.h> #include <vespa/vespalib/net/crypto_engine.h> #include <vespa/vespalib/net/tls/tls_crypto_engine.h> #include <vespa/vespalib/net/tls/maybe_tls_crypto_engine.h> diff --git a/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp b/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp index 5dc85bc567f..4631f19807e 100644 --- a/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp +++ b/vespalib/src/tests/net/tls/auto_reloading_tls_crypto_engine/auto_reloading_tls_crypto_engine_test.cpp @@ -7,9 +7,7 @@ #include <vespa/vespalib/net/tls/transport_security_options_reading.h> #include <vespa/vespalib/net/tls/impl/openssl_tls_context_impl.h> #include <vespa/vespalib/testkit/test_kit.h> - -#include <chrono> - +#include <vespa/vespalib/testkit/time_bomb.h> #include <openssl/ssl.h> using namespace vespalib; diff --git a/vespalib/src/tests/portal/handle_manager/handle_manager_test.cpp b/vespalib/src/tests/portal/handle_manager/handle_manager_test.cpp index 4ce25aa0a7a..844622577e2 100644 --- a/vespalib/src/tests/portal/handle_manager/handle_manager_test.cpp +++ b/vespalib/src/tests/portal/handle_manager/handle_manager_test.cpp @@ -1,11 +1,8 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/time_bomb.h> #include <vespa/vespalib/portal/handle_manager.h> -#include <vespa/vespalib/util/gate.h> - -#include <thread> -#include <chrono> #include <atomic> using namespace vespalib; @@ -85,9 +82,9 @@ TEST_MT_FF("require that destroy waits for active handle guards", 2, Fixture(), { auto guard = f1.manager.lock(f1.handle); TEST_BARRIER(); // #1 - EXPECT_TRUE(!f1.gate.await(20)); + EXPECT_TRUE(!f1.gate.await(20ms)); } - EXPECT_TRUE(f1.gate.await(60000)); + EXPECT_TRUE(f1.gate.await(60s)); } else { TEST_BARRIER(); // #1 EXPECT_TRUE(f1.manager.destroy(f1.handle)); diff --git a/vespalib/src/tests/portal/portal_test.cpp b/vespalib/src/tests/portal/portal_test.cpp index 07d3b9e1bfb..d708f81bdde 100644 --- a/vespalib/src/tests/portal/portal_test.cpp +++ b/vespalib/src/tests/portal/portal_test.cpp @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/time_bomb.h> #include <vespa/vespalib/portal/portal.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/stringfmt.h> @@ -305,7 +306,7 @@ TEST_MT_FF("require that GET requests can be completed in another thread", 2, if (thread_id == 0) { Portal::GetRequest req = f1.latch.read(); f1.exit_callback.countDown(); - std::this_thread::sleep_for(std::chrono::milliseconds(5)); + std::this_thread::sleep_for(5ms); req.respond_with_content("text/plain", "hello"); } else { auto result = fetch(f1.portal->listen_port(), null_crypto(), "/test"); @@ -318,9 +319,9 @@ TEST_MT_FFF("require that bind token destruction waits for active callbacks", 3, { if (thread_id == 0) { Portal::GetRequest req = f1.latch.read(); - EXPECT_TRUE(!f2.await(20)); + EXPECT_TRUE(!f2.await(20ms)); f1.exit_callback.countDown(); - EXPECT_TRUE(f2.await(60000)); + EXPECT_TRUE(f2.await(60s)); req.respond_with_content("application/json", "[1,2,3]"); } else if (thread_id == 1) { f1.enter_callback.await(); @@ -338,9 +339,9 @@ TEST_MT_FFF("require that portal destruction waits for request completion", 3, if (thread_id == 0) { Portal::GetRequest req = f1.latch.read(); f1.exit_callback.countDown(); - EXPECT_TRUE(!f2.await(20)); + EXPECT_TRUE(!f2.await(20ms)); req.respond_with_content("application/json", "[1,2,3]"); - EXPECT_TRUE(f2.await(60000)); + EXPECT_TRUE(f2.await(60s)); } else if (thread_id == 1) { f1.enter_callback.await(); f1.bound.reset(); diff --git a/vespalib/src/tests/portal/reactor/reactor_test.cpp b/vespalib/src/tests/portal/reactor/reactor_test.cpp index b0abc2c9e46..b14bb1dbd3f 100644 --- a/vespalib/src/tests/portal/reactor/reactor_test.cpp +++ b/vespalib/src/tests/portal/reactor/reactor_test.cpp @@ -1,14 +1,11 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/time_bomb.h> #include <vespa/vespalib/net/socket_handle.h> #include <vespa/vespalib/net/socket_utils.h> #include <vespa/vespalib/portal/reactor.h> #include <vespa/vespalib/util/gate.h> - -#include <thread> -#include <chrono> - #include <sys/types.h> #include <sys/socket.h> #include <fcntl.h> @@ -150,9 +147,9 @@ TEST_MT_FFFF("require that reactor token destruction waits for io event handling if (thread_id == 0) { f2.enter_callback.await(); TEST_BARRIER(); // #1 - EXPECT_TRUE(!f3.await(20)); + EXPECT_TRUE(!f3.await(20ms)); f2.exit_callback.countDown(); - EXPECT_TRUE(f3.await(60000)); + EXPECT_TRUE(f3.await(60s)); } else { TEST_BARRIER(); // #1 f2.token.reset(); diff --git a/vespalib/src/tests/rendezvous/rendezvous_test.cpp b/vespalib/src/tests/rendezvous/rendezvous_test.cpp index f4ec7870ad5..6a3e87563ed 100644 --- a/vespalib/src/tests/rendezvous/rendezvous_test.cpp +++ b/vespalib/src/tests/rendezvous/rendezvous_test.cpp @@ -149,13 +149,13 @@ TEST_MT_FFFF("require that mingle is not called until all threads are present", for (bool ext_id: {false, true}) { CountDownLatch &latch = ext_id ? f4 : f2; if (thread_id == 0) { - EXPECT_FALSE(latch.await(20)); + EXPECT_FALSE(latch.await(20ms)); if (ext_id) { EXPECT_EQUAL(3u, f3.rendezvous(thread_id, thread_id).first); } else { EXPECT_EQUAL(3u, f1.rendezvous(thread_id).first); } - EXPECT_TRUE(latch.await(25000)); + EXPECT_TRUE(latch.await(25s)); } else { if (ext_id) { EXPECT_EQUAL(3u, f3.rendezvous(thread_id, thread_id).first); @@ -183,7 +183,7 @@ TEST_MT_FF("require that rendezvous can be run with additional threads", 100, Ad EXPECT_EQUAL(4950u, f1.rendezvous(res.first).first); f2.countDown(); } - EXPECT_TRUE(f2.await(25000)); + EXPECT_TRUE(f2.await(25s)); } TEST_MT_FF("require that mingle can modify its own copy of input values", 10, Modify<false>(num_threads), Modify<true>(num_threads)) { diff --git a/vespalib/src/tests/simple_thread_bundle/simple_thread_bundle_test.cpp b/vespalib/src/tests/simple_thread_bundle/simple_thread_bundle_test.cpp index d3c562c583c..178a6c49fca 100644 --- a/vespalib/src/tests/simple_thread_bundle/simple_thread_bundle_test.cpp +++ b/vespalib/src/tests/simple_thread_bundle/simple_thread_bundle_test.cpp @@ -3,6 +3,7 @@ #include <vespa/vespalib/util/simple_thread_bundle.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/box.h> +#include <thread> using namespace vespalib; using namespace vespalib::fixed_thread_bundle; @@ -112,9 +113,9 @@ TEST_MT_FFF("require that bundle run waits for all targets", 2, SimpleThreadBund f2.check(Box<size_t>().add(1).add(1).add(1)); f3.done.countDown(); } else { - EXPECT_FALSE(f3.done.await(20)); + EXPECT_FALSE(f3.done.await(20ms)); f3.start.countDown(); - EXPECT_TRUE(f3.done.await(10000)); + EXPECT_TRUE(f3.done.await(10s)); } } diff --git a/vespalib/src/tests/simple_thread_bundle/threading_speed_test.cpp b/vespalib/src/tests/simple_thread_bundle/threading_speed_test.cpp index 71bae66ed2f..f1021ba3b09 100644 --- a/vespalib/src/tests/simple_thread_bundle/threading_speed_test.cpp +++ b/vespalib/src/tests/simple_thread_bundle/threading_speed_test.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/simple_thread_bundle.h> #include <vespa/vespalib/util/box.h> +#include <thread> using namespace vespalib; diff --git a/vespalib/src/tests/sync/sync_test.cpp b/vespalib/src/tests/sync/sync_test.cpp index 417a9779f9d..e58135d8723 100644 --- a/vespalib/src/tests/sync/sync_test.cpp +++ b/vespalib/src/tests/sync/sync_test.cpp @@ -37,12 +37,12 @@ TEST("test gate dropping below zero") { TEST("test gate non blocking await return correct states") { Gate gate; EXPECT_EQUAL(gate.getCount(), 1u); - EXPECT_EQUAL(gate.await(0), false); - EXPECT_EQUAL(gate.await(10), false); + EXPECT_EQUAL(gate.await(0ms), false); + EXPECT_EQUAL(gate.await(10ms), false); gate.countDown(); EXPECT_EQUAL(gate.getCount(), 0u); - EXPECT_EQUAL(gate.await(0), true); - EXPECT_EQUAL(gate.await(10), true); + EXPECT_EQUAL(gate.await(0ms), true); + EXPECT_EQUAL(gate.await(10ms), true); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/testkit-time_bomb/testkit-time_bomb_test.cpp b/vespalib/src/tests/testkit-time_bomb/testkit-time_bomb_test.cpp index af595db5ae7..5496fedb703 100644 --- a/vespalib/src/tests/testkit-time_bomb/testkit-time_bomb_test.cpp +++ b/vespalib/src/tests/testkit-time_bomb/testkit-time_bomb_test.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/testkit/time_bomb.h> TEST_MT_F("use time bomb in multi-threaded test", 4, vespalib::TimeBomb(60)) { EXPECT_TRUE(true); diff --git a/vespalib/src/tests/thread/thread_test.cpp b/vespalib/src/tests/thread/thread_test.cpp index bcd38190c7e..5820f329b4f 100644 --- a/vespalib/src/tests/thread/thread_test.cpp +++ b/vespalib/src/tests/thread/thread_test.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/thread.h> +#include <thread> using namespace vespalib; diff --git a/vespalib/src/tests/time/time_box_test.cpp b/vespalib/src/tests/time/time_box_test.cpp index 3c58b929487..32ac68e7254 100644 --- a/vespalib/src/tests/time/time_box_test.cpp +++ b/vespalib/src/tests/time/time_box_test.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/time/time_box.h> +#include <thread> TEST("require that long-lived timebox returns falling time left numbers") { vespalib::TimeBox box(3600); diff --git a/vespalib/src/tests/websocket/websocket_test.cpp b/vespalib/src/tests/websocket/websocket_test.cpp index db031b247b4..d9cd70c09d1 100644 --- a/vespalib/src/tests/websocket/websocket_test.cpp +++ b/vespalib/src/tests/websocket/websocket_test.cpp @@ -5,8 +5,7 @@ #include <vespa/vespalib/websocket/acceptor.h> #include <vespa/vespalib/websocket/key.h> #include <vespa/vespalib/websocket/buffer.h> -#include <thread> -#include <chrono> +#include <vespa/vespalib/util/gate.h> using namespace vespalib; using namespace vespalib::ws; @@ -103,7 +102,7 @@ TEST("require that an acceptor can accept connections asynchronously") { Receptor<Socket> server; Acceptor acceptor(0, server); Socket::UP client = SimpleSocket::connect(SocketSpec::from_port(acceptor.port())); - server.gate.await(60000); + server.gate.await(60s); ASSERT_TRUE(server.obj.get() != nullptr); ASSERT_TRUE(client.get() != nullptr); TEST_DO(verify_socket_io_async(*server.obj, *client)); diff --git a/vespalib/src/vespa/vespalib/gtest/gtest.h b/vespalib/src/vespa/vespalib/gtest/gtest.h index 87362687103..f4aaa670e76 100644 --- a/vespalib/src/vespa/vespalib/gtest/gtest.h +++ b/vespalib/src/vespa/vespalib/gtest/gtest.h @@ -26,3 +26,16 @@ main(int argc, char* argv[]) \ #else #define VESPA_GTEST_TYPED_TEST_SUITE TYPED_TEST_CASE #endif + +#define VESPA_EXPECT_EXCEPTION(TRY_BLOCK, EXCEPTION_TYPE, MESSAGE) \ + try { \ + TRY_BLOCK; \ + FAIL() << "exception '" << MESSAGE << "' not thrown at all!"; \ + } catch(EXCEPTION_TYPE& e) { \ + EXPECT_TRUE(contains(stringref(e.what()), stringref(MESSAGE))) << \ + " e.what(): " << e.what() << "\n"; \ + } catch(...) { \ + FAIL() << "wrong exception type thrown"; \ + throw; \ + } + diff --git a/vespalib/src/vespa/vespalib/testkit/test_kit.h b/vespalib/src/vespa/vespalib/testkit/test_kit.h index 17746c5b0fc..0587396ceda 100644 --- a/vespalib/src/vespa/vespalib/testkit/test_kit.h +++ b/vespalib/src/vespa/vespalib/testkit/test_kit.h @@ -9,5 +9,4 @@ #include "test_master.h" #include "test_hook.h" #include "test_state_guard.h" -#include "time_bomb.h" #include <vespa/vespalib/util/time.h> diff --git a/vespalib/src/vespa/vespalib/testkit/time_bomb.cpp b/vespalib/src/vespa/vespalib/testkit/time_bomb.cpp index cbefa285384..2d75fcdd18f 100644 --- a/vespalib/src/vespa/vespalib/testkit/time_bomb.cpp +++ b/vespalib/src/vespa/vespalib/testkit/time_bomb.cpp @@ -10,14 +10,14 @@ namespace { void bomb(Gate &gate, size_t seconds) { if (seconds > 5) { - if (gate.await((seconds - 5) * 1000)) { + if (gate.await(from_s(seconds - 5))) { return; } } size_t countdown = std::min(seconds, size_t(5)); while (countdown > 0) { fprintf(stderr, "...%zu...\n", countdown--); - if (gate.await(1000)) { + if (gate.await(1s)) { return; } } diff --git a/vespalib/src/vespa/vespalib/util/address_space.cpp b/vespalib/src/vespa/vespalib/util/address_space.cpp index 058c00e835e..113e4ba1478 100644 --- a/vespalib/src/vespa/vespalib/util/address_space.cpp +++ b/vespalib/src/vespa/vespalib/util/address_space.cpp @@ -16,7 +16,7 @@ AddressSpace::AddressSpace(size_t used_, size_t dead_, size_t limit_) std::ostream &operator << (std::ostream &out, const AddressSpace &rhs) { - return out << "used=" << rhs.used() << ", dead=" << rhs.dead() << ", limit=" << rhs.limit(); + return out << "{used=" << rhs.used() << ", dead=" << rhs.dead() << ", limit=" << rhs.limit() << "}"; } } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/util/count_down_latch.h b/vespalib/src/vespa/vespalib/util/count_down_latch.h index 420be8f7ab2..54b36b414d6 100644 --- a/vespalib/src/vespa/vespalib/util/count_down_latch.h +++ b/vespalib/src/vespa/vespalib/util/count_down_latch.h @@ -2,9 +2,9 @@ #pragma once +#include "time.h" #include <mutex> #include <condition_variable> -#include <chrono> namespace vespalib { @@ -71,10 +71,9 @@ public: * @param maxwait the maximum number of milliseconds to wait * @return true if the counter reached 0, false if we timed out **/ - bool await(int maxwait) { - auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(maxwait); + bool await(vespalib::duration maxwait) { std::unique_lock<std::mutex> guard(_lock); - return _cond.wait_until(guard, deadline, [this]() { return (_count == 0); }); + return _cond.wait_for(guard, maxwait, [this]() { return (_count == 0); }); } /** diff --git a/vespalib/src/vespa/vespalib/util/executor.h b/vespalib/src/vespa/vespalib/util/executor.h index 57ad28344b9..ce610f4e84c 100644 --- a/vespalib/src/vespa/vespalib/util/executor.h +++ b/vespalib/src/vespa/vespalib/util/executor.h @@ -20,7 +20,7 @@ public: struct Task { typedef std::unique_ptr<Task> UP; virtual void run() = 0; - virtual ~Task() {} + virtual ~Task() = default; }; enum class OptimizeFor {LATENCY, THROUGHPUT, ADAPTIVE}; @@ -41,7 +41,7 @@ public: * In case you have a lazy executor that naps inbetween. **/ virtual void wakeup() = 0; - virtual ~Executor() =default; + virtual ~Executor() = default; }; } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/util/growstrategy.h b/vespalib/src/vespa/vespalib/util/growstrategy.h index bb3b9196997..c7ada75f317 100644 --- a/vespalib/src/vespa/vespalib/util/growstrategy.h +++ b/vespalib/src/vespa/vespalib/util/growstrategy.h @@ -2,7 +2,7 @@ #pragma once -#include <cstdint> +#include <cstddef> namespace vespalib { diff --git a/vespamalloc/src/tests/allocfree/allocfree.cpp b/vespamalloc/src/tests/allocfree/allocfree.cpp index 86050d4aee9..4f79ba6bccb 100644 --- a/vespamalloc/src/tests/allocfree/allocfree.cpp +++ b/vespamalloc/src/tests/allocfree/allocfree.cpp @@ -2,6 +2,7 @@ #include "producerconsumer.h" #include <vespa/vespalib/testkit/testapp.h> #include <map> +#include <thread> #include <vespa/log/log.h> LOG_SETUP("allocfree_test"); diff --git a/vespamalloc/src/tests/allocfree/linklist.cpp b/vespamalloc/src/tests/allocfree/linklist.cpp index 74af380458a..af44ec3430b 100644 --- a/vespamalloc/src/tests/allocfree/linklist.cpp +++ b/vespamalloc/src/tests/allocfree/linklist.cpp @@ -3,6 +3,8 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespamalloc/malloc/allocchunk.h> #include <vespamalloc/util/callstack.h> +#include <thread> + #include <vespa/log/log.h> LOG_SETUP("linklist_test"); diff --git a/yolean/src/main/java/com/yahoo/yolean/chain/Chain.java b/yolean/src/main/java/com/yahoo/yolean/chain/Chain.java index e57e83c644f..4d638847ed9 100644 --- a/yolean/src/main/java/com/yahoo/yolean/chain/Chain.java +++ b/yolean/src/main/java/com/yahoo/yolean/chain/Chain.java @@ -3,10 +3,8 @@ package com.yahoo.yolean.chain; import com.google.common.collect.ImmutableList; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.Iterator; import java.util.List; diff --git a/zookeeper-server/CMakeLists.txt b/zookeeper-server/CMakeLists.txt index 2fa0caeb5fb..401ac3ffd9d 100644 --- a/zookeeper-server/CMakeLists.txt +++ b/zookeeper-server/CMakeLists.txt @@ -1,4 +1,3 @@ # Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. add_subdirectory(zookeeper-server-common) -add_subdirectory(zookeeper-server-3.5.6) add_subdirectory(zookeeper-server-3.6.2) diff --git a/zookeeper-server/pom.xml b/zookeeper-server/pom.xml index 9a3425230a5..56b66903098 100644 --- a/zookeeper-server/pom.xml +++ b/zookeeper-server/pom.xml @@ -13,7 +13,6 @@ <version>7-SNAPSHOT</version> <modules> <module>zookeeper-server-common</module> - <module>zookeeper-server-3.5.6</module> <module>zookeeper-server-3.6.2</module> </modules> <dependencies> diff --git a/zookeeper-server/zookeeper-server-3.5.6/CMakeLists.txt b/zookeeper-server/zookeeper-server-3.5.6/CMakeLists.txt deleted file mode 100644 index e5b439c9f4e..00000000000 --- a/zookeeper-server/zookeeper-server-3.5.6/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(zookeeper-server-3.5.6) diff --git a/zookeeper-server/zookeeper-server-3.5.6/pom.xml b/zookeeper-server/zookeeper-server-3.5.6/pom.xml deleted file mode 100644 index 060d1df7b2e..00000000000 --- a/zookeeper-server/zookeeper-server-3.5.6/pom.xml +++ /dev/null @@ -1,78 +0,0 @@ -<?xml version="1.0"?> -<!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>com.yahoo.vespa</groupId> - <artifactId>zookeeper-server</artifactId> - <version>7-SNAPSHOT</version> - <relativePath>../pom.xml</relativePath> - </parent> - <artifactId>zookeeper-server-3.5.6</artifactId> - <packaging>container-plugin</packaging> - <version>7-SNAPSHOT</version> - <dependencies> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>zookeeper-server-common</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> - <version>3.5.6</version> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-jdk14</artifactId> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - <version>1.7.5</version> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - </dependencies> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <compilerArgs> - <arg>-Xlint:all</arg> - <arg>-Werror</arg> - </compilerArgs> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> - <forkMode>once</forkMode> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-install-plugin</artifactId> - <configuration> - <updateReleaseInfo>true</updateReleaseInfo> - </configuration> - </plugin> - <plugin> - <groupId>com.yahoo.vespa</groupId> - <artifactId>bundle-plugin</artifactId> - <extensions>true</extensions> - <configuration> - <importPackage>com.sun.management</importPackage> - <bundleSymbolicName>zookeeper-server</bundleSymbolicName> - </configuration> - </plugin> - </plugins> - </build> -</project> diff --git a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java deleted file mode 100644 index d614aecbad2..00000000000 --- a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import com.google.inject.Inject; -import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.component.AbstractComponent; - -import java.nio.file.Path; - -/** - * Starts or reconfigures zookeeper cluster - * - * @author hmusum - */ -public class ReconfigurableVespaZooKeeperServer extends AbstractComponent implements VespaZooKeeperServer { - - private final VespaQuorumPeer peer; - - @Inject - public ReconfigurableVespaZooKeeperServer(Reconfigurer reconfigurer, ZookeeperServerConfig zookeeperServerConfig) { - this.peer = new VespaQuorumPeer(); - reconfigurer.startOrReconfigure(zookeeperServerConfig, this); - } - - @Override - public void shutdown() { - peer.shutdown(); - } - - public void start(Path configFilePath) { - peer.start(configFilePath); - } - - @Override - public boolean reconfigurable() { - return true; - } - -} diff --git a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java deleted file mode 100644 index 86325f587eb..00000000000 --- a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import com.yahoo.protect.Process; -import org.apache.zookeeper.server.admin.AdminServer; -import org.apache.zookeeper.server.quorum.QuorumPeerConfig; -import org.apache.zookeeper.server.quorum.QuorumPeerMain; - -import java.io.IOException; -import java.nio.file.Path; - -/** - * Starts/stops a ZooKeeper server. Extends QuorumPeerMain to be able to call initializeAndRun() and wraps - * exceptions so it can be used by code that does not depend on ZooKeeper. - * - * @author hmusum - */ -class VespaQuorumPeer extends QuorumPeerMain { - - public void start(Path path) { - initializeAndRun(new String[]{ path.toFile().getAbsolutePath()}); - } - - public void shutdown() { - if (quorumPeer != null) { - try { - quorumPeer.shutdown(); - quorumPeer.join(); // Wait for shutdown to complete - } catch (RuntimeException|InterruptedException e) { - // If shutdown fails, we have no other option than forcing the JVM to stop and letting it be restarted. - // - // When a VespaZooKeeperServer component receives a new config, the container will try to start a new - // server with the new config, this will fail until the old server is deconstructed. If the old server - // fails to deconstruct/shut down, the new one will never start and if that happens forcing a restart is - // the better option. - Process.logAndDie("Failed to shut down ZooKeeper properly, forcing shutdown", e); - } - } - } - - @Override - protected void initializeAndRun(String[] args) { - try { - super.initializeAndRun(args); - } catch (QuorumPeerConfig.ConfigException | IOException | AdminServer.AdminServerException e) { - throw new RuntimeException("Exception when initializing or running ZooKeeper server", e); - } - } - -} diff --git a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java deleted file mode 100644 index 57ae62c0ebc..00000000000 --- a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.admin.ZooKeeperAdmin; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * @author hmusum - */ -@SuppressWarnings("unused") // Created by injection -public class VespaZooKeeperAdminImpl implements VespaZooKeeperAdmin { - - private static final Logger log = java.util.logging.Logger.getLogger(VespaZooKeeperAdminImpl.class.getName()); - - @Override - public void reconfigure(String connectionSpec, String joiningServers, String leavingServers) throws ReconfigException { - try { - ZooKeeperAdmin zooKeeperAdmin = new ZooKeeperAdmin(connectionSpec, - (int) sessionTimeout().toMillis(), - (event) -> log.log(Level.INFO, event.toString())); - long fromConfig = -1; - // Using string parameters because the List variant of reconfigure fails to join empty lists (observed on 3.5.6, fixed in 3.7.0) - byte[] appliedConfig = zooKeeperAdmin.reconfigure(joiningServers, leavingServers, null, fromConfig, null); - log.log(Level.INFO, "Applied ZooKeeper config: " + new String(appliedConfig, StandardCharsets.UTF_8)); - } catch (KeeperException e) { - if (retryOn(e)) - throw new ReconfigException(e); - else - throw new RuntimeException(e); - } catch (IOException | InterruptedException e) { - throw new RuntimeException(e); - } - } - - private static boolean retryOn(KeeperException e) { - return e instanceof KeeperException.ReconfigInProgress || - e instanceof KeeperException.ConnectionLossException || - e instanceof KeeperException.NewConfigNoQuorum; - } - -} - diff --git a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java deleted file mode 100644 index 4dfcbeea444..00000000000 --- a/zookeeper-server/zookeeper-server-3.5.6/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.zookeeper; - -import com.google.inject.Inject; -import com.yahoo.cloud.config.ZookeeperServerConfig; -import com.yahoo.component.AbstractComponent; - -import java.nio.file.Path; - -/** - * @author Ulf Lilleengen - * @author hmusum - */ -public class VespaZooKeeperServerImpl extends AbstractComponent implements VespaZooKeeperServer { - - private final VespaQuorumPeer peer; - private final ZooKeeperRunner runner; - - @Inject - public VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig) { - this.peer = new VespaQuorumPeer(); - this.runner = new ZooKeeperRunner(zookeeperServerConfig, this); - } - - @Override - public void deconstruct() { - runner.shutdown(); - super.deconstruct(); - } - - @Override - public void shutdown() { - peer.shutdown(); - } - - @Override - public void start(Path configFilePath) { - peer.start(configFilePath); - } - - @Override - public boolean reconfigurable() { - return false; - } - -} |