diff options
97 files changed, 1606 insertions, 374 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index d3621de617a..72df4cf524b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -90,6 +90,7 @@ add_subdirectory(flags) add_subdirectory(fnet) add_subdirectory(fsa) add_subdirectory(hosted-zone-api) +add_subdirectory(jdisc-cloud-aws) add_subdirectory(jdisc_core) add_subdirectory(jdisc-security-filters) add_subdirectory(jdisc_http_service) diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java index ad65435c770..9f439cbd992 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurer.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.config.content.StorDistributionConfig; import com.yahoo.cloud.config.ZookeepersConfig; import java.time.Duration; +import java.util.Map; /** * When the cluster controller is reconfigured, a new instance of this is created, which will propagate configured @@ -75,6 +76,8 @@ public class ClusterControllerClusterConfigurer { options.clusterHasGlobalDocumentTypes = config.cluster_has_global_document_types(); options.minMergeCompletionRatio = config.min_merge_completion_ratio(); options.enableTwoPhaseClusterStateActivation = config.enable_two_phase_cluster_state_transitions(); + options.clusterFeedBlockEnabled = config.enable_cluster_feed_block(); + options.clusterFeedBlockLimit = Map.copyOf(config.cluster_feed_block_limit()); } private void configure(SlobroksConfig config) { diff --git a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurerTest.java b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurerTest.java index 37131349602..9d2d7610469 100644 --- a/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurerTest.java +++ b/clustercontroller-apps/src/test/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/ClusterControllerClusterConfigurerTest.java @@ -31,7 +31,10 @@ public class ClusterControllerClusterConfigurerTest { .cluster_name("storage") .index(0) .zookeeper_server("zoo") - .min_node_ratio_per_group(0.123); + .min_node_ratio_per_group(0.123) + .enable_cluster_feed_block(true) + .cluster_feed_block_limit("foo", 0.5) + .cluster_feed_block_limit("bar", 0.7); SlobroksConfig.Builder slobroksConfig = new SlobroksConfig.Builder(); SlobroksConfig.Slobrok.Builder slobrok = new SlobroksConfig.Slobrok.Builder(); slobrok.connectionspec("foo"); @@ -57,6 +60,9 @@ public class ClusterControllerClusterConfigurerTest { ); assertTrue(configurer.getOptions() != null); assertEquals(0.123, configurer.getOptions().minNodeRatioPerGroup, 0.01); + assertTrue(configurer.getOptions().clusterFeedBlockEnabled); + assertEquals(0.5, configurer.getOptions().clusterFeedBlockLimit.get("foo"), 0.01); + assertEquals(0.7, configurer.getOptions().clusterFeedBlockLimit.get("bar"), 0.01); try{ zookeepersConfig.zookeeperserverlist(""); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundle.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundle.java index 2f9f67a4b6b..0ca4f5632a8 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundle.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundle.java @@ -243,6 +243,10 @@ public class ClusterStateBundle { return Optional.ofNullable(feedBlock); } + public FeedBlock getFeedBlockOrNull() { + return feedBlock; + } + public boolean clusterFeedIsBlocked() { return (feedBlock != null && feedBlock.blockFeedInCluster()); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java index 2f065a9ba75..6e8bfbd4a0c 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculator.java @@ -62,6 +62,8 @@ public class EventDiffCalculator { final Optional<String> bucketSpace; final AnnotatedClusterState fromState; final AnnotatedClusterState toState; + final ClusterStateBundle.FeedBlock feedBlockFrom; + final ClusterStateBundle.FeedBlock feedBlockTo; final long currentTime; final long maxMaintenanceGracePeriodTimeMs; @@ -69,12 +71,16 @@ public class EventDiffCalculator { Optional<String> bucketSpace, AnnotatedClusterState fromState, AnnotatedClusterState toState, + ClusterStateBundle.FeedBlock feedBlockFrom, + ClusterStateBundle.FeedBlock feedBlockTo, long currentTime, long maxMaintenanceGracePeriodTimeMs) { this.cluster = cluster; this.bucketSpace = bucketSpace; this.fromState = fromState; this.toState = toState; + this.feedBlockFrom = feedBlockFrom; + this.feedBlockTo = feedBlockTo; this.currentTime = currentTime; this.maxMaintenanceGracePeriodTimeMs = maxMaintenanceGracePeriodTimeMs; } @@ -94,6 +100,8 @@ public class EventDiffCalculator { Optional.empty(), params.fromState.getBaselineAnnotatedState(), params.toState.getBaselineAnnotatedState(), + params.fromState.getFeedBlockOrNull(), + params.toState.getFeedBlockOrNull(), params.currentTime, params.maxMaintenanceGracePeriodTimeMs); } @@ -117,6 +125,19 @@ public class EventDiffCalculator { events.add(createClusterEvent("Cluster is down", params)); } } + // TODO should we emit any events when description changes? + if (feedBlockStateHasChanged(params)) { + if (params.feedBlockTo != null) { + events.add(createClusterEvent(String.format("Cluster feed blocked due to resource exhaustion: %s", + params.feedBlockTo.getDescription()), params)); + } else { + events.add(createClusterEvent("Cluster feed no longer blocked", params)); + } + } + } + + private static boolean feedBlockStateHasChanged(PerStateParams params) { + return ((params.feedBlockFrom == null) != (params.feedBlockTo == null)); } private static ClusterEvent createClusterEvent(String description, PerStateParams params) { @@ -228,6 +249,8 @@ public class EventDiffCalculator { Optional.of(bucketSpace), fromDerivedState, toDerivedState, + null, // Not used in per-space event derivation + null, // Ditto params.currentTime, params.maxMaintenanceGracePeriodTimeMs); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index d29b2387db3..6dc4b1e8015 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -74,6 +74,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd private boolean processingCycle = false; private boolean wantedStateChanged = false; private long cycleCount = 0; + private long lastMetricUpdateCycleCount = 0; private long nextStateSendTime = 0; private Long controllerThreadId = null; @@ -336,9 +337,27 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd @Override public void handleUpdatedHostInfo(NodeInfo nodeInfo, HostInfo newHostInfo) { verifyInControllerThread(); + triggerBundleRecomputationIfResourceExhaustionStateChanged(nodeInfo, newHostInfo); stateVersionTracker.handleUpdatedHostInfo(nodeInfo, newHostInfo); } + private void triggerBundleRecomputationIfResourceExhaustionStateChanged(NodeInfo nodeInfo, HostInfo newHostInfo) { + if (!options.clusterFeedBlockEnabled) { + return; + } + // TODO hysteresis to prevent oscillations! + // TODO also ensure we trigger if CC options have changed + var calc = createResourceExhaustionCalculator(); + // Important: nodeInfo contains the _current_ host info _prior_ to newHostInfo being applied. + boolean previouslyExhausted = !calc.enumerateNodeResourceExhaustions(nodeInfo).isEmpty(); + boolean nowExhausted = !calc.resourceExhaustionsFromHostInfo(nodeInfo.getNode(), newHostInfo).isEmpty(); + if (previouslyExhausted != nowExhausted) { + log.fine(() -> String.format("Triggering state recomputation due to change in cluster feed block: %s -> %s", + previouslyExhausted, nowExhausted)); + stateChangeHandler.setStateChangedFlag(); + } + } + @Override public void handleNewNode(NodeInfo node) { verifyInControllerThread(); @@ -366,6 +385,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd ClusterState baselineState = stateBundle.getBaselineClusterState(); newStates.add(stateBundle); metricUpdater.updateClusterStateMetrics(cluster, baselineState); + lastMetricUpdateCycleCount = cycleCount; systemStateBroadcaster.handleNewClusterStates(stateBundle); // Iff master, always store new version in ZooKeeper _before_ publishing to any // nodes so that a cluster controller crash after publishing but before a successful @@ -375,6 +395,19 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd } } + private boolean maybePublishOldMetrics() { + verifyInControllerThread(); + if (cycleCount > 300 + lastMetricUpdateCycleCount) { + ClusterStateBundle stateBundle = stateVersionTracker.getVersionedClusterStateBundle(); + ClusterState baselineState = stateBundle.getBaselineClusterState(); + metricUpdater.updateClusterStateMetrics(cluster, baselineState); + lastMetricUpdateCycleCount = cycleCount; + return true; + } else { + return false; + } + } + private void storeClusterStateMetaDataToZooKeeper(ClusterStateBundle stateBundle) { try { database.saveLatestSystemStateVersion(databaseContext, stateBundle.getVersion()); @@ -587,6 +620,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd if ( ! isRunning()) { return; } didWork |= processNextQueuedRemoteTask(); didWork |= completeSatisfiedVersionDependentTasks(); + didWork |= maybePublishOldMetrics(); processingCycle = false; ++cycleCount; @@ -877,6 +911,8 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd .bucketSpaces(configuredBucketSpaces) .stateDeriver(createBucketSpaceStateDeriver()) .deferredActivation(options.enableTwoPhaseClusterStateActivation) + .feedBlock(createResourceExhaustionCalculator() + .inferContentClusterFeedBlockOrNull(cluster.getNodeInfo())) .deriveAndBuild(); stateVersionTracker.updateLatestCandidateStateBundle(candidateBundle); invokeCandidateStateListeners(candidateBundle); @@ -915,6 +951,10 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd } } + private ResourceExhaustionCalculator createResourceExhaustionCalculator() { + return new ResourceExhaustionCalculator(options.clusterFeedBlockEnabled, options.clusterFeedBlockLimit); + } + private static ClusterStateDeriver createIdentityClonedBucketSpaceStateDeriver() { return (state, space) -> state.clone(); } @@ -1010,6 +1050,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd ClusterStateBundle previousBundle = database.getLatestClusterStateBundle(); database.loadStartTimestamps(cluster); database.loadWantedStates(databaseContext); + // TODO determine if we need any specialized handling here if feed block is set in the loaded bundle log.info(() -> String.format("Loaded previous cluster state bundle from ZooKeeper: %s", previousBundle)); stateVersionTracker.setClusterStateBundleRetrievedFromZooKeeper(previousBundle); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java index 2044eb1eab0..a088b50f078 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java @@ -10,6 +10,7 @@ import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; import java.time.Duration; import java.util.Collection; +import java.util.Collections; import java.util.Locale; import java.util.Map; import java.util.Set; @@ -132,6 +133,10 @@ public class FleetControllerOptions implements Cloneable { public int maxDivergentNodesPrintedInTaskErrorMessages = 10; + public boolean clusterFeedBlockEnabled = false; + // Resource type -> limit in [0, 1] + public Map<String, Double> clusterFeedBlockLimit = Collections.emptyMap(); + // TODO: Replace usage of this by usage where the nodes are explicitly passed (below) public FleetControllerOptions(String clusterName) { this.clusterName = clusterName; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeResourceExhaustion.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeResourceExhaustion.java new file mode 100644 index 00000000000..609fea2b91e --- /dev/null +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeResourceExhaustion.java @@ -0,0 +1,42 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.clustercontroller.core; + +import com.yahoo.vdslib.state.Node; +import com.yahoo.vespa.clustercontroller.core.hostinfo.ResourceUsage; + +import java.util.Objects; + +/** + * Wrapper that identifies a resource type that has been exhausted on a given node, + * complete with both actual usage and the limit it exceeded. + */ +public class NodeResourceExhaustion { + public final Node node; + public final String resourceType; + public final ResourceUsage resourceUsage; + public final double limit; + + public NodeResourceExhaustion(Node node, String resourceType, + ResourceUsage resourceUsage, double limit) { + this.node = node; + this.resourceType = resourceType; + this.resourceUsage = resourceUsage; + this.limit = limit; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NodeResourceExhaustion that = (NodeResourceExhaustion) o; + return Double.compare(that.limit, limit) == 0 && + Objects.equals(node, that.node) && + Objects.equals(resourceType, that.resourceType) && + Objects.equals(resourceUsage, that.resourceUsage); + } + + @Override + public int hashCode() { + return Objects.hash(node, resourceType, resourceUsage, limit); + } +} diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java new file mode 100644 index 00000000000..80b8a6110f1 --- /dev/null +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculator.java @@ -0,0 +1,79 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.clustercontroller.core; + +import com.yahoo.vdslib.state.Node; +import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Given a mapping of (opaque) resource names and their exclusive limits, + * this class acts as an utility to easily enumerate all the resources that + * a given node (or set of nodes) have exhausted. + */ +public class ResourceExhaustionCalculator { + + private final boolean feedBlockEnabled; + private final Map<String, Double> feedBlockLimits; + + public ResourceExhaustionCalculator(boolean feedBlockEnabled, Map<String, Double> feedBlockLimits) { + this.feedBlockEnabled = feedBlockEnabled; + this.feedBlockLimits = feedBlockLimits; + } + + public ClusterStateBundle.FeedBlock inferContentClusterFeedBlockOrNull(Collection<NodeInfo> nodeInfos) { + if (!feedBlockEnabled) { + return null; + } + var exhaustions = enumerateNodeResourceExhaustionsAcrossAllNodes(nodeInfos); + if (exhaustions.isEmpty()) { + return null; + } + int maxDescriptions = 3; + String description = exhaustions.stream() + .limit(maxDescriptions) + .map(n -> String.format("%s on node %s (%.3g > %.3g)", + n.resourceType, n.node.getIndex(), + n.resourceUsage.getUsage(), n.limit)) + .collect(Collectors.joining(", ")); + if (exhaustions.size() > maxDescriptions) { + description += String.format(" (... and %d more)", exhaustions.size() - maxDescriptions); + } + return ClusterStateBundle.FeedBlock.blockedWithDescription(description); + } + + public List<NodeResourceExhaustion> resourceExhaustionsFromHostInfo(Node node, HostInfo hostInfo) { + List<NodeResourceExhaustion> exceedingLimit = null; + for (var usage : hostInfo.getContentNode().getResourceUsage().entrySet()) { + double limit = feedBlockLimits.getOrDefault(usage.getKey(), 1.0); + if (usage.getValue().getUsage() > limit) { + if (exceedingLimit == null) { + exceedingLimit = new ArrayList<>(); + } + exceedingLimit.add(new NodeResourceExhaustion(node, usage.getKey(), usage.getValue(), limit)); + } + } + return (exceedingLimit != null) ? exceedingLimit : Collections.emptyList(); + } + + public List<NodeResourceExhaustion> enumerateNodeResourceExhaustions(NodeInfo nodeInfo) { + if (!nodeInfo.isStorage()) { + return Collections.emptyList(); + } + return resourceExhaustionsFromHostInfo(nodeInfo.getNode(), nodeInfo.getHostInfo()); + } + + // Returns 0-n entries per content node in the cluster, where n is the number of exhausted + // resource types on any given node. + public List<NodeResourceExhaustion> enumerateNodeResourceExhaustionsAcrossAllNodes(Collection<NodeInfo> nodeInfos) { + return nodeInfos.stream() + .flatMap(info -> enumerateNodeResourceExhaustions(info).stream()) + .collect(Collectors.toList()); + } + +} diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/ContentNode.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/ContentNode.java new file mode 100644 index 00000000000..69c49ea2c1f --- /dev/null +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/ContentNode.java @@ -0,0 +1,25 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.clustercontroller.core.hostinfo; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +/** + * HostInfo information only returned by content nodes (i.e. search nodes) + */ +public class ContentNode { + @JsonProperty("resource-usage") + private Map<String, ResourceUsage> resourceUsage = new HashMap<>(); + + public Map<String, ResourceUsage> getResourceUsage() { + return Collections.unmodifiableMap(resourceUsage); + } + + public Optional<ResourceUsage> resourceUsageOf(String type) { + return Optional.ofNullable(resourceUsage.get(type)); + } +} diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfo.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfo.java index a6a3afcf6b2..71f61588c6c 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfo.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfo.java @@ -28,6 +28,7 @@ public class HostInfo { @JsonProperty("vtag") private Vtag vtag = new Vtag(null); @JsonProperty("distributor") private Distributor distributor = new Distributor(); @JsonProperty("metrics") private Metrics metrics = new Metrics(); + @JsonProperty("content-node") private ContentNode contentNode = new ContentNode(); public Vtag getVtag() { return vtag; @@ -37,6 +38,10 @@ public class HostInfo { return distributor; } + public ContentNode getContentNode() { + return contentNode; + } + public Metrics getMetrics() { return metrics; } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/ResourceUsage.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/ResourceUsage.java new file mode 100644 index 00000000000..e47ec5452a4 --- /dev/null +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/ResourceUsage.java @@ -0,0 +1,36 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.clustercontroller.core.hostinfo; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + +/** + * Encapsulation of the usage levels for a particular resource type. The resource type + * itself is not tracked in this class; this must be done on a higher level. + */ +public class ResourceUsage { + private final Double usage; + + public ResourceUsage(@JsonProperty("usage") Double usage) { + this.usage = usage; + } + + /** Resource usage in [0, 1] */ + public Double getUsage() { + return usage; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ResourceUsage that = (ResourceUsage) o; + return Objects.equals(usage, that.usage); + } + + @Override + public int hashCode() { + return Objects.hash(usage); + } +} diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java new file mode 100644 index 00000000000..2ac7113741b --- /dev/null +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFeedBlockTest.java @@ -0,0 +1,134 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.clustercontroller.core; + +import com.yahoo.jrt.Supervisor; +import com.yahoo.jrt.Transport; +import com.yahoo.vdslib.state.Node; +import com.yahoo.vdslib.state.NodeState; +import com.yahoo.vdslib.state.NodeType; +import com.yahoo.vdslib.state.State; +import com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler; +import com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabaseFactory; +import com.yahoo.vespa.clustercontroller.utils.util.NoMetricReporter; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.mapOf; +import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.usage; +import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.createResourceUsageJson; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class ClusterFeedBlockTest extends FleetControllerTest { + + private static final int NODE_COUNT = 3; + + // TODO dedupe fixture and setup stuff with other tests + private Supervisor supervisor; + private FleetController ctrl; + private DummyCommunicator communicator; + private EventLog eventLog; + private int dummyConfigGeneration = 2; + + @Before + public void setUp() { + supervisor = new Supervisor(new Transport()); + } + + private void initialize(FleetControllerOptions options) throws Exception { + List<Node> nodes = new ArrayList<>(); + for (int i = 0; i < options.nodes.size(); ++i) { + nodes.add(new Node(NodeType.STORAGE, i)); + nodes.add(new Node(NodeType.DISTRIBUTOR, i)); + } + + communicator = new DummyCommunicator(nodes, timer); + MetricUpdater metricUpdater = new MetricUpdater(new NoMetricReporter(), options.fleetControllerIndex); + eventLog = new EventLog(timer, metricUpdater); + ContentCluster cluster = new ContentCluster(options.clusterName, options.nodes, options.storageDistribution, + options.minStorageNodesUp, options.minRatioOfStorageNodesUp); + NodeStateGatherer stateGatherer = new NodeStateGatherer(timer, timer, eventLog); + DatabaseHandler database = new DatabaseHandler(new ZooKeeperDatabaseFactory(), timer, options.zooKeeperServerAddress, options.fleetControllerIndex, timer); + StateChangeHandler stateGenerator = new StateChangeHandler(timer, eventLog, metricUpdater); + SystemStateBroadcaster stateBroadcaster = new SystemStateBroadcaster(timer, timer); + MasterElectionHandler masterElectionHandler = new MasterElectionHandler(options.fleetControllerIndex, options.fleetControllerCount, timer, timer); + ctrl = new FleetController(timer, eventLog, cluster, stateGatherer, communicator, null, null, communicator, database, stateGenerator, stateBroadcaster, masterElectionHandler, metricUpdater, options); + + ctrl.tick(); + markAllNodesAsUp(options); + ctrl.tick(); + } + + private void markAllNodesAsUp(FleetControllerOptions options) throws Exception { + for (int i = 0; i < options.nodes.size(); ++i) { + communicator.setNodeState(new Node(NodeType.STORAGE, i), State.UP, ""); + communicator.setNodeState(new Node(NodeType.DISTRIBUTOR, i), State.UP, ""); + } + ctrl.tick(); + } + + public void tearDown() throws Exception { + if (supervisor != null) { + supervisor.transport().shutdown().join(); + supervisor = null; + } + super.tearDown(); + } + + private static FleetControllerOptions createOptions(Map<String, Double> feedBlockLimits) { + FleetControllerOptions options = defaultOptions("mycluster"); + options.setStorageDistribution(DistributionBuilder.forFlatCluster(NODE_COUNT)); + options.nodes = new HashSet<>(DistributionBuilder.buildConfiguredNodes(NODE_COUNT)); + options.clusterFeedBlockEnabled = true; + options.clusterFeedBlockLimit = Map.copyOf(feedBlockLimits); + return options; + } + + private void reportResourceUsageFromNode(int nodeIndex, Map<String, Double> resourceUsages) throws Exception { + String hostInfo = createResourceUsageJson(resourceUsages); + communicator.setNodeState(new Node(NodeType.STORAGE, nodeIndex), new NodeState(NodeType.STORAGE, State.UP), hostInfo); + ctrl.tick(); + } + + // TODO some form of hysteresis + @Test + public void cluster_feed_can_be_blocked_and_unblocked_by_single_node() throws Exception { + initialize(createOptions(mapOf(usage("cheese", 0.7), usage("wine", 0.4)))); + assertFalse(ctrl.getClusterStateBundle().clusterFeedIsBlocked()); + + // Too much cheese in use, must block feed! + reportResourceUsageFromNode(1, mapOf(usage("cheese", 0.8), usage("wine", 0.3))); + assertTrue(ctrl.getClusterStateBundle().clusterFeedIsBlocked()); + // TODO check desc? + + // Wine usage has gone up too, we should remain blocked + reportResourceUsageFromNode(1, mapOf(usage("cheese", 0.8), usage("wine", 0.5))); + assertTrue(ctrl.getClusterStateBundle().clusterFeedIsBlocked()); + // TODO check desc? + + // Back to normal wine and cheese levels + reportResourceUsageFromNode(1, mapOf(usage("cheese", 0.6), usage("wine", 0.3))); + assertFalse(ctrl.getClusterStateBundle().clusterFeedIsBlocked()); + } + + @Test + public void cluster_feed_block_state_is_recomputed_when_options_are_updated() throws Exception { + initialize(createOptions(mapOf(usage("cheese", 0.7), usage("wine", 0.4)))); + assertFalse(ctrl.getClusterStateBundle().clusterFeedIsBlocked()); + + reportResourceUsageFromNode(1, mapOf(usage("cheese", 0.8), usage("wine", 0.3))); + assertTrue(ctrl.getClusterStateBundle().clusterFeedIsBlocked()); + + // Increase cheese allowance. Should now automatically unblock since reported usage is lower. + ctrl.updateOptions(createOptions(mapOf(usage("cheese", 0.9), usage("wine", 0.4))), dummyConfigGeneration); + ctrl.tick(); // Options propagation + ctrl.tick(); // State recomputation + assertFalse(ctrl.getClusterStateBundle().clusterFeedIsBlocked()); + } + +} diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFixture.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFixture.java index 2df9279e450..a6cf10d4022 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFixture.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFixture.java @@ -219,11 +219,11 @@ public class ClusterFixture { return this.cluster; } - static Node storageNode(int index) { + public static Node storageNode(int index) { return new Node(NodeType.STORAGE, index); } - static Node distributorNode(int index) { + public static Node distributorNode(int index) { return new Node(NodeType.DISTRIBUTOR, index); } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java index ab8d73be99d..fe913e177ca 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/EventDiffCalculatorTest.java @@ -31,6 +31,8 @@ public class EventDiffCalculatorTest { AnnotatedClusterState.Builder baselineAfter = new AnnotatedClusterState.Builder(); Map<String, AnnotatedClusterState.Builder> derivedBefore = new HashMap<>(); Map<String, AnnotatedClusterState.Builder> derivedAfter = new HashMap<>(); + ClusterStateBundle.FeedBlock feedBlockBefore = null; + ClusterStateBundle.FeedBlock feedBlockAfter = null; long currentTimeMs = 0; long maxMaintenanceGracePeriodTimeMs = 10_000; @@ -86,6 +88,14 @@ public class EventDiffCalculatorTest { getBuilder(derivedAfter, bucketSpace).storageNodeReason(nodeIndex, reason); return this; } + EventFixture feedBlockBefore(ClusterStateBundle.FeedBlock feedBlock) { + this.feedBlockBefore = feedBlock; + return this; + } + EventFixture feedBlockAfter(ClusterStateBundle.FeedBlock feedBlock) { + this.feedBlockAfter = feedBlock; + return this; + } private static AnnotatedClusterState.Builder getBuilder(Map<String, AnnotatedClusterState.Builder> derivedStates, String bucketSpace) { return derivedStates.computeIfAbsent(bucketSpace, key -> new AnnotatedClusterState.Builder()); } @@ -94,8 +104,8 @@ public class EventDiffCalculatorTest { return EventDiffCalculator.computeEventDiff( EventDiffCalculator.params() .cluster(clusterFixture.cluster()) - .fromState(ClusterStateBundle.of(baselineBefore.build(), toDerivedStates(derivedBefore))) - .toState(ClusterStateBundle.of(baselineAfter.build(), toDerivedStates(derivedAfter))) + .fromState(ClusterStateBundle.of(baselineBefore.build(), toDerivedStates(derivedBefore), feedBlockBefore, false)) + .toState(ClusterStateBundle.of(baselineAfter.build(), toDerivedStates(derivedAfter), feedBlockAfter, false)) .currentTimeMs(currentTimeMs) .maxMaintenanceGracePeriodTimeMs(maxMaintenanceGracePeriodTimeMs)); } @@ -444,4 +454,43 @@ public class EventDiffCalculatorTest { nodeEventForBaseline()))); } + @Test + public void feed_block_engage_edge_emits_cluster_event() { + final EventFixture fixture = EventFixture.createForNodes(3) + .clusterStateBefore("distributor:3 storage:3") + .feedBlockBefore(null) + .clusterStateAfter("distributor:3 storage:3") + .feedBlockAfter(ClusterStateBundle.FeedBlock.blockedWithDescription("we're closed")); + + final List<Event> events = fixture.computeEventDiff(); + assertThat(events.size(), equalTo(1)); + assertThat(events, hasItem( + clusterEventWithDescription("Cluster feed blocked due to resource exhaustion: we're closed"))); + } + + @Test + public void feed_block_disengage_edge_emits_cluster_event() { + final EventFixture fixture = EventFixture.createForNodes(3) + .clusterStateBefore("distributor:3 storage:3") + .feedBlockBefore(ClusterStateBundle.FeedBlock.blockedWithDescription("we're closed")) + .clusterStateAfter("distributor:3 storage:3") + .feedBlockAfter(null); + + final List<Event> events = fixture.computeEventDiff(); + assertThat(events.size(), equalTo(1)); + assertThat(events, hasItem(clusterEventWithDescription("Cluster feed no longer blocked"))); + } + + @Test + public void feed_block_engaged_to_engaged_edge_does_not_emit_new_cluster_event() { + final EventFixture fixture = EventFixture.createForNodes(3) + .clusterStateBefore("distributor:3 storage:3") + .feedBlockBefore(ClusterStateBundle.FeedBlock.blockedWithDescription("we're closed")) + .clusterStateAfter("distributor:3 storage:3") + .feedBlockAfter(ClusterStateBundle.FeedBlock.blockedWithDescription("yep yep, still closed")); + + final List<Event> events = fixture.computeEventDiff(); + assertThat(events.size(), equalTo(0)); + } + } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FeedBlockUtil.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FeedBlockUtil.java new file mode 100644 index 00000000000..e2894705352 --- /dev/null +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FeedBlockUtil.java @@ -0,0 +1,49 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.clustercontroller.core; + +import java.util.Arrays; +import java.util.Map; +import java.util.stream.Collectors; + +public class FeedBlockUtil { + + static class NodeAndUsages { + public final int index; + public final Map<String, Double> usages; + + public NodeAndUsages(int index, Map<String, Double> usages) { + this.index = index; + this.usages = usages; + } + } + + static class NameAndUsage { + public final String name; + public final double usage; + + public NameAndUsage(String name, double usage) { + this.name = name; + this.usage = usage; + } + } + + static NameAndUsage usage(String name, double usage) { + return new NameAndUsage(name, usage); + } + + static Map<String, Double> mapOf(NameAndUsage... usages) { + return Arrays.stream(usages).collect(Collectors.toMap(u -> u.name, u -> u.usage)); + } + + static NodeAndUsages forNode(int index, NameAndUsage... usages) { + return new NodeAndUsages(index, mapOf(usages)); + } + + static String createResourceUsageJson(Map<String, Double> usages) { + String usageInnerJson = usages.entrySet().stream() + .map(kv -> String.format("\"%s\":{\"usage\": %.3g}", kv.getKey(), kv.getValue())) + .collect(Collectors.joining(",")); + return String.format("{\"content-node\":{\"resource-usage\":{%s}}}", usageInnerJson); + } + +} diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculatorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculatorTest.java new file mode 100644 index 00000000000..5a5cda1f4ed --- /dev/null +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculatorTest.java @@ -0,0 +1,94 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.clustercontroller.core; + +import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo; +import org.junit.Test; + +import java.util.Arrays; + +import static com.yahoo.vespa.clustercontroller.core.ClusterFixture.storageNode; +import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.NodeAndUsages; +import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.forNode; +import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.mapOf; +import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.usage; +import static com.yahoo.vespa.clustercontroller.core.FeedBlockUtil.createResourceUsageJson; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +public class ResourceExhaustionCalculatorTest { + + private static ClusterFixture createFixtureWithReportedUsages(NodeAndUsages... nodeAndUsages) { + var highestIndex = Arrays.stream(nodeAndUsages).mapToInt(u -> u.index).max(); + if (highestIndex.isEmpty()) { + throw new IllegalArgumentException("Can't have an empty cluster"); + } + var cf = ClusterFixture.forFlatCluster(highestIndex.getAsInt() + 1).bringEntireClusterUp(); + for (var nu : nodeAndUsages) { + cf.cluster().getNodeInfo(storageNode(nu.index)) + .setHostInfo(HostInfo.createHostInfo(createResourceUsageJson(nu.usages))); + } + return cf; + } + + @Test + public void no_feed_block_returned_when_no_resources_lower_than_limit() { + var calc = new ResourceExhaustionCalculator(true, mapOf(usage("disk", 0.5), usage("memory", 0.8))); + var cf = createFixtureWithReportedUsages(forNode(1, usage("disk", 0.49), usage("memory", 0.79)), + forNode(2, usage("disk", 0.4), usage("memory", 0.6))); + var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); + assertNull(feedBlock); + } + + @Test + public void feed_block_returned_when_single_resource_beyond_limit() { + var calc = new ResourceExhaustionCalculator(true, mapOf(usage("disk", 0.5), usage("memory", 0.8))); + var cf = createFixtureWithReportedUsages(forNode(1, usage("disk", 0.51), usage("memory", 0.79)), + forNode(2, usage("disk", 0.4), usage("memory", 0.6))); + var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); + assertNotNull(feedBlock); + assertTrue(feedBlock.blockFeedInCluster()); + assertEquals("disk on node 1 (0.510 > 0.500)", feedBlock.getDescription()); + } + + @Test + public void feed_block_returned_when_multiple_resources_beyond_limit() { + var calc = new ResourceExhaustionCalculator(true, mapOf(usage("disk", 0.4), usage("memory", 0.8))); + var cf = createFixtureWithReportedUsages(forNode(1, usage("disk", 0.51), usage("memory", 0.85)), + forNode(2, usage("disk", 0.45), usage("memory", 0.6))); + var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); + assertNotNull(feedBlock); + assertTrue(feedBlock.blockFeedInCluster()); + assertEquals("disk on node 1 (0.510 > 0.400), " + + "memory on node 1 (0.850 > 0.800), " + + "disk on node 2 (0.450 > 0.400)", + feedBlock.getDescription()); + } + + @Test + public void feed_block_description_is_bounded_in_number_of_described_resources() { + var calc = new ResourceExhaustionCalculator(true, mapOf(usage("disk", 0.4), usage("memory", 0.8))); + var cf = createFixtureWithReportedUsages(forNode(1, usage("disk", 0.51), usage("memory", 0.85)), + forNode(2, usage("disk", 0.45), usage("memory", 0.6)), + forNode(3, usage("disk", 0.6), usage("memory", 0.9))); + var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); + assertNotNull(feedBlock); + assertTrue(feedBlock.blockFeedInCluster()); + assertEquals("disk on node 1 (0.510 > 0.400), " + + "memory on node 1 (0.850 > 0.800), " + + "disk on node 2 (0.450 > 0.400) (... and 2 more)", + feedBlock.getDescription()); + } + + @Test + public void no_feed_block_returned_when_feed_block_disabled() { + var calc = new ResourceExhaustionCalculator(false, mapOf(usage("disk", 0.5), usage("memory", 0.8))); + var cf = createFixtureWithReportedUsages(forNode(1, usage("disk", 0.51), usage("memory", 0.79)), + forNode(2, usage("disk", 0.4), usage("memory", 0.6))); + var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); + assertNull(feedBlock); + } + +} diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfoTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfoTest.java index 01fa926e610..f9b0a4ca36f 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfoTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfoTest.java @@ -16,7 +16,9 @@ import java.util.TreeMap; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -35,6 +37,7 @@ public class HostInfoTest { HostInfo hostInfo = HostInfo.createHostInfo("{}"); assertThat(hostInfo.getVtag().getVersionOrNull(), is(nullValue())); assertThat(hostInfo.getDistributor().getStorageNodes().size(), is(0)); + assertThat(hostInfo.getContentNode().getResourceUsage().size(), is(0)); assertThat(hostInfo.getMetrics().getMetrics().size(), is(0)); assertThat(hostInfo.getClusterStateVersionOrNull(), is(nullValue())); } @@ -67,6 +70,12 @@ public class HostInfoTest { .getValueAt("vds.datastored.bucket_space.buckets_total", Map.of("bucketSpace", "global")) .map(Metrics.Value::getLast), equalTo(Optional.of(0L))); + + var resourceUsage = hostInfo.getContentNode().getResourceUsage(); + assertEquals(resourceUsage.size(), 2); + assertEquals(Optional.ofNullable(resourceUsage.get("memory")).map(ResourceUsage::getUsage).orElse(0.0), 0.85, 0.00001); + assertEquals(Optional.ofNullable(resourceUsage.get("disk")).map(ResourceUsage::getUsage).orElse(0.0), 0.6, 0.00001); + assertNull(resourceUsage.get("flux-capacitor")); } @Test diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java index 1cc5c93c28a..061ad42e028 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -4,22 +4,24 @@ package com.yahoo.config.model; import com.yahoo.cloud.config.ApplicationIdConfig; import com.yahoo.cloud.config.ClusterListConfig; import com.yahoo.cloud.config.ModelConfig; -import com.yahoo.cloud.config.SlobroksConfig; -import com.yahoo.cloud.config.ZookeepersConfig; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.component.Version; -import com.yahoo.vespa.config.content.LoadTypeConfig; import com.yahoo.cloud.config.ModelConfig.Hosts; import com.yahoo.cloud.config.ModelConfig.Hosts.Services; import com.yahoo.cloud.config.ModelConfig.Hosts.Services.Ports; +import com.yahoo.cloud.config.SlobroksConfig; +import com.yahoo.cloud.config.ZookeepersConfig; import com.yahoo.cloud.config.log.LogdConfig; +import com.yahoo.component.Version; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.document.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; import com.yahoo.messagebus.MessagebusConfig; import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; +import com.yahoo.vespa.config.content.DistributionConfig; +import com.yahoo.vespa.config.content.LoadTypeConfig; import com.yahoo.vespa.configmodel.producers.DocumentManager; import com.yahoo.vespa.configmodel.producers.DocumentTypes; import com.yahoo.vespa.documentmodel.DocumentModel; @@ -165,6 +167,13 @@ public class ApplicationConfigProducerRoot extends AbstractConfigProducer<Abstra } @Override + public void getConfig(DocumentProtocolPoliciesConfig.Builder builder) { + if (routing != null) { + routing.getConfig(builder); + } + } + + @Override public void getConfig(MessagebusConfig.Builder builder) { if (routing != null) { routing.getConfig(builder); @@ -213,6 +222,13 @@ public class ApplicationConfigProducerRoot extends AbstractConfigProducer<Abstra } @Override + public void getConfig(DistributionConfig.Builder builder) { + for (ContentCluster cluster : ((VespaModel) getRoot()).getContentClusters().values()) { + cluster.getConfig(builder); + } + } + + @Override public void getConfig(AllClustersBucketSpacesConfig.Builder builder) { VespaModel model = (VespaModel) getRoot(); for (ContentCluster cluster : model.getContentClusters().values()) { diff --git a/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java b/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java index 9dcd94bf455..312890aac13 100644 --- a/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/CommonConfigsProducer.java @@ -2,6 +2,8 @@ package com.yahoo.config.model; import com.yahoo.cloud.config.ApplicationIdConfig; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; +import com.yahoo.vespa.config.content.DistributionConfig; import com.yahoo.vespa.config.content.LoadTypeConfig; import com.yahoo.cloud.config.log.LogdConfig; import com.yahoo.cloud.config.SlobroksConfig; @@ -25,11 +27,13 @@ public interface CommonConfigsProducer extends DocumentmanagerConfig.Producer, DocumenttypesConfig.Producer, MessagebusConfig.Producer, DocumentrouteselectorpolicyConfig.Producer, + DocumentProtocolPoliciesConfig.Producer, LogdConfig.Producer, SlobroksConfig.Producer, ZookeepersConfig.Producer, LoadTypeConfig.Producer, ClusterListConfig.Producer, + DistributionConfig.Producer, AllClustersBucketSpacesConfig.Producer, ModelConfig.Producer, ApplicationIdConfig.Producer { diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java index d6cccc9f2be..0cfe35a51b5 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/SingleNodeProvisioner.java @@ -10,7 +10,6 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.net.HostName; -import com.yahoo.vespa.config.content.StorDistributionConfig; import java.util.ArrayList; import java.util.List; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index e75a9f2b125..04660f2b990 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -53,7 +53,7 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon /** The optional PRELOAD libraries for this Service. */ // Please keep non-null, as passed to command line in service startup - private String preload = Defaults.getDefaults().underVespaHome("lib64/vespa/malloc/libvespamalloc.so"); + private String preload = null; // If larger or equal to 0 it mean that explicit mmaps shall not be included in coredump. private long mmapNoCoreLimit = -1L; @@ -84,6 +84,10 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon private boolean initialized = false; + protected String defaultPreload() { + return Defaults.getDefaults().underVespaHome("lib64/vespa/malloc/libvespamalloc.so"); + } + /** * Preferred constructor when building from XML. Use this if you are building * in doBuild() in an AbstractConfigProducerBuilder. @@ -365,7 +369,9 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon setJvmOptions(args + getSeparator(jvmOptions) + jvmOptions); } } - public String getPreLoad() { return preload; } + public String getPreLoad() { + return preload != null ? preload : defaultPreload(); + } public void setPreLoad(String preload) { this.preload = preload; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index fd20bc1d12e..f7a4f2e52a2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -144,4 +144,8 @@ public class MetricsProxyContainer extends Container implements addSimpleComponent(componentClass.getName(), null, METRICS_PROXY_BUNDLE_NAME); } + @Override + protected String defaultPreload() { + return ""; + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index 3e2adeaacc9..a0b99516ce3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -97,6 +97,7 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC addPlatformBundle(METRICS_PROXY_BUNDLE_FILE); addClusterComponents(); + setJvmGCOptions(deployState.getProperties().jvmGCOptions()); } private void addClusterComponents() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 7ec2dc67cf2..a627e030156 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -15,6 +15,7 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.metrics.MetricsmanagerConfig; import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; +import com.yahoo.vespa.config.content.DistributionConfig; import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.config.content.MessagetyperouteselectorpolicyConfig; import com.yahoo.vespa.config.content.StorDistributionConfig; @@ -67,6 +68,8 @@ import java.util.TreeMap; import java.util.logging.Level; import java.util.stream.Collectors; +import static java.util.stream.Collectors.toList; + /** * A content cluster. * @@ -74,6 +77,7 @@ import java.util.stream.Collectors; * @author bratseth */ public class ContentCluster extends AbstractConfigProducer implements + DistributionConfig.Producer, StorDistributionConfig.Producer, StorDistributormanagerConfig.Producer, FleetcontrollerConfig.Producer, @@ -748,6 +752,39 @@ public class ContentCluster extends AbstractConfigProducer implements } } + @Override + public void getConfig(DistributionConfig.Builder builder) { + DistributionConfig.Cluster.Builder clusterBuilder = new DistributionConfig.Cluster.Builder(); + StorDistributionConfig.Builder storDistributionBuilder = new StorDistributionConfig.Builder(); + getConfig(storDistributionBuilder); + StorDistributionConfig config = storDistributionBuilder.build(); + + clusterBuilder.active_per_leaf_group(config.active_per_leaf_group()); + clusterBuilder.ready_copies(config.ready_copies()); + clusterBuilder.redundancy(config.redundancy()); + clusterBuilder.initial_redundancy(config.initial_redundancy()); + + for (StorDistributionConfig.Group group : config.group()) { + DistributionConfig.Cluster.Group.Builder groupBuilder = new DistributionConfig.Cluster.Group.Builder(); + groupBuilder.index(group.index()) + .name(group.name()) + .capacity(group.capacity()) + .partitions(group.partitions()); + + for (var node : group.nodes()) { + DistributionConfig.Cluster.Group.Nodes.Builder nodesBuilder = new DistributionConfig.Cluster.Group.Nodes.Builder(); + nodesBuilder.index(node.index()) + .retired(node.retired()); + + groupBuilder.nodes(nodesBuilder); + } + + clusterBuilder.group(groupBuilder); + } + + builder.cluster(getConfigId(), clusterBuilder); + } + /** * Mark whether the config emitted by this cluster currently should be applied by clients already running with * a previous generation of it only by restarting the consuming processes. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java index e6f4969f593..15e6c5993b3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java @@ -2,8 +2,9 @@ package com.yahoo.vespa.model.routing; import com.yahoo.config.model.ConfigModelRepo; -import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; import com.yahoo.document.select.DocumentSelector; +import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; import com.yahoo.messagebus.routing.ApplicationSpec; import com.yahoo.messagebus.routing.HopSpec; import com.yahoo.messagebus.routing.RouteSpec; @@ -12,9 +13,9 @@ import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.ContainerModel; import com.yahoo.vespa.model.container.docproc.ContainerDocproc; -import com.yahoo.vespa.model.content.cluster.ContentCluster; -import com.yahoo.vespa.model.content.Content; import com.yahoo.vespa.model.container.docproc.DocprocChain; +import com.yahoo.vespa.model.content.Content; +import com.yahoo.vespa.model.content.cluster.ContentCluster; import java.util.ArrayList; import java.util.Collection; @@ -31,7 +32,9 @@ import java.util.TreeMap; * * @author Simon Thoresen Hult */ -public final class DocumentProtocol implements Protocol, DocumentrouteselectorpolicyConfig.Producer { +public final class DocumentProtocol implements Protocol, + DocumentrouteselectorpolicyConfig.Producer, + DocumentProtocolPoliciesConfig.Producer { private static final String NAME = "document"; private final ApplicationSpec application; @@ -101,6 +104,42 @@ public final class DocumentProtocol implements Protocol, Documentrouteselectorpo } } + @Override + public void getConfig(DocumentProtocolPoliciesConfig.Builder builder) { + for (ContentCluster cluster : Content.getContentClusters(repo)) { + DocumentProtocolPoliciesConfig.Cluster.Builder clusterBuilder = new DocumentProtocolPoliciesConfig.Cluster.Builder(); + addSelector(cluster.getConfigId(), cluster.getRoutingSelector(), clusterBuilder); + if (cluster.getSearch().hasIndexedCluster()) + addRoutes(getDirectRouteName(cluster.getConfigId()), getIndexedRouteName(cluster.getConfigId()), clusterBuilder); + + builder.cluster(cluster.getConfigId(), clusterBuilder); + } + } + + private static void addRoutes(String directRoute, String indexedRoute, DocumentProtocolPoliciesConfig.Cluster.Builder builder) { + builder.defaultRoute(directRoute) + .route(new DocumentProtocolPoliciesConfig.Cluster.Route.Builder() + .messageType(com.yahoo.documentapi.messagebus.protocol.DocumentProtocol.MESSAGE_PUTDOCUMENT) + .name(indexedRoute)) + .route(new DocumentProtocolPoliciesConfig.Cluster.Route.Builder() + .messageType(com.yahoo.documentapi.messagebus.protocol.DocumentProtocol.MESSAGE_REMOVEDOCUMENT) + .name(indexedRoute)) + .route(new DocumentProtocolPoliciesConfig.Cluster.Route.Builder() + .messageType(com.yahoo.documentapi.messagebus.protocol.DocumentProtocol.MESSAGE_UPDATEDOCUMENT) + .name(indexedRoute)); + } + + private static void addSelector(String clusterConfigId, String selector, DocumentProtocolPoliciesConfig.Cluster.Builder builder) { + try { + new DocumentSelector(selector); + } catch (com.yahoo.document.select.parser.ParseException e) { + throw new IllegalArgumentException("Failed to parse selector '" + selector + + "' for route '" + clusterConfigId + + "' in policy 'DocumentRouteSelector'."); + } + builder.selector(selector); + } + private static void addRoute(String clusterConfigId, String selector, DocumentrouteselectorpolicyConfig.Builder builder) { try { new DocumentSelector(selector); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/routing/Routing.java b/config-model/src/main/java/com/yahoo/vespa/model/routing/Routing.java index 36beb766f5b..6637c84df10 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/routing/Routing.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/routing/Routing.java @@ -4,10 +4,20 @@ package com.yahoo.vespa.model.routing; import com.yahoo.config.model.ConfigModel; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.ConfigModelRepo; -import com.yahoo.messagebus.routing.*; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; import com.yahoo.messagebus.MessagebusConfig; import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; -import java.util.*; +import com.yahoo.messagebus.routing.ApplicationSpec; +import com.yahoo.messagebus.routing.HopSpec; +import com.yahoo.messagebus.routing.RouteSpec; +import com.yahoo.messagebus.routing.RoutingSpec; +import com.yahoo.messagebus.routing.RoutingTableSpec; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; /** * This is the routing plugin of the Vespa model. This class is responsible for parsing all routing information given @@ -82,6 +92,14 @@ public class Routing extends ConfigModel { } } + public void getConfig(DocumentProtocolPoliciesConfig.Builder builder) { + for (Protocol protocol : protocols) { + if (protocol instanceof DocumentProtocol) { + ((DocumentProtocol) protocol).getConfig(builder); + } + } + } + public void getConfig(DocumentrouteselectorpolicyConfig.Builder builder) { for (Protocol protocol : protocols) { if (protocol instanceof DocumentProtocol) { diff --git a/config-model/src/test/cfg/routing/defaultconfig/document-protocol-policies.cfg b/config-model/src/test/cfg/routing/defaultconfig/document-protocol-policies.cfg new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/config-model/src/test/cfg/routing/defaultconfig/document-protocol-policies.cfg @@ -0,0 +1 @@ + diff --git a/config-model/src/test/cfg/routing/hopconfig/document-protocol-policies.cfg b/config-model/src/test/cfg/routing/hopconfig/document-protocol-policies.cfg new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/config-model/src/test/cfg/routing/hopconfig/document-protocol-policies.cfg @@ -0,0 +1 @@ + diff --git a/config-model/src/test/cfg/routing/replacehop/document-protocol-policies.cfg b/config-model/src/test/cfg/routing/replacehop/document-protocol-policies.cfg new file mode 100644 index 00000000000..68659362350 --- /dev/null +++ b/config-model/src/test/cfg/routing/replacehop/document-protocol-policies.cfg @@ -0,0 +1,8 @@ +cluster{music}.defaultRoute "music-direct" +cluster{music}.route[0].name "music-index" +cluster{music}.route[0].messageType 100004 +cluster{music}.route[1].name "music-index" +cluster{music}.route[1].messageType 100005 +cluster{music}.route[2].name "music-index" +cluster{music}.route[2].messageType 100006 +cluster{music}.selector "(music)" diff --git a/config-model/src/test/cfg/routing/replaceroute/document-protocol-policies.cfg b/config-model/src/test/cfg/routing/replaceroute/document-protocol-policies.cfg new file mode 100644 index 00000000000..68659362350 --- /dev/null +++ b/config-model/src/test/cfg/routing/replaceroute/document-protocol-policies.cfg @@ -0,0 +1,8 @@ +cluster{music}.defaultRoute "music-direct" +cluster{music}.route[0].name "music-index" +cluster{music}.route[0].messageType 100004 +cluster{music}.route[1].name "music-index" +cluster{music}.route[1].messageType 100005 +cluster{music}.route[2].name "music-index" +cluster{music}.route[2].messageType 100006 +cluster{music}.selector "(music)" diff --git a/config-model/src/test/cfg/routing/routeconfig/document-protocol-policies.cfg b/config-model/src/test/cfg/routing/routeconfig/document-protocol-policies.cfg new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/config-model/src/test/cfg/routing/routeconfig/document-protocol-policies.cfg @@ -0,0 +1 @@ + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java index 34f06519ac9..c71bbc6d89c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerClusterTest.java @@ -23,9 +23,9 @@ import com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.App import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; import org.junit.Test; - import java.util.Collection; +import static com.yahoo.vespa.model.container.ContainerCluster.G1GC; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.METRICS_PROXY_BUNDLE_FILE; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyContainerCluster.zoneString; import static com.yahoo.vespa.model.admin.metricsproxy.MetricsProxyModelTester.CLUSTER_CONFIG_ID; @@ -81,7 +81,7 @@ public class MetricsProxyContainerClusterTest { assertEquals(0, qrStartConfig.jvm().heapSizeAsPercentageOfPhysicalMemory()); assertEquals(2, qrStartConfig.jvm().availableProcessors()); assertFalse(qrStartConfig.jvm().verbosegc()); - assertEquals("-XX:+UseG1GC -XX:MaxTenuringThreshold=15", qrStartConfig.jvm().gcopts()); + assertEquals(G1GC, qrStartConfig.jvm().gcopts()); assertEquals(512, qrStartConfig.jvm().stacksize()); assertEquals(0, qrStartConfig.jvm().directMemorySizeCache()); assertEquals(32, qrStartConfig.jvm().compressedClassSpaceSize()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java index eddad6fce89..b3ace3ef8de 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java @@ -87,6 +87,14 @@ public class MetricsProxyContainerTest { } @Test + public void preload_is_empty() { + VespaModel model = getModel(servicesWithContent(), self_hosted); + MetricsProxyContainer container = (MetricsProxyContainer)model.id2producer().get(CONTAINER_CONFIG_ID); + + assertEquals("", container.getPreLoad()); + } + + @Test public void hosted_application_propagates_node_dimensions() { String services = servicesWithContent(); VespaModel hostedModel = getModel(services, hosted); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 9f17a1c4142..095434c8e04 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -14,6 +14,7 @@ import com.yahoo.container.ComponentsConfig; import com.yahoo.messagebus.routing.RoutingTableSpec; import com.yahoo.metrics.MetricsmanagerConfig; import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; +import com.yahoo.vespa.config.content.DistributionConfig; import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.config.content.StorDistributionConfig; import com.yahoo.vespa.config.content.StorFilestorConfig; @@ -99,12 +100,22 @@ public class ContentClusterTest extends ContentBaseTest { " </group>" + "</content>" ); + DistributionConfig.Builder distributionBuilder = new DistributionConfig.Builder(); + cc.getConfig(distributionBuilder); + DistributionConfig distributionConfig = distributionBuilder.build(); + assertEquals(3, distributionConfig.cluster("storage").ready_copies()); + assertEquals(15, distributionConfig.cluster("storage").initial_redundancy()); + assertEquals(15, distributionConfig.cluster("storage").redundancy()); + assertEquals(4, distributionConfig.cluster("storage").group().size()); + assertEquals(1, distributionConfig.cluster().size()); + StorDistributionConfig.Builder storBuilder = new StorDistributionConfig.Builder(); cc.getConfig(storBuilder); StorDistributionConfig storConfig = new StorDistributionConfig(storBuilder); assertEquals(15, storConfig.initial_redundancy()); assertEquals(15, storConfig.redundancy()); assertEquals(3, storConfig.ready_copies()); + ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder(); cc.getSearch().getConfig(protonBuilder); ProtonConfig protonConfig = new ProtonConfig(protonBuilder); @@ -132,12 +143,20 @@ public class ContentClusterTest extends ContentBaseTest { " </group>" + "</content>" ); + DistributionConfig.Builder distributionBuilder = new DistributionConfig.Builder(); + cc.getConfig(distributionBuilder); + DistributionConfig distributionConfig = distributionBuilder.build(); + assertEquals(3, distributionConfig.cluster("storage").ready_copies()); + assertEquals(4, distributionConfig.cluster("storage").initial_redundancy()); + assertEquals(5, distributionConfig.cluster("storage").redundancy()); + StorDistributionConfig.Builder storBuilder = new StorDistributionConfig.Builder(); cc.getConfig(storBuilder); StorDistributionConfig storConfig = new StorDistributionConfig(storBuilder); assertEquals(4, storConfig.initial_redundancy()); assertEquals(5, storConfig.redundancy()); assertEquals(3, storConfig.ready_copies()); + ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder(); cc.getSearch().getConfig(protonBuilder); ProtonConfig protonConfig = new ProtonConfig(protonBuilder); @@ -163,8 +182,7 @@ public class ContentClusterTest extends ContentBaseTest { @Test public void testRedundancyDefaults() { - StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); - parse( + ContentCluster cc = parse( "<content version=\"1.0\" id=\"storage\">\n" + " <documents/>" + " <group>" + @@ -173,8 +191,15 @@ public class ContentClusterTest extends ContentBaseTest { " <node hostalias=\"mockhost\" distribution-key=\"2\"/>\"" + " </group>" + "</content>" - ).getConfig(builder); + ); + + DistributionConfig.Builder distributionBuilder = new DistributionConfig.Builder(); + cc.getConfig(distributionBuilder); + DistributionConfig distributionConfig = distributionBuilder.build(); + assertEquals(3, distributionConfig.cluster("storage").redundancy()); + StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); + cc.getConfig(builder); StorDistributionConfig config = new StorDistributionConfig(builder); assertEquals(2, config.initial_redundancy()); assertEquals(3, config.redundancy()); @@ -548,6 +573,13 @@ public class ContentClusterTest extends ContentBaseTest { "</content>" ); + DistributionConfig.Builder bob = new DistributionConfig.Builder(); + cluster.getConfig(bob); + DistributionConfig.Cluster.Group group = bob.build().cluster("test").group(0); + assertEquals("invalid", group.name()); + assertEquals("invalid", group.index()); + assertEquals(2, group.nodes().size()); + StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); cluster.getConfig(builder); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java index d290d4ec953..80ab6745b79 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedHierarchicDistributionTest.java @@ -2,10 +2,6 @@ package com.yahoo.vespa.model.content; import com.yahoo.vespa.config.content.StorDistributionConfig; -import com.yahoo.config.model.test.MockRoot; -import com.yahoo.vespa.model.Host; -import com.yahoo.vespa.model.HostResource; -import com.yahoo.vespa.model.SimpleConfigProducer; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.search.DispatchGroup; import com.yahoo.vespa.model.search.SearchInterface; @@ -17,9 +13,9 @@ import java.util.List; import java.util.Optional; import static com.yahoo.config.model.test.TestUtil.joinLines; -import static org.hamcrest.Matchers.containsString; import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createCluster; import static com.yahoo.vespa.model.content.utils.ContentClusterUtils.createClusterXml; +import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageGroupTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageGroupTest.java index cb457cabf6c..b9495d45e08 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageGroupTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageGroupTest.java @@ -2,13 +2,14 @@ package com.yahoo.vespa.model.content; import com.yahoo.config.model.test.MockRoot; +import com.yahoo.vespa.config.content.DistributionConfig; import com.yahoo.vespa.config.content.StorDistributionConfig; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import org.junit.Test; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * Test for storage groups. @@ -21,7 +22,6 @@ public class StorageGroupTest { @Test public void testSingleGroup() throws Exception { - StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); ContentCluster cluster = parse( "<content id=\"storage\">\n" + " <documents/>" + @@ -32,8 +32,6 @@ public class StorageGroupTest { "</content>" ); - cluster.getConfig(builder); - assertEquals("content", cluster.getStorageNodes().getChildren().get("0").getServicePropertyString("clustertype")); assertEquals("storage", cluster.getStorageNodes().getChildren().get("0").getServicePropertyString("clustername")); assertEquals("0", cluster.getStorageNodes().getChildren().get("0").getServicePropertyString("index")); @@ -42,6 +40,8 @@ public class StorageGroupTest { assertEquals("storage", cluster.getDistributorNodes().getChildren().get("0").getServicePropertyString("clustername")); assertEquals("0", cluster.getDistributorNodes().getChildren().get("0").getServicePropertyString("index")); + StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); + cluster.getConfig(builder); StorDistributionConfig config = new StorDistributionConfig(builder); assertEquals(1, config.group().size()); @@ -51,6 +51,17 @@ public class StorageGroupTest { assertEquals(0, config.group(0).nodes(0).index()); assertEquals(1, config.group(0).nodes(1).index()); //assertNotNull(cluster.getRootGroup().getNodes().get(0).getHost()); + + DistributionConfig.Builder distributionBuilder = new DistributionConfig.Builder(); + cluster.getConfig(distributionBuilder); + DistributionConfig.Cluster clusterConfig = distributionBuilder.build().cluster("storage"); + + assertEquals(1, clusterConfig.group().size()); + assertEquals("invalid", clusterConfig.group(0).index()); + assertEquals("invalid", clusterConfig.group(0).name()); + assertEquals(2, clusterConfig.group(0).nodes().size()); + assertEquals(0, clusterConfig.group(0).nodes(0).index()); + assertEquals(1, clusterConfig.group(0).nodes(1).index()); } @Test @@ -70,15 +81,14 @@ public class StorageGroupTest { " </group>\n" + "</cluster>" ); - assertTrue(false); + fail(); } catch (Exception e) { } } @Test public void testNestedGroups() throws Exception { - StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); - parse( + ContentCluster cluster = parse( "<content version=\"1.0\" id=\"storage\">\n" + " <redundancy>4</redundancy>" + " <documents/>" + @@ -101,8 +111,10 @@ public class StorageGroupTest { " </group>\n" + " </group>\n" + "</content>" - ).getConfig(builder); + ); + StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); + cluster.getConfig(builder); StorDistributionConfig config = new StorDistributionConfig(builder); assertEquals(5, config.group().size()); @@ -128,12 +140,39 @@ public class StorageGroupTest { assertEquals(5, config.group(4).nodes(1).index()); assertEquals("1|*", config.group(0).partitions()); + + DistributionConfig.Builder distributionBuilder = new DistributionConfig.Builder(); + cluster.getConfig(distributionBuilder); + DistributionConfig.Cluster clusterConfig = distributionBuilder.build().cluster("storage"); + + assertEquals(5, clusterConfig.group().size()); + assertEquals("invalid", clusterConfig.group(0).index()); + assertEquals("0", clusterConfig.group(1).index()); + assertEquals("1", clusterConfig.group(2).index()); + assertEquals("1.0", clusterConfig.group(3).index()); + assertEquals("1.1", clusterConfig.group(4).index()); + assertEquals("invalid", clusterConfig.group(0).name()); + assertEquals("sub1", clusterConfig.group(1).name()); + assertEquals("sub2", clusterConfig.group(2).name()); + assertEquals("sub3", clusterConfig.group(3).name()); + assertEquals("sub4", clusterConfig.group(4).name()); + assertEquals(2, clusterConfig.group(1).nodes().size()); + assertEquals(0, clusterConfig.group(1).nodes(0).index()); + assertEquals(1, clusterConfig.group(1).nodes(1).index()); + assertEquals(0, clusterConfig.group(2).nodes().size()); + assertEquals(2, clusterConfig.group(3).nodes().size()); + assertEquals(2, clusterConfig.group(3).nodes(0).index()); + assertEquals(3, clusterConfig.group(3).nodes(1).index()); + assertEquals(2, clusterConfig.group(4).nodes().size()); + assertEquals(4, clusterConfig.group(4).nodes(0).index()); + assertEquals(5, clusterConfig.group(4).nodes(1).index()); + + assertEquals("1|*", clusterConfig.group(0).partitions()); } @Test public void testGroupCapacity() throws Exception { - StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); - parse( + ContentCluster cluster = parse( "<content version=\"1.0\" id=\"storage\">\n" + " <redundancy>2</redundancy>" + " <documents/>" + @@ -149,13 +188,25 @@ public class StorageGroupTest { " </group>\n" + " </group>\n" + "</content>" - ).getConfig(builder); + ); + StorDistributionConfig.Builder builder = new StorDistributionConfig.Builder(); + cluster.getConfig(builder); StorDistributionConfig config = new StorDistributionConfig(builder); assertEquals(3, config.group().size()); assertEquals(5.5, config.group(0).capacity(), 0.001); assertEquals(2, config.group(1).capacity(), 0.001); assertEquals(3.5, config.group(2).capacity(), 0.001); + + DistributionConfig.Builder distributionBuilder = new DistributionConfig.Builder(); + cluster.getConfig(distributionBuilder); + DistributionConfig.Cluster clusterConfig = distributionBuilder.build().cluster("storage"); + + assertEquals(3, clusterConfig.group().size()); + assertEquals(5.5, clusterConfig.group(0).capacity(), 0.001); + assertEquals(2, clusterConfig.group(1).capacity(), 0.001); + assertEquals(3.5, clusterConfig.group(2).capacity(), 0.001); } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java index 9e7370a933c..2663ad1f348 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/routing/test/RoutingTestCase.java @@ -5,12 +5,20 @@ import com.yahoo.config.ConfigInstance; import com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig; import com.yahoo.io.IOUtils; import com.yahoo.messagebus.MessagebusConfig; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; import org.junit.Ignore; import org.junit.Test; -import java.io.*; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.PrintWriter; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -96,6 +104,10 @@ public class RoutingTestCase { model.getConfig(drB, ""); DocumentrouteselectorpolicyConfig dr = new DocumentrouteselectorpolicyConfig(drB); assertConfigFileContains(application, files, "documentrouteselectorpolicy.cfg", dr); + + DocumentProtocolPoliciesConfig.Builder policies = new DocumentProtocolPoliciesConfig.Builder(); + model.getConfig(policies, ""); + assertConfigFileContains(application, files, "document-protocol-policies.cfg", policies.build()); } else { StringBuilder msg = new StringBuilder(); for (String error : errors) { diff --git a/configdefinitions/src/vespa/CMakeLists.txt b/configdefinitions/src/vespa/CMakeLists.txt index c6e4d612a0a..4868a6bba2e 100644 --- a/configdefinitions/src/vespa/CMakeLists.txt +++ b/configdefinitions/src/vespa/CMakeLists.txt @@ -56,6 +56,8 @@ vespa_generate_config(configdefinitions slobroks.def) install_config_definition(slobroks.def cloud.config.slobroks.def) vespa_generate_config(configdefinitions specialtokens.def) install_config_definition(specialtokens.def vespa.configdefinition.specialtokens.def) +vespa_generate_config(configdefinitions distribution.def) +install_config_definition(distribution.def vespa.config.content.distribution.def) vespa_generate_config(configdefinitions stor-distribution.def) install_config_definition(stor-distribution.def vespa.config.content.stor-distribution.def) vespa_generate_config(configdefinitions stor-filestor.def) diff --git a/configdefinitions/src/vespa/distribution.def b/configdefinitions/src/vespa/distribution.def new file mode 100644 index 00000000000..d0cb4165ac9 --- /dev/null +++ b/configdefinitions/src/vespa/distribution.def @@ -0,0 +1,46 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.config.content + +## +## Super config for distribution in each content cluster, keyed by cluster id. +## + +# If this is set to true, the distributor will try to enforce one active copy of +# buckets per hierarchical leaf group. This lets the top level dispatcher send +# queries only to the nodes of one group, saving computational effort. +# If used, hierarchical grouping can not be used for other purposes. +# Using this option implies that: +# - ready_copies == redundancy +# - Only one level of hierarchical grouping may be defined. +# - That level distributes copies to all defined groups. +cluster{}.active_per_leaf_group bool default=false + +# The number of copies that should be "ready" to be active. Maximum is redundancy. +cluster{}.ready_copies int default=0 + +# How many copies of a document are stored, across nodes. +cluster{}.redundancy int default=3 + +# Initial redundancy allows put-operations to return as completed after +# a subset of all copies have been stored. +# A value of 0 disable this, and causes normal redundancy behavior instead. +cluster{}.initial_redundancy int default=0 + +# Hierarchical grouping divides the nodes into a tree of groups. The index is the +# string representation of a path from the root node in this tree, e.g., "1.2.1". +cluster{}.group[].index string + +# Each group needs to have a name. Obviously. Duh. +cluster{}.group[].name string + +# Capacity of the given group. +cluster{}.group[].capacity double default=1 + +# Partitions define how copies are divided among child groups/nodes. +cluster{}.group[].partitions string default="" + +# Leaf groups will have a set of nodes within them. Branch groups will have none. +cluster{}.group[].nodes[].index int + +# Whether this node is retired, and data should migrate out of it. +cluster{}.group[].nodes[].retired bool default=false
\ No newline at end of file diff --git a/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java b/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java index 5e3f873bba9..8b75e2d7660 100644 --- a/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java +++ b/container-core/src/main/java/com/yahoo/container/core/documentapi/DocumentAccessProvider.java @@ -7,6 +7,8 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.messagebus.MessagebusConfig; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; +import com.yahoo.vespa.config.content.DistributionConfig; import com.yahoo.vespa.config.content.LoadTypeConfig; /** @@ -19,10 +21,11 @@ public class DocumentAccessProvider extends AbstractComponent implements Provide private final VespaDocumentAccess access; @Inject - // TODO jonmv: Have Slobrok and RPC config injected as well. public DocumentAccessProvider(DocumentmanagerConfig documentmanagerConfig, LoadTypeConfig loadTypeConfig, - SlobroksConfig slobroksConfig, MessagebusConfig messagebusConfig) { - this.access = new VespaDocumentAccess(documentmanagerConfig, loadTypeConfig, slobroksConfig, messagebusConfig); + SlobroksConfig slobroksConfig, MessagebusConfig messagebusConfig, + DocumentProtocolPoliciesConfig policiesConfig, DistributionConfig distributionConfig) { + this.access = new VespaDocumentAccess(documentmanagerConfig, loadTypeConfig, slobroksConfig, messagebusConfig, + policiesConfig, distributionConfig); } @Override diff --git a/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java b/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java index 2918ffb2c80..d55df15b2fd 100644 --- a/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java +++ b/container-core/src/main/java/com/yahoo/container/core/documentapi/VespaDocumentAccess.java @@ -20,6 +20,8 @@ import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusParams; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.messagebus.MessagebusConfig; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; +import com.yahoo.vespa.config.content.DistributionConfig; import com.yahoo.vespa.config.content.LoadTypeConfig; /** @@ -39,9 +41,12 @@ public class VespaDocumentAccess extends DocumentAccess { VespaDocumentAccess(DocumentmanagerConfig documentmanagerConfig, LoadTypeConfig loadTypeConfig, SlobroksConfig slobroksConfig, - MessagebusConfig messagebusConfig) { + MessagebusConfig messagebusConfig, + DocumentProtocolPoliciesConfig policiesConfig, + DistributionConfig distributionConfig) { super(new DocumentAccessParams().setDocumentmanagerConfig(documentmanagerConfig)); - this.parameters = new MessageBusParams(new LoadTypeSet(loadTypeConfig)); + this.parameters = new MessageBusParams(new LoadTypeSet(loadTypeConfig)) + .setDocumentProtocolPoliciesConfig(policiesConfig, distributionConfig); this.parameters.setDocumentmanagerConfig(documentmanagerConfig); this.parameters.getRPCNetworkParams().setSlobroksConfig(slobroksConfig); this.parameters.getMessageBusParams().setMessageBusConfig(messagebusConfig); diff --git a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java index 1867da0317b..4940f707fd6 100644 --- a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java @@ -232,7 +232,7 @@ public class StatisticsSearcher extends Searcher { incrQueryCount(metricContext); logQuery(query); - long start = System.currentTimeMillis(); // Start time, in millisecs. + long start_ns = System.nanoTime(); // Start time, in nanoseconds. qps(metricContext); Result result; //handle exceptions thrown below in searchers @@ -243,14 +243,14 @@ public class StatisticsSearcher extends Searcher { throw e; } - long end = System.currentTimeMillis(); // Start time, in millisecs. - long latency = end - start; - if (latency >= 0) { - addLatency(latency, metricContext); + long end_ns = System.nanoTime(); // End time, in nanoseconds + long latency_ns = end_ns - start_ns; + if (latency_ns >= 0) { + addLatency(latency_ns, metricContext); } else { getLogger().log(Level.WARNING, - "Apparently negative latency measure, start: " + start - + ", end: " + end + ", for query: " + query.toString()); + "Apparently negative latency measure, start: " + start_ns + + ", end: " + end_ns + ", for query: " + query.toString()); } if (result.hits().getError() != null) { incrErrorCount(result, metricContext); @@ -288,7 +288,8 @@ public class StatisticsSearcher extends Searcher { } } - private void addLatency(long latency, Metric.Context metricContext) { + private void addLatency(long latency_ns, Metric.Context metricContext) { + double latency = 0.000001 * latency_ns; //myStats.addLatency(latency); queryLatency.put(latency); metric.set(QUERY_LATENCY_METRIC, latency, metricContext); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 341bf7de9d6..cc1a0a455c4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -136,7 +136,7 @@ public class Endpoint { public String upstreamIdOf(DeploymentId deployment) { if (scope != Scope.global) throw new IllegalArgumentException("Scope " + scope + " does not have upstream name"); if (!routingMethod.isShared()) throw new IllegalArgumentException("Routing method " + routingMethod + " does not have upstream name"); - return upstreamIdOf(name(), deployment.applicationId(), deployment.zoneId()); + return upstreamIdOf(cluster.value(), deployment.applicationId(), deployment.zoneId()); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java index 7aab759f676..15136ed79eb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java @@ -56,7 +56,7 @@ public class NodeVersion { /** Returns the duration of the change in this, measured relative to instant */ public Duration changeDuration(Instant instant) { - if (!changing()) return Duration.ZERO; + if (!upgrading()) return Duration.ZERO; if (suspendedAt.isEmpty()) return Duration.ZERO; // Node hasn't suspended to apply the change yet return Duration.between(suspendedAt.get(), instant).abs(); } @@ -88,9 +88,9 @@ public class NodeVersion { return Objects.hash(hostname, zone, currentVersion, wantedVersion, suspendedAt); } - /** Returns whether this is changing (upgrading or downgrading) */ - private boolean changing() { - return !currentVersion.equals(wantedVersion); + /** Returns whether this is upgrading */ + private boolean upgrading() { + return currentVersion.isBefore(wantedVersion); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java index eb97fa0725c..2d81d7304a1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; import org.junit.Test; +import java.util.List; import java.util.Map; import static org.junit.Assert.assertEquals; @@ -271,16 +272,16 @@ public class EndpointTest { // With non-default cluster "c1.a1.t1.us-north-1.prod", - Endpoint.of(app1).target(EndpointId.of("c1")).on(Port.tls(4443)).in(SystemName.main) + Endpoint.of(app1).target(EndpointId.of("ignored1"), ClusterSpec.Id.from("c1"), List.of(zone)).on(Port.tls(4443)).in(SystemName.main) ); var tests2 = Map.of( - // With non-default instance + // With non-default instance and default cluster "i2.a2.t2.us-north-1.prod", - Endpoint.of(app2).target(EndpointId.defaultId()).on(Port.tls(4443)).in(SystemName.main), + Endpoint.of(app2).target(EndpointId.defaultId(), ClusterSpec.Id.from("default"), List.of(zone)).on(Port.tls(4443)).in(SystemName.main), // With non-default instance and cluster "c2.i2.a2.t2.us-north-1.prod", - Endpoint.of(app2).target(EndpointId.of("c2")).on(Port.tls(4443)).in(SystemName.main) + Endpoint.of(app2).target(EndpointId.of("ignored2"), ClusterSpec.Id.from("c2"), List.of(zone)).on(Port.tls(4443)).in(SystemName.main) ); tests1.forEach((expected, endpoint) -> assertEquals(expected, endpoint.upstreamIdOf(new DeploymentId(app1, zone)))); tests2.forEach((expected, endpoint) -> assertEquals(expected, endpoint.upstreamIdOf(new DeploymentId(app2, zone)))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json index f7c512842fd..934e0cf43b9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json @@ -1,6 +1,6 @@ { "globalrotationoverride": [ - "instance1.application1.tenant1.us-west-1.prod", + "foo.instance1.application1.tenant1.us-west-1.prod", { "status": "in", "reason": "", diff --git a/dist/vespa.spec b/dist/vespa.spec index 50d591bde5c..b19d1c9a201 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -612,6 +612,7 @@ fi %{_prefix}/lib/jars/jackson-*.jar %{_prefix}/lib/jars/javassist-*.jar %{_prefix}/lib/jars/javax.*.jar +%{_prefix}/lib/jars/jdisc-cloud-aws-jar-with-dependencies.jar %{_prefix}/lib/jars/jdisc_core-jar-with-dependencies.jar %{_prefix}/lib/jars/jdisc_http_service-jar-with-dependencies.jar %{_prefix}/lib/jars/jdisc-security-filters-jar-with-dependencies.jar diff --git a/documentapi/abi-spec.json b/documentapi/abi-spec.json index 39d6898215c..36c2e1fda99 100644 --- a/documentapi/abi-spec.json +++ b/documentapi/abi-spec.json @@ -1128,6 +1128,7 @@ "public com.yahoo.documentapi.messagebus.MessageBusParams setRoutingConfigId(java.lang.String)", "public java.lang.String getProtocolConfigId()", "public com.yahoo.documentapi.messagebus.MessageBusParams setProtocolConfigId(java.lang.String)", + "public com.yahoo.documentapi.messagebus.MessageBusParams setDocumentProtocolPoliciesConfig(com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig, com.yahoo.vespa.config.content.DistributionConfig)", "public com.yahoo.documentapi.messagebus.MessageBusParams setRouteName(java.lang.String)", "public com.yahoo.documentapi.messagebus.MessageBusParams setRoute(java.lang.String)", "public com.yahoo.documentapi.messagebus.MessageBusParams setRouteNameForGet(java.lang.String)", @@ -1573,6 +1574,7 @@ "fields": [ "protected final java.lang.String clusterName", "protected final java.lang.String distributionConfigId", + "protected final com.yahoo.vespa.config.content.DistributionConfig distributionConfig", "protected final com.yahoo.documentapi.messagebus.protocol.ContentPolicy$SlobrokHostPatternGenerator slobrokHostPatternGenerator" ] }, @@ -1595,8 +1597,7 @@ "public" ], "methods": [ - "public void <init>(java.lang.String)", - "public void <init>(java.util.Map)", + "public void <init>(java.lang.String, com.yahoo.vespa.config.content.DistributionConfig)", "public void <init>(com.yahoo.documentapi.messagebus.protocol.ContentPolicy$Parameters)", "public void select(com.yahoo.messagebus.routing.RoutingContext)", "public void merge(com.yahoo.messagebus.routing.RoutingContext)", @@ -1806,6 +1807,7 @@ "public static com.yahoo.documentapi.messagebus.protocol.DocumentProtocol$Priority getPriorityByName(java.lang.String)", "public void <init>(com.yahoo.document.DocumentTypeManager)", "public void <init>(com.yahoo.document.DocumentTypeManager, java.lang.String)", + "public void <init>(com.yahoo.document.DocumentTypeManager, com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet, com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig, com.yahoo.vespa.config.content.DistributionConfig)", "public void <init>(com.yahoo.document.DocumentTypeManager, java.lang.String, com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet)", "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocol putRoutingPolicyFactory(java.lang.String, com.yahoo.documentapi.messagebus.protocol.RoutingPolicyFactory)", "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocol putRoutableFactory(int, com.yahoo.documentapi.messagebus.protocol.RoutableFactory, com.yahoo.component.VersionSpecification)", @@ -1890,6 +1892,138 @@ "public static final int ERROR_SUSPENDED" ] }, + "com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigInstance$Builder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig)", + "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Builder cluster(java.lang.String, com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Cluster$Builder)", + "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Builder cluster(java.util.Map)", + "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", + "public final java.lang.String getDefMd5()", + "public final java.lang.String getDefName()", + "public final java.lang.String getDefNamespace()", + "public final boolean getApplyOnRestart()", + "public final void setApplyOnRestart(boolean)", + "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig build()" + ], + "fields": [ + "public java.util.Map cluster" + ] + }, + "com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Cluster$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigBuilder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Cluster)", + "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Cluster$Builder defaultRoute(java.lang.String)", + "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Cluster$Builder route(com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Cluster$Route$Builder)", + "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Cluster$Builder route(java.util.List)", + "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Cluster$Builder selector(java.lang.String)", + "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Cluster build()" + ], + "fields": [ + "public java.util.List route" + ] + }, + "com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Cluster$Route$Builder": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigBuilder" + ], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>()", + "public void <init>(com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Cluster$Route)", + "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Cluster$Route$Builder name(java.lang.String)", + "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Cluster$Route$Builder messageType(int)", + "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Cluster$Route build()" + ], + "fields": [] + }, + "com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Cluster$Route": { + "superClass": "com.yahoo.config.InnerNode", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Cluster$Route$Builder)", + "public java.lang.String name()", + "public int messageType()" + ], + "fields": [] + }, + "com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Cluster": { + "superClass": "com.yahoo.config.InnerNode", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public void <init>(com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Cluster$Builder)", + "public java.lang.String defaultRoute()", + "public java.util.List route()", + "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Cluster$Route route(int)", + "public java.lang.String selector()" + ], + "fields": [] + }, + "com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Producer": { + "superClass": "java.lang.Object", + "interfaces": [ + "com.yahoo.config.ConfigInstance$Producer" + ], + "attributes": [ + "public", + "interface", + "abstract" + ], + "methods": [ + "public abstract void getConfig(com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Builder)" + ], + "fields": [] + }, + "com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig": { + "superClass": "com.yahoo.config.ConfigInstance", + "interfaces": [], + "attributes": [ + "public", + "final" + ], + "methods": [ + "public static java.lang.String getDefMd5()", + "public static java.lang.String getDefName()", + "public static java.lang.String getDefNamespace()", + "public static java.lang.String getDefVersion()", + "public void <init>(com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Builder)", + "public java.util.Map cluster()", + "public com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig$Cluster cluster(java.lang.String)" + ], + "fields": [ + "public static final java.lang.String CONFIG_DEF_MD5", + "public static final java.lang.String CONFIG_DEF_NAME", + "public static final java.lang.String CONFIG_DEF_NAMESPACE", + "public static final java.lang.String CONFIG_DEF_VERSION", + "public static final java.lang.String[] CONFIG_DEF_SCHEMA" + ] + }, "com.yahoo.documentapi.messagebus.protocol.DocumentProtocolRoutingPolicy": { "superClass": "java.lang.Object", "interfaces": [ @@ -1928,6 +2062,7 @@ "public" ], "methods": [ + "public void <init>(com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig)", "public void <init>(java.lang.String)", "public synchronized java.lang.String getError()", "public void configure(com.yahoo.documentapi.messagebus.protocol.DocumentrouteselectorpolicyConfig)", @@ -2952,18 +3087,6 @@ ], "fields": [] }, - "com.yahoo.documentapi.messagebus.protocol.RoutingPolicyFactories": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public", - "abstract" - ], - "methods": [ - "public void <init>()" - ], - "fields": [] - }, "com.yahoo.documentapi.messagebus.protocol.RoutingPolicyFactory": { "superClass": "java.lang.Object", "interfaces": [], @@ -2973,8 +3096,7 @@ "abstract" ], "methods": [ - "public abstract com.yahoo.documentapi.messagebus.protocol.DocumentProtocolRoutingPolicy createPolicy(java.lang.String)", - "public abstract void destroy()" + "public abstract com.yahoo.documentapi.messagebus.protocol.DocumentProtocolRoutingPolicy createPolicy(java.lang.String)" ], "fields": [] }, diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusParams.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusParams.java index a838f3b8723..e167e0057e2 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusParams.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusParams.java @@ -3,8 +3,13 @@ package com.yahoo.documentapi.messagebus; import com.yahoo.documentapi.DocumentAccessParams; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; import com.yahoo.messagebus.SourceSessionParams; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; +import com.yahoo.vespa.config.content.DistributionConfig; + +import static java.util.Objects.requireNonNull; /** * @author Einar M R Rosenvinge @@ -13,6 +18,8 @@ public class MessageBusParams extends DocumentAccessParams { private String routingConfigId = null; private String protocolConfigId = null; + private DocumentProtocolPoliciesConfig policiesConfig = null; + private DistributionConfig distributionConfig = null; private String route = "route:default"; private String routeForGet = "route:default-get"; private int traceLevel = 0; @@ -79,6 +86,14 @@ public class MessageBusParams extends DocumentAccessParams { return this; } + /** Sets the config used by the {@link DocumentProtocol} policies. */ + public MessageBusParams setDocumentProtocolPoliciesConfig(DocumentProtocolPoliciesConfig policiesConfig, + DistributionConfig distributionConfig) { + this.policiesConfig = requireNonNull(policiesConfig); + this.distributionConfig = requireNonNull(distributionConfig); + return this; + } + /** * Sets the name of the route to send appropriate requests to. This is a convenience method for prefixing a route * with "route:", and using {@link #setRoute} instead. diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ContentPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ContentPolicy.java index c03231543df..f8e6989bbfa 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ContentPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ContentPolicy.java @@ -6,7 +6,6 @@ import com.yahoo.document.BucketId; import com.yahoo.document.BucketIdFactory; import com.yahoo.jrt.slobrok.api.IMirror; import com.yahoo.jrt.slobrok.api.Mirror; -import java.util.logging.Level; import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; import com.yahoo.messagebus.ErrorCode; @@ -22,6 +21,7 @@ import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; +import com.yahoo.vespa.config.content.DistributionConfig; import java.util.ArrayList; import java.util.Collections; @@ -32,6 +32,7 @@ import java.util.Random; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -221,18 +222,28 @@ public class ContentPolicy extends SlobrokPolicy { /** Class parsing the semicolon separated parameter string and exposes the appropriate value to the policy. */ public static class Parameters { + protected final String clusterName; protected final String distributionConfigId; + protected final DistributionConfig distributionConfig; protected final SlobrokHostPatternGenerator slobrokHostPatternGenerator; public Parameters(Map<String, String> params) { + this(params, null); + } + + private Parameters(Map<String, String> params, DistributionConfig config) { clusterName = params.get("cluster"); + if (clusterName == null) + throw new IllegalArgumentException("Required parameter 'cluster', the name of the content cluster, not set"); + distributionConfig = config; + if (distributionConfig != null && distributionConfig.cluster(clusterName) == null) + throw new IllegalArgumentException("Distribution config for cluster '" + clusterName + "' not found"); distributionConfigId = params.get("clusterconfigid"); // TODO jonmv: remove slobrokHostPatternGenerator = createPatternGenerator(); - if (clusterName == null) throw new IllegalArgumentException("Required parameter 'cluster', the name of the content cluster, not set"); } - String getDistributionConfigId() { + private String getDistributionConfigId() { return distributionConfigId == null ? clusterName : distributionConfigId; } public String getClusterName() { @@ -245,7 +256,8 @@ public class ContentPolicy extends SlobrokPolicy { return new TargetCachingSlobrokHostFetcher(slobrokHostPatternGenerator, policy, percent); } public Distribution createDistribution(SlobrokPolicy policy) { - return new Distribution(getDistributionConfigId()); + return distributionConfig == null ? new Distribution(getDistributionConfigId()) + : new Distribution(distributionConfig.cluster(clusterName)); } /** @@ -548,12 +560,8 @@ public class ContentPolicy extends SlobrokPolicy { private final Parameters parameters; /** Constructor used in production. */ - public ContentPolicy(String param) { - this(parse(param)); - } - - public ContentPolicy(Map<String, String> params) { - this(new Parameters(params)); + public ContentPolicy(String param, DistributionConfig config) { + this(new Parameters(parse(param), config)); } /** Constructor specifying a bit more in detail, so we can override what needs to be overridden in tests */ diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java index 2680ed011af..547d7f76dc5 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentProtocol.java @@ -15,6 +15,7 @@ import com.yahoo.messagebus.routing.RoutingContext; import com.yahoo.messagebus.routing.RoutingNodeIterator; import com.yahoo.messagebus.routing.RoutingPolicy; import com.yahoo.text.Utf8String; +import com.yahoo.vespa.config.content.DistributionConfig; import java.util.Collections; import java.util.HashSet; @@ -24,6 +25,8 @@ import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; +import static java.util.Objects.requireNonNull; + /** * Implements the message bus protocol that is used by all components of Vespa. * @@ -243,27 +246,36 @@ public class DocumentProtocol implements Protocol { this(docMan, configId, new LoadTypeSet()); } + public DocumentProtocol(DocumentTypeManager documentTypeManager, LoadTypeSet loadTypes, + DocumentProtocolPoliciesConfig policiesConfig, DistributionConfig distributionConfig) { + this(requireNonNull(documentTypeManager), null, requireNonNull(loadTypes), + requireNonNull(policiesConfig), requireNonNull(distributionConfig)); + } + public DocumentProtocol(DocumentTypeManager docMan, String configId, LoadTypeSet set) { - // Prepare config string for routing policy factories. - String cfg = (configId == null ? "client" : configId); - if (docMan != null) { + this(docMan, configId == null ? "client" : configId, set, null, null); + } + + private DocumentProtocol(DocumentTypeManager docMan, String configId, LoadTypeSet set, + DocumentProtocolPoliciesConfig policiesConfig, DistributionConfig distributionConfig) { + if (docMan != null) this.docMan = docMan; - } else { + else { this.docMan = new DocumentTypeManager(); - DocumentTypeManagerConfigurer.configure(this.docMan, cfg); + DocumentTypeManagerConfigurer.configure(this.docMan, configId); } - routableRepository = new RoutableRepository(set); + this.routableRepository = new RoutableRepository(set); // When adding factories to this list, please KEEP THEM ORDERED alphabetically like they are now. putRoutingPolicyFactory("AND", new RoutingPolicyFactories.AndPolicyFactory()); - putRoutingPolicyFactory("Content", new RoutingPolicyFactories.ContentPolicyFactory()); - putRoutingPolicyFactory("DocumentRouteSelector", new RoutingPolicyFactories.DocumentRouteSelectorPolicyFactory(cfg)); + putRoutingPolicyFactory("Content", new RoutingPolicyFactories.ContentPolicyFactory(distributionConfig)); + putRoutingPolicyFactory("DocumentRouteSelector", new RoutingPolicyFactories.DocumentRouteSelectorPolicyFactory(configId, policiesConfig)); putRoutingPolicyFactory("Extern", new RoutingPolicyFactories.ExternPolicyFactory()); putRoutingPolicyFactory("LocalService", new RoutingPolicyFactories.LocalServicePolicyFactory()); - putRoutingPolicyFactory("MessageType", new RoutingPolicyFactories.MessageTypePolicyFactory(cfg)); + putRoutingPolicyFactory("MessageType", new RoutingPolicyFactories.MessageTypePolicyFactory(configId, policiesConfig)); putRoutingPolicyFactory("RoundRobin", new RoutingPolicyFactories.RoundRobinPolicyFactory()); putRoutingPolicyFactory("LoadBalancer", new RoutingPolicyFactories.LoadBalancerPolicyFactory()); - putRoutingPolicyFactory("Storage", new RoutingPolicyFactories.ContentPolicyFactory()); + putRoutingPolicyFactory("Storage", new RoutingPolicyFactories.ContentPolicyFactory(distributionConfig)); putRoutingPolicyFactory("SubsetService", new RoutingPolicyFactories.SubsetServicePolicyFactory()); // Prepare version specifications to use when adding routable factories. diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java index 8fbd1548f68..07fd098c9b4 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java @@ -5,13 +5,14 @@ import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.document.DocumentGet; import com.yahoo.document.select.DocumentSelector; import com.yahoo.document.select.Result; -import java.util.logging.Level; +import com.yahoo.document.select.parser.ParseException; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.routing.Route; import com.yahoo.messagebus.routing.RoutingContext; import java.util.HashMap; import java.util.Map; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -30,6 +31,24 @@ public class DocumentRouteSelectorPolicy private String error = "Not configured."; private ConfigSubscriber subscriber; + /** This policy is constructed with the proper config at its time of creation. */ + public DocumentRouteSelectorPolicy(DocumentProtocolPoliciesConfig config) { + Map<String, DocumentSelector> selectors = new HashMap<>(); + config.cluster().forEach((name, cluster) -> { + try { + selectors.put(name, new DocumentSelector(cluster.selector())); + } + catch (ParseException e) { + throw new IllegalArgumentException("Error parsing selector '" + + cluster.selector() + + "' for route '" + name +"'", + e); + } + }); + this.config = Map.copyOf(selectors); + this.error = null; + } + /** * This policy is constructed with a configuration identifier that can be subscribed to for the document selector * config. If the string is either null or empty it will default to the proper one. diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MessageTypePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MessageTypePolicy.java index 4226c1e6cac..34124cf48db 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MessageTypePolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/MessageTypePolicy.java @@ -5,10 +5,13 @@ import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.messagebus.routing.Route; import com.yahoo.messagebus.routing.RoutingContext; import com.yahoo.vespa.config.content.MessagetyperouteselectorpolicyConfig; + import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; +import static java.util.stream.Collectors.toUnmodifiableMap; + /** * @author baldersheim */ @@ -18,6 +21,13 @@ public class MessageTypePolicy implements DocumentProtocolRoutingPolicy, ConfigS private ConfigSubscriber subscriber; private volatile Route defaultRoute; + MessageTypePolicy(DocumentProtocolPoliciesConfig.Cluster config) { + configRef.set(config.route().stream() + .collect(toUnmodifiableMap(route -> route.messageType(), + route -> Route.parse(route.name())))); + defaultRoute = Route.parse(config.defaultRoute()); + } + MessageTypePolicy(String configId) { subscriber = new ConfigSubscriber(); subscriber.subscribe(this, MessagetyperouteselectorpolicyConfig.class, configId); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactories.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactories.java index 7b44a1a4f0d..c313422ab1b 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactories.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactories.java @@ -1,54 +1,72 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi.messagebus.protocol; +import com.yahoo.documentapi.messagebus.protocol.DocumentProtocolPoliciesConfig; +import com.yahoo.vespa.config.content.DistributionConfig; + /** * @author Simon Thoresen Hult + * @author jonmv */ -public abstract class RoutingPolicyFactories { +class RoutingPolicyFactories { + + private RoutingPolicyFactories() { } static class AndPolicyFactory implements RoutingPolicyFactory { public DocumentProtocolRoutingPolicy createPolicy(String param) { return new ANDPolicy(param); } - - - public void destroy() { - } } static class ContentPolicyFactory implements RoutingPolicyFactory { + private final DistributionConfig distributionConfig; + public ContentPolicyFactory(DistributionConfig config) { this.distributionConfig = config; } public DocumentProtocolRoutingPolicy createPolicy(String param) { - return new ContentPolicy(param); - } - - public void destroy() { + return new ContentPolicy(param, distributionConfig); } } static class MessageTypePolicyFactory implements RoutingPolicyFactory { + private final String configId; + private final DocumentProtocolPoliciesConfig config; - public MessageTypePolicyFactory(String configId) { + public MessageTypePolicyFactory(String configId, DocumentProtocolPoliciesConfig config) { this.configId = configId; + this.config = config; } + public DocumentProtocolRoutingPolicy createPolicy(String param) { - return new MessageTypePolicy((param == null || param.isEmpty()) ? configId : param); - } + if (config != null) { + if (config.cluster(param) == null) + return new ErrorPolicy("No message type config for cluster '" + param + "'"); - public void destroy() { + return new MessageTypePolicy(config.cluster(param)); + } + return new MessageTypePolicy(param == null || param.isEmpty() ? configId : param); } } static class DocumentRouteSelectorPolicyFactory implements RoutingPolicyFactory { private final String configId; + private final DocumentProtocolPoliciesConfig config; - public DocumentRouteSelectorPolicyFactory(String configId) { + public DocumentRouteSelectorPolicyFactory(String configId, DocumentProtocolPoliciesConfig config) { this.configId = configId; + this.config = config; } public DocumentProtocolRoutingPolicy createPolicy(String param) { - DocumentRouteSelectorPolicy ret = new DocumentRouteSelectorPolicy((param == null || param.isEmpty()) ? + if (config != null) { + try { + return new DocumentRouteSelectorPolicy(config); + } + catch (IllegalArgumentException e) { + return new ErrorPolicy(e.getMessage()); + } + } + DocumentRouteSelectorPolicy ret = new DocumentRouteSelectorPolicy(param == null || param.isEmpty() ? configId : param); String error = ret.getError(); if (error != null) { @@ -56,10 +74,6 @@ public abstract class RoutingPolicyFactories { } return ret; } - - - public void destroy() { - } } static class ExternPolicyFactory implements RoutingPolicyFactory { @@ -71,49 +85,30 @@ public abstract class RoutingPolicyFactories { } return ret; } - - - public void destroy() { - } } static class LocalServicePolicyFactory implements RoutingPolicyFactory { public DocumentProtocolRoutingPolicy createPolicy(String param) { return new LocalServicePolicy(param); } - - - public void destroy() { - } } static class RoundRobinPolicyFactory implements RoutingPolicyFactory { public DocumentProtocolRoutingPolicy createPolicy(String param) { return new RoundRobinPolicy(); } - - - public void destroy() { - } } static class LoadBalancerPolicyFactory implements RoutingPolicyFactory { public DocumentProtocolRoutingPolicy createPolicy(String param) { return new LoadBalancerPolicy(param); } - - - public void destroy() { - } } static class SubsetServicePolicyFactory implements RoutingPolicyFactory { public DocumentProtocolRoutingPolicy createPolicy(String param) { return new SubsetServicePolicy(param); } - - - public void destroy() { - } } + } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactory.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactory.java index 6ea5020607e..3e368832c98 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactory.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyFactory.java @@ -24,11 +24,6 @@ public interface RoutingPolicyFactory { * @param param The parameter to use when creating the policy. * @return The created routing policy. */ - public DocumentProtocolRoutingPolicy createPolicy(String param); + DocumentProtocolRoutingPolicy createPolicy(String param); - /** - * Destroys this factory and frees up any resources it has held. Making further calls on a destroyed - * factory causes a runtime exception. - */ - public void destroy(); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SlobrokPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SlobrokPolicy.java index f4e1bb33dd1..1ffce622d78 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SlobrokPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SlobrokPolicy.java @@ -10,10 +10,10 @@ import java.util.Map; import java.util.TreeMap; /** - * Abstract class for policies that allow you to specify which slobrok to use for the - * routing. + * Abstract class for policies that allow you to specify which slobrok to use for the routing. */ public abstract class SlobrokPolicy implements DocumentProtocolRoutingPolicy { + private boolean firstTry = true; protected List<Mirror.Entry> lookup(RoutingContext context, String pattern) { diff --git a/documentapi/src/main/resources/configdefinitions/document-protocol-policies.def b/documentapi/src/main/resources/configdefinitions/document-protocol-policies.def new file mode 100644 index 00000000000..ace4f254821 --- /dev/null +++ b/documentapi/src/main/resources/configdefinitions/document-protocol-policies.def @@ -0,0 +1,25 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=documentapi.messagebus.protocol + +## +## Super config for all policies in the document protocol, keyed by content cluster ids. +## Note: ContentPolicy also uses the "distribution" config. +## + +# +# Config used by MessageTypeRouteSelectorPolicy +# +# Default route if no override is set for a type. +cluster{}.defaultRoute string + +# The name of the route. +cluster{}.route[].name string + +# The document protocol message type triggering this route. +cluster{}.route[].messageType int + +# +# Config used by DocumentRouteSelectorPolicy +# +# The document selector for this cluster route. +cluster{}.selector string diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTest.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTest.java index f324245b612..5aa3994a757 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTest.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTest.java @@ -5,6 +5,7 @@ import org.junit.Ignore; import org.junit.Test; public class ContentPolicyTest extends Simulator { + /** * Verify that a resent message with failures doesn't ruin overall performance. (By dumping the cached state too often * so other requests are sent to wrong target) @@ -17,6 +18,7 @@ public class ContentPolicyTest extends Simulator { + "Last correctnode 99, wrongnode 1, downnode 0, worked 92, failed 8", new PersistentFailureTestParameters().addBadNode(new BadNode(3, FailureType.TRANSIENT_ERROR))); } + /** * Verify that a resent message with failures doesn't ruin overall performance. (By dumping the cached state too often * so other requests are sent to wrong target) @@ -29,6 +31,7 @@ public class ContentPolicyTest extends Simulator { + "Last correctnode 99, wrongnode 1, downnode 0, worked 92, failed 8", new PersistentFailureTestParameters().addBadNode(new BadNode(3, FailureType.FATAL_ERROR))); } + /** * Verify that a node responding with old cluster state doesn't ruin overall performance (By dumping/switching cached * state too often) @@ -41,6 +44,7 @@ public class ContentPolicyTest extends Simulator { + "Last correctnode 100, wrongnode 0, downnode 0, worked 100, failed 0", new PersistentFailureTestParameters().addBadNode(new BadNode(3, FailureType.OLD_CLUSTER_STATE).setDownInCurrentState())); } + /** * Verify that a reset cluster state version doesn't keep sending requests to the wrong node. * We expect a few failures in first half. We should have detected the issue before second half, so there all should be fine. @@ -52,6 +56,7 @@ public class ContentPolicyTest extends Simulator { + "Last correctnode .*, wrongnode 0, downnode 0, worked .*, failed 0", new PersistentFailureTestParameters().addBadNode(new BadNode(3, FailureType.RESET_CLUSTER_STATE).setDownInCurrentState())); } + /** * Verify that a reset cluster state version doesn't keep sending requests to the wrong node. * We expect a few failures in first half. We should have detected the issue before second half, so there all should be fine. @@ -70,6 +75,7 @@ public class ContentPolicyTest extends Simulator { + "Last correctnode .*, wrongnode 100, downnode 100, worked 0, failed 100", new PersistentFailureTestParameters().addBadNode(new BadNode(3, FailureType.RESET_CLUSTER_STATE_NO_GOOD_NODES).setDownInCurrentState())); } + /** * Verify that a reset cluster state version doesn't keep sending requests to the wrong node. * We expect a few failures in first half. We should have detected the issue before second half, so there all should be fine. @@ -86,6 +92,7 @@ public class ContentPolicyTest extends Simulator { + "Last correctnode .*, wrongnode 91, downnode 0, worked 0, failed 100", new PersistentFailureTestParameters().addBadNode(new BadNode(3, FailureType.RESET_CLUSTER_STATE_NO_GOOD_NODES))); } + /** * Verify that a reset cluster state version doesn't keep sending requests to the wrong node. * Another scenario where we have a node coming up in correct state. @@ -98,6 +105,7 @@ public class ContentPolicyTest extends Simulator { + "Last correctnode .*, wrongnode 0, downnode 0, worked .*, failed 0", new PersistentFailureTestParameters().newNodeAdded().addBadNode(new BadNode(3, FailureType.RESET_CLUSTER_STATE).setDownInCurrentState())); } + /** Test node that is not in slobrok. Until fleetcontroller detects this, we expect 10% of the requests to go to wrong node. */ @Test @Ignore // FIXME test has been implicitly disabled for ages, figure out and fix diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTestEnvironment.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTestEnvironment.java index 479e0b0f422..6d2477e1871 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTestEnvironment.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/ContentPolicyTestEnvironment.java @@ -34,11 +34,13 @@ import java.util.TreeSet; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public abstract class ContentPolicyTestEnvironment { - protected StoragePolicyTestFactory policyFactory; + protected ContentPolicyTestFactory policyFactory; protected PolicyTestFrame frame; private Set<Integer> nodes; protected static int[] bucketOneNodePreference = new int[]{ 3, 5, 7, 6, 8, 0, 9, 2, 1, 4 }; @@ -51,7 +53,7 @@ public abstract class ContentPolicyTestEnvironment { frame = new PolicyTestFrame(manager); nodes = new TreeSet<>(); DocumentProtocol protocol = (DocumentProtocol) frame.getMessageBus().getProtocol((Utf8Array)DocumentProtocol.NAME); - policyFactory = new StoragePolicyTestFactory(nodes); + policyFactory = new ContentPolicyTestFactory(nodes); protocol.putRoutingPolicyFactory("storage", policyFactory); frame.setMessage(createMessage("id:ns:testdoc:n=1:foo")); frame.setHop(new HopSpec("test", "[storage:cluster=foo]")); @@ -104,7 +106,7 @@ public abstract class ContentPolicyTestEnvironment { public static class TestHostFetcher extends ContentPolicy.HostFetcher { private final String clusterName; - private RandomGen randomizer = new RandomGen(1234); + private final RandomGen randomizer = new RandomGen(1234); private final Set<Integer> nodes; private Integer avoidPickingAtRandom = null; @@ -121,13 +123,14 @@ public abstract class ContentPolicyTestEnvironment { try{ if (distributor == null) { if (nodes.size() == 1) { - assertTrue(avoidPickingAtRandom != nodes.iterator().next()); + assertNotSame(avoidPickingAtRandom, nodes.iterator().next()); distributor = nodes.iterator().next(); } else { Iterator<Integer> it = nodes.iterator(); for (int i = 0, n = randomizer.nextInt(nodes.size() - 1); i<n; ++i) it.next(); distributor = it.next(); - if (avoidPickingAtRandom != null && distributor == avoidPickingAtRandom) distributor = it.next(); + if (avoidPickingAtRandom != null && avoidPickingAtRandom.equals(distributor)) + distributor = it.next(); } } if (nodes.contains(distributor)) { @@ -137,8 +140,7 @@ public abstract class ContentPolicyTestEnvironment { } } catch (RuntimeException e) { e.printStackTrace(); - assertTrue(e.getMessage(), false); - throw e; + throw new AssertionError(e.getMessage()); } } } @@ -160,12 +162,12 @@ public abstract class ContentPolicyTestEnvironment { public Distribution createDistribution(SlobrokPolicy policy) { return distribution; } } - public static class StoragePolicyTestFactory implements RoutingPolicyFactory { + public static class ContentPolicyTestFactory implements RoutingPolicyFactory { private Set<Integer> nodes; - private final LinkedList<TestParameters> parameterInstances = new LinkedList<TestParameters>(); + private final LinkedList<TestParameters> parameterInstances = new LinkedList<>(); private Integer avoidPickingAtRandom = null; - public StoragePolicyTestFactory(Set<Integer> nodes) { + public ContentPolicyTestFactory(Set<Integer> nodes) { this.nodes = nodes; } public DocumentProtocolRoutingPolicy createPolicy(String parameters) { diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/Simulator.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/Simulator.java index d23dd9ea998..be880e69781 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/Simulator.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/storagepolicy/Simulator.java @@ -31,7 +31,7 @@ public abstract class Simulator extends ContentPolicyTestEnvironment { RESET_CLUSTER_STATE_NO_GOOD_NODES, NODE_NOT_IN_SLOBROK }; - private Integer getIdealTarget(String idString, String clusterState) { + private int getIdealTarget(String idString, String clusterState) { DocumentId did = new DocumentId(idString); BucketIdFactory factory = new BucketIdFactory(); BucketId bid = factory.getBucketId(did); @@ -145,6 +145,7 @@ public abstract class Simulator extends ContentPolicyTestEnvironment { return currentClusterState; } } + public void runSimulation(String expected, PersistentFailureTestParameters params) { params.validate(); // Set nodes in slobrok @@ -157,16 +158,16 @@ public abstract class Simulator extends ContentPolicyTestEnvironment { replyWrongDistribution(target, "foo", null, params.getInitialClusterState().toString()); } RandomGen randomizer = new RandomGen(432121); - int correctnode[] = new int[2], - wrongnode[] = new int[2], - failed[] = new int[2], - worked[] = new int[2], - downnode[] = new int[2]; + int[] correctnode = new int[2], + wrongnode = new int[2], + failed = new int[2], + worked = new int[2], + downnode = new int[2]; for (int step = 0, steps = (params.getTotalRequests() / params.getParallellRequests()); step < steps; ++step) { int half = (step < steps / 2 ? 0 : 1); if (debug) System.err.println("Starting step " + step + " in half " + half); - String docId[] = new String[params.getParallellRequests()]; - RoutingNode targets[] = new RoutingNode[params.getParallellRequests()]; + String[] docId = new String[params.getParallellRequests()]; + RoutingNode[] targets = new RoutingNode[params.getParallellRequests()]; for (int i=0; i<params.getParallellRequests(); ++i) { docId[i] = "id:ns:testdoc::" + (step * params.getParallellRequests() + i); frame.setMessage(createMessage(docId[i])); @@ -206,7 +207,6 @@ public abstract class Simulator extends ContentPolicyTestEnvironment { } } StringBuilder actual = new StringBuilder(); - String result[][] = new String[2][]; for (int i=0; i<2; ++i) { actual.append(i == 0 ? "First " : " Last ") .append("correctnode ").append(correctnode[i]) @@ -215,7 +215,7 @@ public abstract class Simulator extends ContentPolicyTestEnvironment { .append(", worked ").append(worked[i]) .append(", failed ").append(failed[i]); } - if (!Pattern.matches(expected, actual.toString())) { + if ( ! Pattern.matches(expected, actual.toString())) { assertEquals(expected, actual.toString()); } } diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/CMakeLists.txt b/documentapi/src/vespa/documentapi/messagebus/policies/CMakeLists.txt index 83e1df02a24..82865ea5996 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/CMakeLists.txt +++ b/documentapi/src/vespa/documentapi/messagebus/policies/CMakeLists.txt @@ -16,5 +16,7 @@ vespa_add_library(documentapi_documentapipolicies OBJECT asyncinitializationpolicy.cpp DEPENDS ) +vespa_generate_config(documentapi_documentapipolicies ../../../../main/resources/configdefinitions/document-protocol-policies.def) +install_config_definition(../../../../main/resources/configdefinitions/document-protocol-policies.def documentapi.messagebus.protocol.document-protocol-policies.def) vespa_generate_config(documentapi_documentapipolicies ../../../../main/resources/configdefinitions/documentrouteselectorpolicy.def) install_config_definition(../../../../main/resources/configdefinitions/documentrouteselectorpolicy.def documentapi.messagebus.protocol.documentrouteselectorpolicy.def) diff --git a/jdisc-cloud-aws/CMakeLists.txt b/jdisc-cloud-aws/CMakeLists.txt new file mode 100644 index 00000000000..cc53e39f6a5 --- /dev/null +++ b/jdisc-cloud-aws/CMakeLists.txt @@ -0,0 +1,3 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +install_fat_java_artifact(jdisc-cloud-aws) +install_config_definitions() diff --git a/jdisc-cloud-aws/OWNERS b/jdisc-cloud-aws/OWNERS new file mode 100644 index 00000000000..e3da51bd047 --- /dev/null +++ b/jdisc-cloud-aws/OWNERS @@ -0,0 +1,2 @@ +mortent +oyving diff --git a/jdisc-cloud-aws/pom.xml b/jdisc-cloud-aws/pom.xml new file mode 100644 index 00000000000..9089c5785c7 --- /dev/null +++ b/jdisc-cloud-aws/pom.xml @@ -0,0 +1,43 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>7-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>jdisc-cloud-aws</artifactId> + <version>7-SNAPSHOT</version> + <packaging>container-plugin</packaging> + <name>${project.artifactId}</name> + + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>container-disc</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + </plugin> + </plugins> + </build> +</project>
\ No newline at end of file diff --git a/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStore.java b/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStore.java new file mode 100644 index 00000000000..8e7678723e6 --- /dev/null +++ b/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/AwsParameterStore.java @@ -0,0 +1,21 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.jdisc.cloud.aws; + +import com.yahoo.container.jdisc.secretstore.SecretStore; + +/** + * @author mortent + */ +public class AwsParameterStore implements SecretStore { + + @Override + public String getSecret(String key) { + return null; + } + + @Override + public String getSecret(String key, int version) { + return null; + } +} diff --git a/messagebus/src/main/java/com/yahoo/messagebus/test/SimpleMessage.java b/messagebus/src/main/java/com/yahoo/messagebus/test/SimpleMessage.java index 0e77d94497e..f87bcdbe1bb 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/test/SimpleMessage.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/test/SimpleMessage.java @@ -5,7 +5,7 @@ import com.yahoo.messagebus.Message; import com.yahoo.text.Utf8String; /** - * @author <a href="mailto:havardpe@yahoo-inc.com">Haavard Pettersen</a> + * @author havardpe */ public class SimpleMessage extends Message { diff --git a/messagebus/src/test/java/com/yahoo/messagebus/DynamicThrottlePolicyTest.java b/messagebus/src/test/java/com/yahoo/messagebus/DynamicThrottlePolicyTest.java index 7f48edfd48c..63747803e75 100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/DynamicThrottlePolicyTest.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/DynamicThrottlePolicyTest.java @@ -101,7 +101,7 @@ public class DynamicThrottlePolicyTest { CustomTimer timer = new CustomTimer(); DynamicThrottlePolicy policy = new DynamicThrottlePolicy(timer); int scaleFactor = (int) Math.pow(10, i); - long operations = 3_000 * scaleFactor; + long operations = 3_000L * scaleFactor; int workPerSuccess = 6; int numberOfWorkers = 1; int maximumTasksPerWorker = 100000; @@ -124,7 +124,7 @@ public class DynamicThrottlePolicyTest { CustomTimer timer = new CustomTimer(); DynamicThrottlePolicy policy = new DynamicThrottlePolicy(timer); int scaleFactor = (int) Math.pow(10, i); - long operations = 5_000 * scaleFactor; + long operations = 5_000L * scaleFactor; // workPerSuccess determines the latency of the simulated server, which again determines the impact of the // synthetic attractors of the algorithm, around latencies which give (close to) integer log10(1 / latency). // With a value of 5, the impact is that the algorithm is pushed upwards slightly above 10k window size, @@ -319,7 +319,6 @@ public class DynamicThrottlePolicyTest { .collect(toUnmodifiableList()); } - /** Performs a tick, and returns whether work was done. */ void tick() { for (int i = 0; i < numberOfWorkers; i++) tick(i); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java index 5ea0a5d12c3..8ac5a89aaef 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java @@ -215,6 +215,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { private FileSystem fileSystem = FileSystems.getDefault(); private FlagSource flagSource; private double cpuSpeedUp = 1; + private Path containerStorage; public Builder(NodeSpec node) { this.nodeSpecBuilder = new NodeSpec.Builder(node); @@ -280,6 +281,11 @@ public class NodeAgentContextImpl implements NodeAgentContext { return this; } + public Builder containerStorage(Path path) { + this.containerStorage = path; + return this; + } + public NodeAgentContextImpl build() { return new NodeAgentContextImpl( nodeSpecBuilder.build(), @@ -309,7 +315,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { }), fileSystem, Optional.ofNullable(flagSource).orElseGet(InMemoryFlagSource::new), - fileSystem.getPath("/home/docker/container-storage"), + Optional.ofNullable(containerStorage).orElseGet(() -> fileSystem.getPath("/home/docker/container-storage")), fileSystem.getPath("/opt/vespa"), Optional.ofNullable(vespaUser).orElse("vespa"), Optional.ofNullable(vespaUserOnHost).orElse("container_vespa"), @@ -91,6 +91,7 @@ <module>indexinglanguage</module> <module>jaxrs_client_utils</module> <module>jaxrs_utils</module> + <module>jdisc-cloud-aws</module> <module>jdisc-security-filters</module> <module>jdisc_core</module> <module>jdisc_core_test</module> diff --git a/protocols/getnodestate/host_info.json b/protocols/getnodestate/host_info.json index b14eaa0e13c..7ae5b0043ff 100644 --- a/protocols/getnodestate/host_info.json +++ b/protocols/getnodestate/host_info.json @@ -103,5 +103,15 @@ ] } ] + }, + "content-node": { + "resource-usage": { + "memory": { + "usage": 0.85 + }, + "disk": { + "usage": 0.6 + } + } } } diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp index 0cc9a92895b..cabcd33b2dd 100644 --- a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp @@ -10,6 +10,7 @@ #include <vespa/searchcore/proton/server/idocumentmovehandler.h> #include <vespa/searchcore/proton/server/imaintenancejobrunner.h> #include <vespa/searchcore/proton/server/maintenancedocumentsubdb.h> +#include <vespa/searchcore/proton/server/ibucketmodifiedhandler.h> #include <vespa/searchcore/proton/test/buckethandler.h> #include <vespa/searchcore/proton/test/clusterstatehandler.h> #include <vespa/searchcore/proton/test/disk_mem_usage_notifier.h> @@ -44,7 +45,6 @@ using BucketIdVector = BucketId::List; using DocumentVector = std::vector<Document::SP>; using MoveOperationVector = std::vector<MoveOperation>; using ScanItr = BucketMoveJob::ScanIterator; -using ScanPos = BucketMoveJob::ScanPosition; namespace { diff --git a/searchcore/src/tests/proton/documentdb/move_operation_limiter/move_operation_limiter_test.cpp b/searchcore/src/tests/proton/documentdb/move_operation_limiter/move_operation_limiter_test.cpp index 92ec0d6185b..eaad2ac2576 100644 --- a/searchcore/src/tests/proton/documentdb/move_operation_limiter/move_operation_limiter_test.cpp +++ b/searchcore/src/tests/proton/documentdb/move_operation_limiter/move_operation_limiter_test.cpp @@ -54,6 +54,15 @@ struct Fixture { } }; +TEST_F("require that hasPending reflects if any jobs are outstanding", Fixture) +{ + EXPECT_FALSE(f.limiter->hasPending()); + f.beginOp(); + EXPECT_TRUE(f.limiter->hasPending()); + f.endOp(); + EXPECT_FALSE(f.limiter->hasPending()); +} + TEST_F("require that job is blocked / unblocked when crossing max outstanding ops boundaries", Fixture) { f.beginOp(); diff --git a/searchcore/src/vespa/searchcore/proton/server/blockable_maintenance_job.cpp b/searchcore/src/vespa/searchcore/proton/server/blockable_maintenance_job.cpp index d8b5aa7b129..ce3bd3b8e9b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/blockable_maintenance_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/blockable_maintenance_job.cpp @@ -4,6 +4,7 @@ #include "disk_mem_usage_state.h" #include "imaintenancejobrunner.h" #include "document_db_maintenance_config.h" +#include "move_operation_limiter.h" namespace proton { @@ -49,7 +50,7 @@ BlockableMaintenanceJob::BlockableMaintenanceJob(const vespalib::string &name, BlockableMaintenanceJob::~BlockableMaintenanceJob() { - _moveOpsLimiter->clearJob(); + dynamic_cast<MoveOperationLimiter &>(*_moveOpsLimiter).clearJob(); } bool diff --git a/searchcore/src/vespa/searchcore/proton/server/blockable_maintenance_job.h b/searchcore/src/vespa/searchcore/proton/server/blockable_maintenance_job.h index db7b8d05ca2..b0c69704ebe 100644 --- a/searchcore/src/vespa/searchcore/proton/server/blockable_maintenance_job.h +++ b/searchcore/src/vespa/searchcore/proton/server/blockable_maintenance_job.h @@ -2,7 +2,7 @@ #pragma once #include "i_blockable_maintenance_job.h" -#include "move_operation_limiter.h" +#include "i_move_operation_limiter.h" #include <mutex> #include <unordered_set> @@ -11,6 +11,7 @@ namespace proton { class BlockableMaintenanceJobConfig; class DiskMemUsageState; class IMaintenanceJobRunner; +class IMoveOperationLimiter; /** * Implementation of a maintenance job that can be blocked and unblocked due to various external reasons. @@ -27,12 +28,10 @@ private: bool _blocked; IMaintenanceJobRunner *_runner; double _resourceLimitFactor; + std::shared_ptr<IMoveOperationLimiter> _moveOpsLimiter; void updateBlocked(const LockGuard &guard); - protected: - MoveOperationLimiter::SP _moveOpsLimiter; - void internalNotifyDiskMemUsage(const DiskMemUsageState &state); public: @@ -54,6 +53,7 @@ public: void unBlock(BlockedReason reason) override; bool isBlocked() const override; void registerRunner(IMaintenanceJobRunner *runner) override { _runner = runner; } + IMoveOperationLimiter & getLimiter() { return *_moveOpsLimiter; } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp index 8485cb6835e..cf6ea7f7787 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp @@ -6,6 +6,8 @@ #include "iclusterstatechangednotifier.h" #include "maintenancedocumentsubdb.h" #include "i_disk_mem_usage_notifier.h" +#include "ibucketmodifiedhandler.h" +#include "move_operation_limiter.h" #include <vespa/searchcore/proton/bucketdb/i_bucket_create_notifier.h> #include <vespa/searchcore/proton/documentmetastore/i_document_meta_store.h> @@ -176,7 +178,7 @@ BucketMoveJob(const IBucketStateCalculator::SP &calc, _modifiedHandler(modifiedHandler), _ready(ready), _notReady(notReady), - _mover(*_moveOpsLimiter), + _mover(getLimiter()), _doneScan(false), _scanPos(), _scanPass(FIRST_SCAN_PASS), @@ -186,7 +188,7 @@ BucketMoveJob(const IBucketStateCalculator::SP &calc, _delayedBucketsFrozen(), _frozenBuckets(frozenBuckets), _bucketCreateNotifier(bucketCreateNotifier), - _delayedMover(*_moveOpsLimiter), + _delayedMover(getLimiter()), _clusterStateChangedNotifier(clusterStateChangedNotifier), _bucketStateChangedNotifier(bucketStateChangedNotifier), _diskMemUsageNotifier(diskMemUsageNotifier) @@ -283,8 +285,7 @@ BucketMoveJob::changedCalculator() } bool -BucketMoveJob::scanAndMove(size_t maxBucketsToScan, - size_t maxDocsToMove) +BucketMoveJob::scanAndMove(size_t maxBucketsToScan, size_t maxDocsToMove) { if (done()) { return true; @@ -306,8 +307,7 @@ BucketMoveJob::scanAndMove(size_t maxBucketsToScan, size_t bucketsScanned = 0; for (;;) { if (_mover.bucketDone()) { - ScanResult res = scanBuckets(maxBucketsToScan - - bucketsScanned, bucketGuard); + ScanResult res = scanBuckets(maxBucketsToScan - bucketsScanned, bucketGuard); bucketsScanned += res.first; if (res.second) { if (_scanPass == FIRST_SCAN_PASS && @@ -363,8 +363,7 @@ BucketMoveJob::notifyClusterStateChanged(const IBucketStateCalculator::SP &newCa } void -BucketMoveJob::notifyBucketStateChanged(const BucketId &bucketId, - BucketInfo::ActiveState newState) +BucketMoveJob::notifyBucketStateChanged(const BucketId &bucketId, BucketInfo::ActiveState newState) { // Called by master write thread if (newState == BucketInfo::NOT_ACTIVE) { diff --git a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h index e462b8ac7c6..26755eca7b1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h +++ b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h @@ -6,8 +6,6 @@ #include "documentbucketmover.h" #include "i_disk_mem_usage_listener.h" #include "ibucketfreezelistener.h" -#include "ibucketmodifiedhandler.h" -#include "ibucketstatecalculator.h" #include "ibucketstatechangedhandler.h" #include "iclusterstatechangedhandler.h" #include "ifrozenbuckethandler.h" @@ -15,13 +13,14 @@ #include <vespa/searchcore/proton/bucketdb/i_bucket_create_listener.h> #include <set> -namespace proton -{ +namespace proton { class BlockableMaintenanceJobConfig; class IBucketStateChangedNotifier; class IClusterStateChangedNotifier; class IDiskMemUsageNotifier; +class IBucketModifiedHandler; + namespace bucketdb { class IBucketCreateNotifier; } /** @@ -36,19 +35,16 @@ class BucketMoveJob : public BlockableMaintenanceJob, public IDiskMemUsageListener { public: - struct ScanPosition - { + struct ScanPosition { document::BucketId _lastBucket; ScanPosition() : _lastBucket() { } - ScanPosition(document::BucketId lastBucket) : _lastBucket(lastBucket) { } bool validBucket() const { return _lastBucket.isSet(); } }; typedef BucketDB::ConstMapIterator BucketIterator; - class ScanIterator - { + class ScanIterator { private: BucketDBOwner::Guard _db; BucketIterator _itr; @@ -80,20 +76,20 @@ public: }; private: - typedef std::pair<size_t, bool> ScanResult; - IBucketStateCalculator::SP _calc; - IDocumentMoveHandler &_moveHandler; - IBucketModifiedHandler &_modifiedHandler; - const MaintenanceDocumentSubDB &_ready; - const MaintenanceDocumentSubDB &_notReady; - DocumentBucketMover _mover; - bool _doneScan; - ScanPosition _scanPos; - uint32_t _scanPass; - ScanPosition _endPos; - document::BucketSpace _bucketSpace; - - typedef std::set<document::BucketId> DelayedBucketSet; + using ScanResult = std::pair<size_t, bool>; + std::shared_ptr<IBucketStateCalculator> _calc; + IDocumentMoveHandler &_moveHandler; + IBucketModifiedHandler &_modifiedHandler; + const MaintenanceDocumentSubDB &_ready; + const MaintenanceDocumentSubDB &_notReady; + DocumentBucketMover _mover; + bool _doneScan; + ScanPosition _scanPos; + uint32_t _scanPass; + ScanPosition _endPos; + document::BucketSpace _bucketSpace; + + using DelayedBucketSet = std::set<document::BucketId>; // Delayed buckets that are no longer frozen or active that can be considered for moving. DelayedBucketSet _delayedBuckets; @@ -185,4 +181,3 @@ public: }; } // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.cpp b/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.cpp index 6951125f408..1f2b0cf1a75 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.cpp @@ -12,7 +12,6 @@ using document::BucketId; using document::Document; using document::GlobalId; -using search::DocumentIdT; using storage::spi::Timestamp; namespace proton { @@ -20,9 +19,7 @@ namespace proton { typedef IDocumentMetaStore::Iterator Iterator; bool -DocumentBucketMover::moveDocument(DocumentIdT lid, - const document::GlobalId &gid, - Timestamp timestamp) +DocumentBucketMover::moveDocument(uint32_t lid, const document::GlobalId &gid, Timestamp timestamp) { if ( _source->lidNeedsCommit(lid) ) { return false; @@ -80,13 +77,11 @@ namespace { class MoveKey { public: - DocumentIdT _lid; + uint32_t _lid; document::GlobalId _gid; Timestamp _timestamp; - MoveKey(DocumentIdT lid, - const document::GlobalId &gid, - Timestamp timestamp) + MoveKey(uint32_t lid, const document::GlobalId &gid, Timestamp timestamp) : _lid(lid), _gid(gid), _timestamp(timestamp) @@ -115,7 +110,7 @@ DocumentBucketMover::moveDocuments(size_t maxDocsToMove) typedef std::vector<MoveKey> MoveVec; MoveVec toMove; for (; itr != end && docsMoved < maxDocsToMove; ++itr) { - DocumentIdT lid = itr.getKey().get_lid(); + uint32_t lid = itr.getKey().get_lid(); const RawDocumentMetaData &metaData = _source->meta_store()->getRawMetaData(lid); if (metaData.getBucketUsedBits() != _bucket.getUsedBits()) { ++docsSkipped; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.h b/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.h index eded3a456d8..80e1811268b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.h @@ -4,7 +4,6 @@ #include <vespa/document/bucket/bucketid.h> #include <vespa/document/base/globalid.h> -#include <vespa/searchlib/query/base.h> #include <persistence/spi/types.h> namespace proton { @@ -32,7 +31,7 @@ private: document::GlobalId _lastGid; bool _lastGidValid; - bool moveDocument(search::DocumentIdT lid, + bool moveDocument(uint32_t lid, const document::GlobalId &gid, storage::spi::Timestamp timestamp); diff --git a/searchcore/src/vespa/searchcore/proton/server/i_move_operation_limiter.h b/searchcore/src/vespa/searchcore/proton/server/i_move_operation_limiter.h index d9fc426b08a..cc91413826c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_move_operation_limiter.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_move_operation_limiter.h @@ -11,7 +11,7 @@ namespace proton { * Interface used to limit the number of outstanding move operations a blockable maintenance job can have. */ struct IMoveOperationLimiter { - virtual ~IMoveOperationLimiter() {} + virtual ~IMoveOperationLimiter() = default; virtual std::shared_ptr<vespalib::IDestructorCallback> beginOperation() = 0; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/ibucketmodifiedhandler.h b/searchcore/src/vespa/searchcore/proton/server/ibucketmodifiedhandler.h index 652c303283f..1c23c35e082 100644 --- a/searchcore/src/vespa/searchcore/proton/server/ibucketmodifiedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/ibucketmodifiedhandler.h @@ -10,7 +10,7 @@ class IBucketModifiedHandler { public: virtual void notifyBucketModified(const document::BucketId &bucket) = 0; - virtual ~IBucketModifiedHandler() {} + virtual ~IBucketModifiedHandler() = default; }; } diff --git a/searchcore/src/vespa/searchcore/proton/server/imaintenancejobrunner.h b/searchcore/src/vespa/searchcore/proton/server/imaintenancejobrunner.h index 617aa69631f..f733821a47b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/imaintenancejobrunner.h +++ b/searchcore/src/vespa/searchcore/proton/server/imaintenancejobrunner.h @@ -14,7 +14,7 @@ public: * Schedule job to be run in the future. */ virtual void run() = 0; - virtual ~IMaintenanceJobRunner() { } + virtual ~IMaintenanceJobRunner() = default; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp index e5eebca32eb..095169b84ce 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job.cpp @@ -30,7 +30,7 @@ LidSpaceCompactionJob::scanDocuments(const LidUsageStats &stats) if ( ! op ) { return false; } - vespalib::IDestructorCallback::SP context = _moveOpsLimiter->beginOperation(); + vespalib::IDestructorCallback::SP context = getLimiter().beginOperation(); _opStorer.appendOperation(*op, context); _handler->handleMove(*op, std::move(context)); if (isBlocked(BlockedReason::OUTSTANDING_OPS)) { @@ -39,7 +39,7 @@ LidSpaceCompactionJob::scanDocuments(const LidUsageStats &stats) } } } - return scanDocumentsPost(); + return false; } LidSpaceCompactionJob::LidSpaceCompactionJob(const DocumentDBLidSpaceCompactionConfig &config, diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_base.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_base.cpp index bb6308977ca..d3dae686d2d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_base.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_base.cpp @@ -25,16 +25,16 @@ namespace proton { bool LidSpaceCompactionJobBase::hasTooMuchLidBloat(const LidUsageStats &stats) const { - return (stats.getLidBloat() >= _cfg.getAllowedLidBloat() && - stats.getLidBloatFactor() >= _cfg.getAllowedLidBloatFactor() && - stats.getLidLimit() > stats.getLowestFreeLid()); + return ((stats.getLidBloat() >= _cfg.getAllowedLidBloat()) && + (stats.getLidBloatFactor() >= _cfg.getAllowedLidBloatFactor()) && + (stats.getLidLimit() > stats.getLowestFreeLid())); } bool LidSpaceCompactionJobBase::shouldRestartScanDocuments(const LidUsageStats &stats) const { - return (stats.getUsedLids() + _cfg.getAllowedLidBloat()) < stats.getHighestUsedLid() && - stats.getLowestFreeLid() < stats.getHighestUsedLid(); + return ((stats.getUsedLids() + _cfg.getAllowedLidBloat()) < stats.getHighestUsedLid()) && + (stats.getLowestFreeLid() < stats.getHighestUsedLid()); } DocumentMetaData @@ -43,21 +43,6 @@ LidSpaceCompactionJobBase::getNextDocument(const LidUsageStats &stats, bool retr return _scanItr->next(std::max(stats.getLowestFreeLid(), stats.getUsedLids()), retryLastDocument); } -bool -LidSpaceCompactionJobBase::scanDocumentsPost() -{ - if (!_scanItr->valid()) { - sync(); - if (shouldRestartScanDocuments(_handler->getLidStatus())) { - _scanItr = _handler->getIterator(); - } else { - _scanItr = IDocumentScanIterator::UP(); - _shouldCompactLidSpace = true; - } - } - return false; // more work to do (scan documents or compact lid space) -} - void LidSpaceCompactionJobBase::compactLidSpace(const LidUsageStats &stats) { @@ -143,6 +128,16 @@ LidSpaceCompactionJobBase::run() _handler->getName().c_str()); _is_disabled = false; } + + if (_scanItr && !_scanItr->valid()) { + if (shouldRestartScanDocuments(_handler->getLidStatus())) { + _scanItr = _handler->getIterator(); + } else { + _scanItr = IDocumentScanIterator::UP(); + _shouldCompactLidSpace = true; + } + } + if (_scanItr) { return scanDocuments(stats); } else if (_shouldCompactLidSpace) { diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_base.h b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_base.h index dc9ecff22cc..759a18361f7 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_base.h +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_base.h @@ -52,8 +52,6 @@ private: bool remove_is_ongoing() const; protected: search::DocumentMetaData getNextDocument(const search::LidUsageStats &stats, bool retryLastDocument); - bool scanDocumentsPost(); - virtual void sync() { } public: LidSpaceCompactionJobBase(const DocumentDBLidSpaceCompactionConfig &config, std::shared_ptr<ILidSpaceCompactionHandler> handler, diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_take2.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_take2.cpp index faeb5da1f38..cbf3de20b1f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_take2.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_take2.cpp @@ -27,7 +27,7 @@ CompactionJob::scanDocuments(const LidUsageStats &stats) DocumentMetaData document = getNextDocument(stats, false); if (document.valid()) { Bucket metaBucket(document::Bucket(_bucketSpace, document.bucketId)); - IDestructorCallback::SP context = _moveOpsLimiter->beginOperation(); + IDestructorCallback::SP context = getLimiter().beginOperation(); auto failed = _bucketExecutor.execute(metaBucket, makeBucketTask([this, meta=document, opsTracker=std::move(context)] (const Bucket & bucket, std::shared_ptr<IDestructorCallback> onDone) { assert(bucket.getBucketId() == meta.bucketId); using DoneContext = vespalib::KeepAlive<std::pair<IDestructorCallback::SP, IDestructorCallback::SP>>; @@ -39,7 +39,10 @@ CompactionJob::scanDocuments(const LidUsageStats &stats) } } } - return scanDocumentsPost(); + if (!_scanItr->valid()) { + sync(); + } + return false; } void diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_take2.h b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_take2.h index e5619720825..21239532e66 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_take2.h +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_job_take2.h @@ -7,6 +7,7 @@ namespace storage::spi { struct BucketExecutor;} namespace searchcorespi::index { class IThreadService; } +namespace vespalib { class IDestructorCallback; } namespace proton { class IDiskMemUsageNotifier; class IClusterStateChangedNotifier; @@ -31,7 +32,7 @@ private: bool scanDocuments(const search::LidUsageStats &stats) override; void moveDocument(const search::DocumentMetaData & meta, std::shared_ptr<IDestructorCallback> onDone); void onStop() override; - void sync() override; + void sync(); public: CompactionJob(const DocumentDBLidSpaceCompactionConfig &config, std::shared_ptr<ILidSpaceCompactionHandler> handler, diff --git a/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.cpp b/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.cpp index 42fe492539c..e3d565afb17 100644 --- a/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.cpp @@ -57,6 +57,13 @@ MoveOperationLimiter::isAboveLimit() const return (_outstandingOps >= _maxOutstandingOps); } +bool +MoveOperationLimiter::hasPending() const +{ + LockGuard guard(_mutex); + return (_outstandingOps > 0); +} + std::shared_ptr<vespalib::IDestructorCallback> MoveOperationLimiter::beginOperation() { diff --git a/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.h b/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.h index 04440a7451d..b5c8b1b9998 100644 --- a/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.h +++ b/searchcore/src/vespa/searchcore/proton/server/move_operation_limiter.h @@ -35,9 +35,10 @@ private: public: using SP = std::shared_ptr<MoveOperationLimiter>; MoveOperationLimiter(IBlockableMaintenanceJob *job, uint32_t maxOutstandingOps); - ~MoveOperationLimiter(); + ~MoveOperationLimiter() override; void clearJob(); bool isAboveLimit() const; + bool hasPending() const; std::shared_ptr<vespalib::IDestructorCallback> beginOperation() override; }; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index 67fa22ada03..a46b4205570 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -78,7 +78,8 @@ FileStorManager(const config::ConfigUri & configUri, spi::PersistenceProvider& p _metrics(std::make_unique<FileStorMetrics>()), _closed(false), _lock(), - _host_info_reporter(_component.getStateUpdater()) + _host_info_reporter(_component.getStateUpdater()), + _resource_usage_listener_registration(provider.register_resource_usage_listener(_host_info_reporter)) { _configFetcher.subscribe(configUri.getConfigId(), this); _configFetcher.start(); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h index ae298d70a29..6eaef45e9bd 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h @@ -77,6 +77,7 @@ class FileStorManager : public StorageLinkQueued, std::mutex _lock; std::unique_ptr<vespalib::IDestructorCallback> _bucketExecutorRegistration; ServiceLayerHostInfoReporter _host_info_reporter; + std::unique_ptr<vespalib::IDestructorCallback> _resource_usage_listener_registration; public: FileStorManager(const config::ConfigUri &, spi::PersistenceProvider&, diff --git a/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java b/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java index c9acd625373..accf3942e4d 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java @@ -9,6 +9,7 @@ import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; +import com.yahoo.vespa.config.content.DistributionConfig; import com.yahoo.vespa.config.content.StorDistributionConfig; import com.yahoo.document.BucketId; @@ -40,7 +41,6 @@ public class Distribution { private final boolean distributorAutoOwnershipTransferOnWholeGroupDown; } - private final int[] distributionBitMasks = new int[65]; private ConfigSubscriber configSub; private final AtomicReference<Config> config = new AtomicReference<>(new Config(null, 1, false)); @@ -52,67 +52,103 @@ public class Distribution { return config.getAcquire().redundancy; } - private ConfigSubscriber.SingleSubscriber<StorDistributionConfig> configSubscriber = new ConfigSubscriber.SingleSubscriber<>() { - private int[] getGroupPath(String path) { - if (path.equals("invalid")) { return new int[0]; } - StringTokenizer st = new StringTokenizer(path, "."); - int[] p = new int[st.countTokens()]; - for (int i=0; i<p.length; ++i) { - p[i] = Integer.valueOf(st.nextToken()); - } - return p; + private static int[] getGroupPath(String path) { + if (path.equals("invalid")) { return new int[0]; } + StringTokenizer st = new StringTokenizer(path, "."); + int[] p = new int[st.countTokens()]; + for (int i=0; i<p.length; ++i) { + p[i] = Integer.valueOf(st.nextToken()); } + return p; + } - @Override - public void configure(StorDistributionConfig config) { - try{ - Group root = null; - for (int i=0; i<config.group().size(); ++i) { - StorDistributionConfig.Group cg = config.group().get(i); - int[] path = new int[0]; - if (root != null) { - path = getGroupPath(cg.index()); - } - boolean isLeafGroup = (cg.nodes().size() > 0); - Group group; - int index = (path.length == 0 ? 0 : path[path.length - 1]); - if (isLeafGroup) { - group = new Group(index, cg.name()); - List<ConfiguredNode> nodes = new ArrayList<>(); - for (StorDistributionConfig.Group.Nodes node : cg.nodes()) { - nodes.add(new ConfiguredNode(node.index(), node.retired())); - } - group.setNodes(nodes); - } else { - group = new Group(index, cg.name(), new Group.Distribution(cg.partitions(), config.redundancy())); + // NOTE: keep in sync with the below + private ConfigSubscriber.SingleSubscriber<StorDistributionConfig> configSubscriber = config -> { + try { + Group root = null; + for (int i=0; i<config.group().size(); ++i) { + StorDistributionConfig.Group cg = config.group(i); + int[] path = new int[0]; + if (root != null) { + path = getGroupPath(cg.index()); + } + boolean isLeafGroup = (cg.nodes().size() > 0); + Group group; + int index = (path.length == 0 ? 0 : path[path.length - 1]); + if (isLeafGroup) { + group = new Group(index, cg.name()); + List<ConfiguredNode> nodes = new ArrayList<>(); + for (StorDistributionConfig.Group.Nodes node : cg.nodes()) { + nodes.add(new ConfiguredNode(node.index(), node.retired())); } - group.setCapacity(cg.capacity()); - if (path.length == 0) { - root = group; - } else { - Group parent = root; - for (int j=0; j<path.length - 1; ++j) { - parent = parent.getSubgroups().get(path[j]); - } - parent.addSubGroup(group); + group.setNodes(nodes); + } else { + group = new Group(index, cg.name(), new Group.Distribution(cg.partitions(), config.redundancy())); + } + group.setCapacity(cg.capacity()); + if (path.length == 0) { + root = group; + } else { + Group parent = root; + for (int j=0; j<path.length - 1; ++j) { + parent = parent.getSubgroups().get(path[j]); } + parent.addSubGroup(group); } - if (root == null) - throw new IllegalStateException("Config does not specify a root group"); - root.calculateDistributionHashValues(); - Distribution.this.config.setRelease(new Config(root, config.redundancy(), config.distributor_auto_ownership_transfer_on_whole_group_down())); - } catch (ParseException e) { - throw new IllegalStateException("Failed to parse config", e); } + if (root == null) + throw new IllegalStateException("Config does not specify a root group"); + root.calculateDistributionHashValues(); + Distribution.this.config.setRelease(new Config(root, config.redundancy(), config.distributor_auto_ownership_transfer_on_whole_group_down())); + } catch (ParseException e) { + throw new IllegalStateException("Failed to parse config", e); } }; - public Distribution(String configId) { - int mask = 0; - for (int i=0; i<=64; ++i) { - distributionBitMasks[i] = mask; - mask = (mask << 1) | 1; + // TODO jonmv: De-dupe with this.configSubscriber once common config is used + private void configure(DistributionConfig.Cluster config) { + try { + Group root = null; + for (int i=0; i<config.group().size(); ++i) { + DistributionConfig.Cluster.Group cg = config.group(i); + int[] path = new int[0]; + if (root != null) { + path = getGroupPath(cg.index()); + } + boolean isLeafGroup = (cg.nodes().size() > 0); + Group group; + int index = (path.length == 0 ? 0 : path[path.length - 1]); + if (isLeafGroup) { + group = new Group(index, cg.name()); + List<ConfiguredNode> nodes = new ArrayList<>(); + for (DistributionConfig.Cluster.Group.Nodes node : cg.nodes()) { + nodes.add(new ConfiguredNode(node.index(), node.retired())); + } + group.setNodes(nodes); + } else { + group = new Group(index, cg.name(), new Group.Distribution(cg.partitions(), config.redundancy())); + } + group.setCapacity(cg.capacity()); + if (path.length == 0) { + root = group; + } else { + Group parent = root; + for (int j=0; j<path.length - 1; ++j) { + parent = parent.getSubgroups().get(path[j]); + } + parent.addSubGroup(group); + } + } + if (root == null) + throw new IllegalStateException("Config does not specify a root group"); + root.calculateDistributionHashValues(); + Distribution.this.config.setRelease(new Config(root, config.redundancy(), true)); + } catch (ParseException e) { + throw new IllegalStateException("Failed to parse config", e); } + } + + public Distribution(String configId) { try { configSub = new ConfigSubscriber(); configSub.subscribe(configSubscriber, StorDistributionConfig.class, configId); @@ -123,14 +159,20 @@ public class Distribution { } public Distribution(StorDistributionConfig config) { - int mask = 0; - for (int i=0; i<=64; ++i) { - distributionBitMasks[i] = mask; - mask = (mask << 1) | 1; - } configSubscriber.configure(config); } + public Distribution(DistributionConfig.Cluster config) { + configure(config); + } + + private static long lastNBits(long value, int n) { + if (n < 0 || n > 63) + throw new IllegalArgumentException("n must be in [0, 63], but was " + n); + + return value & ((1L << n) - 1); + } + public void close() { if (configSub!=null) { configSub.close(); @@ -140,22 +182,21 @@ public class Distribution { } private int getGroupSeed(BucketId bucket, ClusterState state, Group group) { - int seed = ((int) bucket.getRawId()) & distributionBitMasks[state.getDistributionBitCount()]; + int seed = (int) lastNBits(bucket.getRawId(), state.getDistributionBitCount()); seed ^= group.getDistributionHash(); return seed; } private int getDistributorSeed(BucketId bucket, ClusterState state) { - return ((int) bucket.getRawId()) & distributionBitMasks[state.getDistributionBitCount()]; + return (int) lastNBits(bucket.getRawId(), state.getDistributionBitCount()); } private int getStorageSeed(BucketId bucket, ClusterState state) { - int seed = ((int) bucket.getRawId()) & distributionBitMasks[state.getDistributionBitCount()]; + int seed = (int) lastNBits(bucket.getRawId(), state.getDistributionBitCount()); if (bucket.getUsedBits() > 33) { int usedBits = bucket.getUsedBits() - 1; - seed ^= (distributionBitMasks[usedBits - 32] - & (bucket.getRawId() >> 32)) << 6; + seed ^= lastNBits(bucket.getRawId() >> 32, usedBits - 32) << 6; } return seed; } @@ -172,6 +213,7 @@ public class Distribution { return Double.compare(o.score, score); } } + private static class ScoredNode { int index; int reliability; @@ -179,6 +221,7 @@ public class Distribution { ScoredNode(int index, int reliability, double score) { this.index = index; this.reliability = reliability; this.score = score; } } + private static boolean allDistributorsDown(Group g, ClusterState clusterState) { if (g.isLeafGroup()) { for (ConfiguredNode node : g.getNodes()) { @@ -192,6 +235,7 @@ public class Distribution { } return true; } + private Group getIdealDistributorGroup(boolean distributorAutoOwnershipTransferOnWholeGroupDown, BucketId bucket, ClusterState clusterState, Group parent, int redundancy) { if (parent.isLeafGroup()) { @@ -220,6 +264,7 @@ public class Distribution { } return getIdealDistributorGroup(distributorAutoOwnershipTransferOnWholeGroupDown, bucket, clusterState, results.first().group, redundancyArray[0]); } + private static class ResultGroup implements Comparable<ResultGroup> { Group group; int redundancy; @@ -234,6 +279,7 @@ public class Distribution { return group.compareTo(o.group); } } + private void getIdealGroups(BucketId bucketId, ClusterState clusterState, Group parent, int redundancy, List<ResultGroup> results) { if (parent.isLeafGroup()) { @@ -424,11 +470,13 @@ public class Distribution { super(message); } } + public static class NoDistributorsAvailableException extends Exception { NoDistributorsAvailableException(String message) { super(message); } } + public int getIdealDistributorNode(ClusterState state, BucketId bucket, String upStates) throws TooFewBucketBitsInUseException, NoDistributorsAvailableException { if (bucket.getUsedBits() < state.getDistributionBitCount()) { throw new TooFewBucketBitsInUseException("Cannot get ideal state for bucket " + bucket + " using " + bucket.getUsedBits() @@ -474,6 +522,7 @@ public class Distribution { } return node.index; } + private boolean visitGroups(GroupVisitor visitor, Map<Integer, Group> groups) { for (Group g : groups.values()) { if (!visitor.visitGroup(g)) return false; @@ -485,12 +534,14 @@ public class Distribution { } return true; } + public void visitGroups(GroupVisitor visitor) { Map<Integer, Group> groups = new TreeMap<>(); Group nodeGraph = config.getAcquire().nodeGraph; groups.put(nodeGraph.getIndex(), nodeGraph); visitGroups(visitor, groups); } + public Set<ConfiguredNode> getNodes() { final Set<ConfiguredNode> nodes = new HashSet<>(); GroupVisitor visitor = new GroupVisitor() { @@ -524,9 +575,11 @@ public class Distribution { sb.append("disk_distribution ").append(diskDistribution.toString()).append("\n"); return sb.toString(); } + public static String getSimpleGroupConfig(int redundancy, int nodeCount) { return getSimpleGroupConfig(redundancy, nodeCount, StorDistributionConfig.Disk_distribution.Enum.MODULO_BID); } + private static String getSimpleGroupConfig(int redundancy, int nodeCount, StorDistributionConfig.Disk_distribution.Enum diskDistribution) { StringBuilder sb = new StringBuilder(); sb.append("raw:redundancy ").append(redundancy).append("\n").append("group[4]\n"); @@ -561,6 +614,5 @@ public class Distribution { sb.append("disk_distribution ").append(diskDistribution.toString()).append("\n"); return sb.toString(); } -} - +} diff --git a/vdslib/src/main/java/com/yahoo/vdslib/distribution/Group.java b/vdslib/src/main/java/com/yahoo/vdslib/distribution/Group.java index 680021893f7..458ab6e291c 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/distribution/Group.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/distribution/Group.java @@ -196,7 +196,7 @@ public class Group implements Comparable<Group> { /** * The distribution class keeps precalculated arrays for distributions for all legal redundancies. The class is - * immutable, such that it can be returned safely out from the group object. + * immutable, such that it can be returned safely out from the group object. (Actually, it's not immutable.) */ public static class Distribution { diff --git a/vdslib/src/test/java/com/yahoo/vdslib/distribution/CrossPlatformTestFactory.java b/vdslib/src/test/java/com/yahoo/vdslib/distribution/CrossPlatformTestFactory.java index 94f7d7a8c94..7dadd9560b5 100644 --- a/vdslib/src/test/java/com/yahoo/vdslib/distribution/CrossPlatformTestFactory.java +++ b/vdslib/src/test/java/com/yahoo/vdslib/distribution/CrossPlatformTestFactory.java @@ -1,7 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vdslib.distribution; -import java.io.*; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; /** * Helper class to implement unit tests that should produce the same result in different implementations. diff --git a/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestCase.java index 8c877704169..0d34cd70953 100644 --- a/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestCase.java +++ b/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestCase.java @@ -418,4 +418,5 @@ public class DistributionTestCase { Distribution distr = new Distribution(new StorDistributionConfig(config)); distr.getIdealDistributorNode(clusterState, new BucketId(16, 0), "uim"); } + } diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java index 70737859342..fc4b3e74b29 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java @@ -90,19 +90,29 @@ public class JunitRunner extends AbstractComponent implements TestRunner { if (execution != null && !execution.isDone()) { throw new IllegalStateException("Test execution already in progress"); } - testRuntimeProvider.initialize(testConfig); - Optional<Bundle> testBundle = findTestBundle(); - if (testBundle.isEmpty()) { - throw new RuntimeException("No test bundle available"); - } + try { + testRuntimeProvider.initialize(testConfig); + Optional<Bundle> testBundle = findTestBundle(); + if (testBundle.isEmpty()) { + throw new RuntimeException("No test bundle available"); + } - Optional<TestDescriptor> testDescriptor = loadTestDescriptor(testBundle.get()); - if (testDescriptor.isEmpty()) { - throw new RuntimeException("Could not find test descriptor"); + Optional<TestDescriptor> testDescriptor = loadTestDescriptor(testBundle.get()); + if (testDescriptor.isEmpty()) { + throw new RuntimeException("Could not find test descriptor"); + } + execution = CompletableFuture.supplyAsync(() -> launchJunit(loadClasses(testBundle.get(), testDescriptor.get(), category))); + } catch (Exception e) { + execution = createReportWithFailedInitialization(e); } - List<Class<?>> testClasses = loadClasses(testBundle.get(), testDescriptor.get(), category); + } - execution = CompletableFuture.supplyAsync(() -> launchJunit(testClasses)); + private static Future<TestReport> createReportWithFailedInitialization(Exception exception) { + TestReport.Failure failure = new TestReport.Failure("init", exception); + return CompletableFuture.completedFuture( + new TestReport.Builder() + .withFailures(List.of(failure)) + .build()); } @Override diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedErrorMessage.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedErrorMessage.java deleted file mode 100644 index 1d7e8535909..00000000000 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedErrorMessage.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.http.server; - -import com.yahoo.document.DocumentId; -import com.yahoo.messagebus.Message; -import com.yahoo.text.Utf8String; - -import java.util.Arrays; - -public class FeedErrorMessage extends Message { - - private long sequenceId; - - public FeedErrorMessage(String operationId) { - try { - DocumentId id = new DocumentId(operationId); - sequenceId = Arrays.hashCode(id.getGlobalId()); - } catch (Exception e) { - sequenceId = 0; - } - } - - @Override - public Utf8String getProtocol() { - return new Utf8String("vespa-feed-handler-internal-bogus-protocol"); - } - - @Override - public int getType() { - return 1234; - } - - @Override - public boolean hasSequenceId() { - return true; - } - - @Override - public long getSequenceId() { - return sequenceId; - } - -} |