diff options
172 files changed, 1798 insertions, 1477 deletions
diff --git a/clustercontroller-core/pom.xml b/clustercontroller-core/pom.xml index fadc6c2a3c2..df34272ef9c 100644 --- a/clustercontroller-core/pom.xml +++ b/clustercontroller-core/pom.xml @@ -125,44 +125,9 @@ <artifactId>maven-surefire-plugin</artifactId> <configuration> <forkCount>4</forkCount> + <rerunFailingTestsCount>3</rerunFailingTestsCount> </configuration> </plugin> </plugins> </build> - <profiles> - <profile> - <id>systemtests</id> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <includes> - <include>**/DatabaseTest.java</include> - <include>**/MasterElectionTest.java</include> - </includes> - </configuration> - </plugin> - </plugins> - </build> - </profile> - <profile> - <id>factory-tests</id> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <excludes> - <exclude>**/DatabaseTest.java</exclude> - <exclude>**/MasterElectionTest.java</exclude> - </excludes> - </configuration> - </plugin> - </plugins> - </build> - </profile> - </profiles> </project> diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterInfo.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterInfo.java index c25dd5a5965..2cfaf64fe83 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterInfo.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterInfo.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.vdslib.distribution.ConfiguredNode; import com.yahoo.vdslib.distribution.Distribution; import com.yahoo.vdslib.state.Node; +import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; import java.util.Collection; import java.util.Collections; @@ -40,11 +41,11 @@ public class ClusterInfo { /** Returns information about the given node id, or null if this node does not exist */ public NodeInfo getNodeInfo(Node node) { return allNodeInfo.get(node); } - Collection<DistributorNodeInfo> getDistributorNodeInfo() { return Collections.unmodifiableCollection(distributorNodeInfo.values()); } + Collection<DistributorNodeInfo> getDistributorNodeInfos() { return Collections.unmodifiableCollection(distributorNodeInfo.values()); } - Collection<StorageNodeInfo> getStorageNodeInfo() { return Collections.unmodifiableCollection(storageNodeInfo.values()); } + Collection<StorageNodeInfo> getStorageNodeInfos() { return Collections.unmodifiableCollection(storageNodeInfo.values()); } - Collection<NodeInfo> getAllNodeInfo() { return Collections.unmodifiableCollection(allNodeInfo.values()); } + Collection<NodeInfo> getAllNodeInfos() { return Collections.unmodifiableCollection(allNodeInfo.values()); } /** Returns the configured nodes of this as a read-only map indexed on node index (distribution key) */ Map<Integer, ConfiguredNode> getConfiguredNodes() { return Collections.unmodifiableMap(nodes); } @@ -52,15 +53,23 @@ public class ClusterInfo { boolean hasConfiguredNode(int index) { return nodes.containsKey(index); } /** Sets the nodes which belongs to this cluster */ - void setNodes(Collection<ConfiguredNode> newNodes, ContentCluster owner, Distribution distribution) { + void setNodes(Collection<ConfiguredNode> newNodes, ContentCluster owner, + Distribution distribution, NodeListener nodeListener) { // Remove info for removed nodes Set<ConfiguredNode> newNodesSet = new HashSet<>(newNodes); for (ConfiguredNode existingNode : this.nodes.values()) { if ( ! newNodesSet.contains(existingNode)) { - Node existingStorageNode = storageNodeInfo.remove(existingNode.index()).getNode(); - Node existingDistributorNode = distributorNodeInfo.remove(existingNode.index()).getNode(); - allNodeInfo.remove(existingDistributorNode); - allNodeInfo.remove(existingStorageNode); + { + Node existingStorageNode = storageNodeInfo.remove(existingNode.index()).getNode(); + allNodeInfo.remove(existingStorageNode); + nodeListener.handleRemovedNode(existingStorageNode); + } + + { + Node existingDistributorNode = distributorNodeInfo.remove(existingNode.index()).getNode(); + allNodeInfo.remove(existingDistributorNode); + nodeListener.handleRemovedNode(existingDistributorNode); + } } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGenerator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGenerator.java index f984c3cb3a2..75c6dbe6cec 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGenerator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateGenerator.java @@ -129,7 +129,7 @@ public class ClusterStateGenerator { final ClusterState workingState = ClusterState.emptyState(); final Map<Node, NodeStateReason> nodeStateReasons = new HashMap<>(); - for (final NodeInfo nodeInfo : cluster.getNodeInfo()) { + for (final NodeInfo nodeInfo : cluster.getNodeInfos()) { final NodeState nodeState = computeEffectiveNodeState(nodeInfo, params, nodeStateReasons); workingState.setNodeState(nodeInfo.getNode(), nodeState); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java index fabc4999fe5..f2a4a9736c3 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java @@ -10,6 +10,7 @@ import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; +import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; import com.yahoo.vespa.clustercontroller.core.status.statuspage.HtmlTable; import com.yahoo.vespa.clustercontroller.core.status.statuspage.VdsClusterHtmlRenderer; import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest; @@ -42,7 +43,7 @@ public class ContentCluster { if (configuredNodes == null) throw new IllegalArgumentException("Nodes must be set"); this.clusterName = clusterName; this.distribution = distribution; - setNodes(configuredNodes); + setNodes(configuredNodes, new NodeListener() {}); } // TODO move out, this doesn't belong in a domain model class @@ -104,14 +105,14 @@ public class ContentCluster { public void setDistribution(Distribution distribution) { this.distribution = distribution; - for (NodeInfo info : clusterInfo.getAllNodeInfo()) { + for (NodeInfo info : clusterInfo.getAllNodeInfos()) { info.setGroup(distribution); } } /** Sets the configured nodes of this cluster */ - public final void setNodes(Collection<ConfiguredNode> configuredNodes) { - clusterInfo.setNodes(configuredNodes, this, distribution); + public final void setNodes(Collection<ConfiguredNode> configuredNodes, NodeListener nodeListener) { + clusterInfo.setNodes(configuredNodes, this, distribution, nodeListener); } public void setStartTimestamp(Node n, long startTimestamp) { @@ -128,7 +129,7 @@ public class ContentCluster { } public void clearStates() { - for (NodeInfo info : clusterInfo.getAllNodeInfo()) { + for (NodeInfo info : clusterInfo.getAllNodeInfos()) { info.setReportedState(null, 0); } } @@ -145,8 +146,8 @@ public class ContentCluster { return clusterInfo.getConfiguredNodes(); } - public Collection<NodeInfo> getNodeInfo() { - return Collections.unmodifiableCollection(clusterInfo.getAllNodeInfo()); + public Collection<NodeInfo> getNodeInfos() { + return Collections.unmodifiableCollection(clusterInfo.getAllNodeInfos()); } public ClusterInfo clusterInfo() { return clusterInfo; } @@ -158,7 +159,7 @@ public class ContentCluster { public String toString() { StringBuilder sb = new StringBuilder(); sb.append("ContentCluster(").append(clusterName).append(") {"); - for (NodeInfo node : clusterInfo.getAllNodeInfo()) { + for (NodeInfo node : clusterInfo.getAllNodeInfos()) { sb.append("\n ").append(node); } sb.append("\n}"); @@ -197,14 +198,14 @@ public class ContentCluster { switch (state) { case MAINTENANCE: // Orchestrator's ALLOWED_TO_BE_DOWN case DOWN: // Orchestrator's PERMANENTLY_DOWN - return clusterInfo.getStorageNodeInfo().stream() - .filter(storageNodeInfo -> { + return clusterInfo.getStorageNodeInfos().stream() + .filter(storageNodeInfo -> { NodeState userWantedState = storageNodeInfo.getUserWantedState(); return userWantedState.getState() == state && Objects.equals(userWantedState.getDescription(), ORCHESTRATOR_RESERVED_DESCRIPTION); }) - .map(NodeInfo::getNodeIndex) - .collect(Collectors.toList()); + .map(NodeInfo::getNodeIndex) + .collect(Collectors.toList()); default: // Note: There is no trace left if the Orchestrator set the state to UP, so that's handled // like any other state: diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index 3137dfff606..7f385c6077c 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -12,8 +12,8 @@ import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler; import com.yahoo.vespa.clustercontroller.core.database.ZooKeeperDatabaseFactory; import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeAddedOrRemovedListener; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeStateOrHostInfoChangeHandler; +import com.yahoo.vespa.clustercontroller.core.listeners.SlobrokListener; +import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; import com.yahoo.vespa.clustercontroller.core.listeners.SystemStateListener; import com.yahoo.vespa.clustercontroller.core.rpc.RPCCommunicator; import com.yahoo.vespa.clustercontroller.core.rpc.RpcServer; @@ -47,7 +47,7 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; -public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAddedOrRemovedListener, SystemStateListener, +public class FleetController implements NodeListener, SlobrokListener, SystemStateListener, Runnable, RemoteClusterControllerTaskScheduler { private static final Logger logger = Logger.getLogger(FleetController.class.getName()); @@ -332,6 +332,13 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd } @Override + public void handleRemovedNode(Node node) { + verifyInControllerThread(); + // Prune orphaned wanted states + wantedStateChanged = true; + } + + @Override public void handleUpdatedHostInfo(NodeInfo nodeInfo, HostInfo newHostInfo) { verifyInControllerThread(); triggerBundleRecomputationIfResourceExhaustionStateChanged(nodeInfo, newHostInfo); @@ -380,7 +387,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd ClusterState baselineState = stateBundle.getBaselineClusterState(); newStates.add(stateBundle); metricUpdater.updateClusterStateMetrics(cluster, baselineState, - ResourceUsageStats.calculateFrom(cluster.getNodeInfo(), options.clusterFeedBlockLimit, stateBundle.getFeedBlock())); + ResourceUsageStats.calculateFrom(cluster.getNodeInfos(), options.clusterFeedBlockLimit, stateBundle.getFeedBlock())); lastMetricUpdateCycleCount = cycleCount; systemStateBroadcaster.handleNewClusterStates(stateBundle); // Iff master, always store new version in ZooKeeper _before_ publishing to any @@ -399,7 +406,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd ClusterStateBundle stateBundle = stateVersionTracker.getVersionedClusterStateBundle(); ClusterState baselineState = stateBundle.getBaselineClusterState(); metricUpdater.updateClusterStateMetrics(cluster, baselineState, - ResourceUsageStats.calculateFrom(cluster.getNodeInfo(), options.clusterFeedBlockLimit, stateBundle.getFeedBlock())); + ResourceUsageStats.calculateFrom(cluster.getNodeInfos(), options.clusterFeedBlockLimit, stateBundle.getFeedBlock())); lastMetricUpdateCycleCount = cycleCount; return true; } else { @@ -511,7 +518,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd eventLog.setMaxSize(options.eventLogMaxSize, options.eventNodeLogMaxSize); cluster.setPollingFrequency(options.statePollingFrequency); cluster.setDistribution(options.storageDistribution); - cluster.setNodes(options.nodes); + cluster.setNodes(options.nodes, databaseContext.getNodeStateUpdateListener()); database.setZooKeeperAddress(options.zooKeeperServerAddress, databaseContext); database.setZooKeeperSessionTimeout(options.zooKeeperSessionTimeout, databaseContext); stateGatherer.setMaxSlobrokDisconnectGracePeriod(options.maxSlobrokDisconnectGracePeriod); @@ -790,8 +797,8 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd @Override public boolean inMasterMoratorium() { return inMasterMoratorium; } }; - context.nodeStateOrHostInfoChangeHandler = this; - context.nodeAddedOrRemovedListener = this; + context.nodeListener = this; + context.slobrokListener = this; return context; } @@ -806,10 +813,10 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd if (bundle == null) { return List.of(); } - return cluster.getNodeInfo().stream(). - filter(n -> effectiveActivatedStateVersion(n, bundle) < version). - map(NodeInfo::getNode). - collect(Collectors.toList()); + return cluster.getNodeInfos().stream(). + filter(n -> effectiveActivatedStateVersion(n, bundle) < version). + map(NodeInfo::getNode). + collect(Collectors.toList()); } private static <E> String stringifyListWithLimits(List<E> list, int limit) { @@ -939,7 +946,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd .stateDeriver(createBucketSpaceStateDeriver()) .deferredActivation(options.enableTwoPhaseClusterStateActivation) .feedBlock(createResourceExhaustionCalculator() - .inferContentClusterFeedBlockOrNull(cluster.getNodeInfo())) + .inferContentClusterFeedBlockOrNull(cluster.getNodeInfos())) .deriveAndBuild(); stateVersionTracker.updateLatestCandidateStateBundle(candidateBundle); invokeCandidateStateListeners(candidateBundle); @@ -1095,7 +1102,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd didWork = true; } if (wantedStateChanged) { - database.saveWantedStates(databaseContext); + didWork |= database.saveWantedStates(databaseContext); wantedStateChanged = false; } } else { @@ -1150,9 +1157,9 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd @Override public FleetController getFleetController() { return FleetController.this; } @Override - public NodeAddedOrRemovedListener getNodeAddedOrRemovedListener() { return FleetController.this; } + public SlobrokListener getNodeAddedOrRemovedListener() { return FleetController.this; } @Override - public NodeStateOrHostInfoChangeHandler getNodeStateUpdateListener() { return FleetController.this; } + public NodeListener getNodeStateUpdateListener() { return FleetController.this; } }; public void waitForCompleteCycle(long timeoutMS) { @@ -1183,7 +1190,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd synchronized (monitor) { while (true) { int ackedNodes = 0; - for (NodeInfo node : cluster.getNodeInfo()) { + for (NodeInfo node : cluster.getNodeInfos()) { if (node.getClusterStateVersionBundleAcknowledged() >= version) { ++ackedNodes; } @@ -1206,7 +1213,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd synchronized (monitor) { while (true) { int distCount = 0, storCount = 0; - for (NodeInfo info : cluster.getNodeInfo()) { + for (NodeInfo info : cluster.getNodeInfos()) { if (!info.isRpcAddressOutdated()) { if (info.isDistributor()) ++distCount; else ++storCount; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeLookup.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeLookup.java index cdf8b24e72d..882ae8894fa 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeLookup.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeLookup.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeAddedOrRemovedListener; +import com.yahoo.vespa.clustercontroller.core.listeners.SlobrokListener; /** * Interface for a node lookup service, such as slobrok, config, or tier controller. @@ -10,7 +10,7 @@ public interface NodeLookup { void shutdown(); - boolean updateCluster(ContentCluster cluster, NodeAddedOrRemovedListener listener); + boolean updateCluster(ContentCluster cluster, SlobrokListener listener); /** * Returns whether the lookup instance has been able to bootstrap itself with information about nodes. diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeChecker.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeChecker.java index eb0368749f0..3ed03e94fda 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeChecker.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeChecker.java @@ -359,7 +359,7 @@ public class NodeStateChangeChecker { // This method verifies both storage nodes and distributors are up (or retired). // The complicated part is making a summary error message. - for (NodeInfo storageNodeInfo : clusterInfo.getStorageNodeInfo()) { + for (NodeInfo storageNodeInfo : clusterInfo.getStorageNodeInfos()) { State wantedState = storageNodeInfo.getUserWantedState().getState(); if (wantedState != State.UP && wantedState != State.RETIRED) { return Result.createDisallowed("Another storage node wants state " + @@ -373,7 +373,7 @@ public class NodeStateChangeChecker { } } - for (NodeInfo distributorNodeInfo : clusterInfo.getDistributorNodeInfo()) { + for (NodeInfo distributorNodeInfo : clusterInfo.getDistributorNodeInfos()) { State wantedState = distributorNodeInfo.getUserWantedState().getState(); if (wantedState != State.UP && wantedState != State.RETIRED) { return Result.createDisallowed("Another distributor wants state " + wantedState.toString().toUpperCase() + @@ -418,10 +418,10 @@ public class NodeStateChangeChecker { * @param clusterStateVersion the cluster state we expect distributors to have */ private Result checkDistributors(Node node, int clusterStateVersion) { - if (clusterInfo.getDistributorNodeInfo().isEmpty()) { + if (clusterInfo.getDistributorNodeInfos().isEmpty()) { return Result.createDisallowed("Not aware of any distributors, probably not safe to upgrade?"); } - for (DistributorNodeInfo distributorNodeInfo : clusterInfo.getDistributorNodeInfo()) { + for (DistributorNodeInfo distributorNodeInfo : clusterInfo.getDistributorNodeInfos()) { Integer distributorClusterStateVersion = distributorNodeInfo.getHostInfo().getClusterStateVersionOrNull(); if (distributorClusterStateVersion == null) { return Result.createDisallowed("Distributor node " + distributorNodeInfo.getNodeIndex() diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java index e3fe371c05e..69e97de84f9 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java @@ -7,7 +7,7 @@ import java.util.logging.Level; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeStateOrHostInfoChangeHandler; +import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; import java.util.LinkedList; import java.util.List; @@ -54,10 +54,10 @@ public class NodeStateGatherer { * Sends state requests to nodes that does not have one pending and is due * for another attempt. */ - public boolean sendMessages(ContentCluster cluster, Communicator communicator, NodeStateOrHostInfoChangeHandler listener) { + public boolean sendMessages(ContentCluster cluster, Communicator communicator, NodeListener listener) { boolean sentAnyMessages = false; long currentTime = timer.getCurrentTimeInMillis(); - for (NodeInfo info : cluster.getNodeInfo()) { + for (NodeInfo info : cluster.getNodeInfos()) { Long requestTime = info.getLatestNodeStateRequestTime(); if (requestTime != null && (currentTime - requestTime < nodeStateRequestTimeoutMS)) continue; // pending request @@ -93,7 +93,7 @@ public class NodeStateGatherer { } /** Reads replies to get node state requests and create events. */ - public boolean processResponses(NodeStateOrHostInfoChangeHandler listener) { + public boolean processResponses(NodeListener listener) { boolean processedAnyResponses = false; long currentTime = timer.getCurrentTimeInMillis(); synchronized(monitor) { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/RemoteClusterControllerTask.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/RemoteClusterControllerTask.java index 949ad6f56a2..3c2143818e3 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/RemoteClusterControllerTask.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/RemoteClusterControllerTask.java @@ -2,8 +2,8 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.vdslib.state.ClusterState; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeAddedOrRemovedListener; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeStateOrHostInfoChangeHandler; +import com.yahoo.vespa.clustercontroller.core.listeners.SlobrokListener; +import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; import java.time.Instant; import java.util.Optional; @@ -15,8 +15,8 @@ public abstract class RemoteClusterControllerTask { public ClusterState currentConsolidatedState; public ClusterStateBundle publishedClusterStateBundle; public MasterInterface masterInfo; - public NodeStateOrHostInfoChangeHandler nodeStateOrHostInfoChangeHandler; - public NodeAddedOrRemovedListener nodeAddedOrRemovedListener; + public NodeListener nodeListener; + public SlobrokListener slobrokListener; } private final Object monitor = new Object(); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java index 46fafddfade..4c832592422 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java @@ -8,7 +8,7 @@ import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeStateOrHostInfoChangeHandler; +import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; import java.util.Map; import java.util.Set; @@ -113,7 +113,7 @@ public class StateChangeHandler { public void handleNewReportedNodeState(final ClusterState currentClusterState, final NodeInfo node, final NodeState reportedState, - final NodeStateOrHostInfoChangeHandler nodeListener) + final NodeListener nodeListener) { final NodeState currentState = currentClusterState.getNodeState(node.getNode()); final Level level = (currentState.equals(reportedState) && node.getVersion() == 0) ? Level.FINEST : Level.FINE; @@ -164,7 +164,7 @@ public class StateChangeHandler { public void handleMissingNode(final ClusterState currentClusterState, final NodeInfo node, - final NodeStateOrHostInfoChangeHandler nodeListener) { + final NodeListener nodeListener) { final long timeNow = timer.getCurrentTimeInMillis(); if (node.getLatestNodeStateRequestTime() != null) { @@ -241,12 +241,12 @@ public class StateChangeHandler { // `--> this will require adding more event edges and premature crash handling to it. Which is fine. public boolean watchTimers(final ContentCluster cluster, final ClusterState currentClusterState, - final NodeStateOrHostInfoChangeHandler nodeListener) + final NodeListener nodeListener) { boolean triggeredAnyTimers = false; final long currentTime = timer.getCurrentTimeInMillis(); - for(NodeInfo node : cluster.getNodeInfo()) { + for(NodeInfo node : cluster.getNodeInfos()) { triggeredAnyTimers |= handleTimeDependentOpsForNode(currentClusterState, nodeListener, currentTime, node); } @@ -257,7 +257,7 @@ public class StateChangeHandler { } private boolean handleTimeDependentOpsForNode(final ClusterState currentClusterState, - final NodeStateOrHostInfoChangeHandler nodeListener, + final NodeListener nodeListener, final long currentTime, final NodeInfo node) { @@ -334,7 +334,7 @@ public class StateChangeHandler { } private boolean reportDownIfOutdatedSlobrokNode(ClusterState currentClusterState, - NodeStateOrHostInfoChangeHandler nodeListener, + NodeListener nodeListener, long currentTime, NodeInfo node, NodeState lastReportedState) @@ -379,7 +379,7 @@ public class StateChangeHandler { private void updateNodeInfoFromReportedState(final NodeInfo node, final NodeState currentState, final NodeState reportedState, - final NodeStateOrHostInfoChangeHandler nodeListener) { + final NodeListener nodeListener) { final long timeNow = timer.getCurrentTimeInMillis(); log.log(Level.FINE, () -> String.format("Finding new cluster state entry for %s switching state %s", node, currentState.getTextualDifference(reportedState))); @@ -400,7 +400,7 @@ public class StateChangeHandler { private void markNodeUnstableIfDownEdgeDuringInit(final NodeInfo node, final NodeState currentState, final NodeState reportedState, - final NodeStateOrHostInfoChangeHandler nodeListener, + final NodeListener nodeListener, final long timeNow) { if (currentState.getState().equals(State.INITIALIZING) && reportedState.getState().oneOf("ds") @@ -419,7 +419,7 @@ public class StateChangeHandler { private boolean handleImplicitCrashEdgeFromReverseInitProgress(final NodeInfo node, final NodeState currentState, final NodeState reportedState, - final NodeStateOrHostInfoChangeHandler nodeListener, + final NodeListener nodeListener, final long timeNow) { if (currentState.getState().equals(State.INITIALIZING) && (reportedState.getState().equals(State.INITIALIZING) && reportedState.getInitProgress() < currentState.getInitProgress())) @@ -438,7 +438,7 @@ public class StateChangeHandler { } private boolean handleReportedNodeCrashEdge(NodeInfo node, NodeState currentState, - NodeState reportedState, NodeStateOrHostInfoChangeHandler nodeListener, + NodeState reportedState, NodeListener nodeListener, long timeNow) { if (nodeUpToDownEdge(node, currentState, reportedState)) { node.setTransitionTime(timeNow); @@ -467,7 +467,7 @@ public class StateChangeHandler { && (node.getWantedState().getState().equals(State.RETIRED) || !reportedState.getState().equals(State.INITIALIZING)); } - private boolean handlePrematureCrash(NodeInfo node, NodeStateOrHostInfoChangeHandler changeListener) { + private boolean handlePrematureCrash(NodeInfo node, NodeListener changeListener) { node.setPrematureCrashCount(node.getPrematureCrashCount() + 1); if (disableUnstableNodes && node.getPrematureCrashCount() > maxPrematureCrashes) { NodeState wantedState = new NodeState(node.getNode().getType(), State.DOWN) diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java index d061f7edbea..2359e4d8389 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java @@ -188,18 +188,18 @@ public class SystemStateBroadcaster { } private List<NodeInfo> resolveStateVersionSendSet(DatabaseHandler.DatabaseContext dbContext) { - return dbContext.getCluster().getNodeInfo().stream() - .filter(this::nodeNeedsClusterStateBundle) - .filter(node -> !newestStateBundleAlreadySentToNode(node)) - .collect(Collectors.toList()); + return dbContext.getCluster().getNodeInfos().stream() + .filter(this::nodeNeedsClusterStateBundle) + .filter(node -> !newestStateBundleAlreadySentToNode(node)) + .collect(Collectors.toList()); } // Precondition: no nodes in the cluster need to receive the current cluster state version bundle private List<NodeInfo> resolveStateActivationSendSet(DatabaseHandler.DatabaseContext dbContext) { - return dbContext.getCluster().getNodeInfo().stream() - .filter(this::nodeNeedsClusterStateActivation) - .filter(node -> !newestStateActivationAlreadySentToNode(node)) - .collect(Collectors.toList()); + return dbContext.getCluster().getNodeInfos().stream() + .filter(this::nodeNeedsClusterStateActivation) + .filter(node -> !newestStateActivationAlreadySentToNode(node)) + .collect(Collectors.toList()); } private boolean newestStateBundleAlreadySentToNode(NodeInfo node) { @@ -222,9 +222,9 @@ public class SystemStateBroadcaster { return; // Nothing to do for the current state } final int currentStateVersion = clusterStateBundle.getVersion(); - boolean anyDistributorsNeedStateBundle = dbContext.getCluster().getNodeInfo().stream() - .filter(NodeInfo::isDistributor) - .anyMatch(this::nodeNeedsClusterStateBundle); + boolean anyDistributorsNeedStateBundle = dbContext.getCluster().getNodeInfos().stream() + .filter(NodeInfo::isDistributor) + .anyMatch(this::nodeNeedsClusterStateBundle); if (!anyDistributorsNeedStateBundle && (currentStateVersion > lastStateVersionBundleAcked)) { markCurrentClusterStateBundleAsReceivedByAllDistributors(); @@ -243,9 +243,9 @@ public class SystemStateBroadcaster { return; } - boolean anyDistributorsNeedActivation = dbContext.getCluster().getNodeInfo().stream() - .filter(NodeInfo::isDistributor) - .anyMatch(this::nodeNeedsClusterStateActivation); + boolean anyDistributorsNeedActivation = dbContext.getCluster().getNodeInfos().stream() + .filter(NodeInfo::isDistributor) + .anyMatch(this::nodeNeedsClusterStateActivation); if (!anyDistributorsNeedActivation && (currentStateVersion > lastClusterStateVersionConverged)) { markCurrentClusterStateAsConverged(database, dbContext, fleetController); @@ -352,7 +352,7 @@ public class SystemStateBroadcaster { private static ClusterState buildModifiedClusterState(ClusterState sourceState, DatabaseHandler.DatabaseContext dbContext) { ClusterState newState = sourceState.clone(); - for (NodeInfo n : dbContext.getCluster().getNodeInfo()) { + for (NodeInfo n : dbContext.getCluster().getNodeInfos()) { NodeState ns = newState.getNodeState(n.getNode()); if (!n.isDistributor() && ns.getStartTimestamp() == 0) { ns.setStartTimestamp(n.getStartTimestamp()); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseFactory.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseFactory.java index 45a416fade4..31f6bbfe932 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseFactory.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseFactory.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core.database; -import com.yahoo.vespa.clustercontroller.core.ContentCluster; - /** * Database factory to enable test mocking of DB features. In practice, this * will always be {@link ZooKeeperDatabase} due to rather heavy ZK feature @@ -11,14 +9,10 @@ import com.yahoo.vespa.clustercontroller.core.ContentCluster; public interface DatabaseFactory { class Params { - ContentCluster cluster; - int nodeIndex; String dbAddress; int dbSessionTimeout; Database.DatabaseListener listener; - Params cluster(ContentCluster c) { this.cluster = c; return this; } - Params nodeIndex(int i) { this.nodeIndex = i; return this; } Params databaseAddress(String address) { this.dbAddress = address; return this; } Params databaseSessionTimeout(int timeout) { this.dbSessionTimeout = timeout; return this; } Params databaseListener(Database.DatabaseListener listener) { this.listener = listener; return this; } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java index 0c6c773a9bc..01b8ed48c80 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java @@ -10,8 +10,8 @@ import com.yahoo.vespa.clustercontroller.core.FleetControllerContext; import com.yahoo.vespa.clustercontroller.core.FleetController; import com.yahoo.vespa.clustercontroller.core.NodeInfo; import com.yahoo.vespa.clustercontroller.core.Timer; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeAddedOrRemovedListener; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeStateOrHostInfoChangeHandler; +import com.yahoo.vespa.clustercontroller.core.listeners.SlobrokListener; +import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; import org.apache.zookeeper.KeeperException; import java.io.PrintWriter; @@ -32,8 +32,8 @@ public class DatabaseHandler { public interface DatabaseContext { ContentCluster getCluster(); FleetController getFleetController(); - NodeAddedOrRemovedListener getNodeAddedOrRemovedListener(); - NodeStateOrHostInfoChangeHandler getNodeStateUpdateListener(); + SlobrokListener getNodeAddedOrRemovedListener(); + NodeListener getNodeStateUpdateListener(); } private static class Data { @@ -191,9 +191,8 @@ public class DatabaseHandler { // being called, but after receiving a database loss event. clearSessionMetaData(false); fleetControllerContext.log(logger, Level.INFO, "Setting up new ZooKeeper session at " + zooKeeperAddress); - DatabaseFactory.Params params = new DatabaseFactory.Params() - .cluster(cluster) - .nodeIndex(fleetControllerContext.id().index()) + DatabaseFactory.Params params = new DatabaseFactory + .Params() .databaseAddress(zooKeeperAddress) .databaseSessionTimeout(zooKeeperSessionTimeout) .databaseListener(dbListener); @@ -426,10 +425,10 @@ public class DatabaseHandler { } } - public void saveWantedStates(DatabaseContext databaseContext) { + public boolean saveWantedStates(DatabaseContext databaseContext) { fleetControllerContext.log(logger, Level.FINE, () -> "Checking whether wanted states have changed compared to zookeeper version."); Map<Node, NodeState> wantedStates = new TreeMap<>(); - for (NodeInfo info : databaseContext.getCluster().getNodeInfo()) { + for (NodeInfo info : databaseContext.getCluster().getNodeInfos()) { if (!info.getUserWantedState().equals(new NodeState(info.getNode().getType(), State.UP))) { wantedStates.put(info.getNode(), info.getUserWantedState()); } @@ -444,6 +443,9 @@ public class DatabaseHandler { fleetControllerContext.log(logger, Level.FINE, () -> "Scheduling new wanted states to be stored into zookeeper."); pendingStore.wantedStates = wantedStates; doNextZooKeeperTask(databaseContext); + return true; + } else { + return false; } } @@ -466,7 +468,11 @@ public class DatabaseHandler { boolean altered = false; for (Node node : wantedStates.keySet()) { NodeInfo nodeInfo = databaseContext.getCluster().getNodeInfo(node); - if (nodeInfo == null) continue; // ignore wanted state of nodes which doesn't exist + if (nodeInfo == null) { + databaseContext.getNodeStateUpdateListener().handleRemovedNode(node); + altered = true; + continue; + } NodeState wantedState = wantedStates.get(node); if ( ! nodeInfo.getUserWantedState().equals(wantedState)) { nodeInfo.setWantedState(wantedState); @@ -477,7 +483,7 @@ public class DatabaseHandler { } // Remove wanted state from any node having a wanted state set that is no longer valid - for (NodeInfo info : databaseContext.getCluster().getNodeInfo()) { + for (NodeInfo info : databaseContext.getCluster().getNodeInfos()) { NodeState wantedState = wantedStates.get(info.getNode()); if (wantedState == null && !info.getUserWantedState().equals(new NodeState(info.getNode().getType(), State.UP))) { info.setWantedState(null); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java index 907f2e0c5e9..0c32d8ef6c2 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java @@ -24,7 +24,7 @@ public class MasterDataGatherer { return Integer.parseInt(nodeName.substring(lastSlash + 1)); } - private final String zooKeeperRoot; // The root path in zookeeper, typically /vespa/fleetcontroller/<clustername>/ + private final ZooKeeperPaths paths; private Map<Integer, Integer> masterData = new TreeMap<>(); // The master state last reported to the fleetcontroller private final Map<Integer, Integer> nextMasterData = new TreeMap<>(); // Temporary master state while gathering new info from zookeeper private final AsyncCallback.ChildrenCallback childListener = new DirCallback(); // Dir change listener @@ -46,7 +46,7 @@ public class MasterDataGatherer { switch (watchedEvent.getType()) { case NodeChildrenChanged: // Fleetcontrollers have either connected or disconnected to ZooKeeper log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": A change occurred in the list of registered fleetcontrollers. Requesting new information"); - session.getChildren(zooKeeperRoot + "indexes", this, childListener, null); + session.getChildren(paths.indexesRoot(), this, childListener, null); break; case NodeDataChanged: // A fleetcontroller has changed what node it is voting for log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Altered data in node " + watchedEvent.getPath() + ". Requesting new vote"); @@ -54,7 +54,7 @@ public class MasterDataGatherer { synchronized (nextMasterData) { nextMasterData.put(index, null); } - session.getData(zooKeeperRoot + "indexes/" + index, this, nodeListener, null); + session.getData(paths.indexOf(index), this, nodeListener, null); break; case NodeCreated: // How can this happen? Can one leave watches on non-existing nodes? log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Got unexpected ZooKeeper event NodeCreated"); @@ -85,8 +85,8 @@ public class MasterDataGatherer { int index = Integer.parseInt(node); nextMasterData.put(index, null); log.log(Level.FINE, () -> "Fleetcontroller " + nodeIndex + ": Attempting to fetch data in node '" - + zooKeeperRoot + index + "' to see vote"); - session.getData(zooKeeperRoot + "indexes/" + index, changeWatcher, nodeListener, null); + + paths.indexOf(index) + "' to see vote"); + session.getData(paths.indexOf(index), changeWatcher, nodeListener, null); // Invocation of cycleCompleted() for fully accumulated election state will happen // as soon as all getData calls have been processed. } @@ -146,8 +146,8 @@ public class MasterDataGatherer { } /** Constructor setting up the various needed members, and initializing the first data fetch to start things up */ - public MasterDataGatherer(ZooKeeper session, String zooKeeperRoot, Database.DatabaseListener listener, int nodeIndex) { - this.zooKeeperRoot = zooKeeperRoot; + public MasterDataGatherer(ZooKeeper session, ZooKeeperPaths paths, Database.DatabaseListener listener, int nodeIndex) { + this.paths = paths; this.session = session; this.listener = listener; this.nodeIndex = nodeIndex; @@ -161,7 +161,7 @@ public class MasterDataGatherer { synchronized (nextMasterData) { masterData = new TreeMap<>(); nextMasterData.clear(); - session.getChildren(zooKeeperRoot + "indexes", changeWatcher, childListener, null); + session.getChildren(paths.indexesRoot(), changeWatcher, childListener, null); } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java index 72c81489351..ea745a56066 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java @@ -6,7 +6,6 @@ import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.AnnotatedClusterState; import com.yahoo.vespa.clustercontroller.core.ClusterStateBundle; -import com.yahoo.vespa.clustercontroller.core.ContentCluster; import com.yahoo.vespa.clustercontroller.core.FleetControllerContext; import com.yahoo.vespa.clustercontroller.core.rpc.EnvelopedClusterStateBundleCodec; import com.yahoo.vespa.clustercontroller.core.rpc.SlimeClusterStateBundleCodec; @@ -38,13 +37,12 @@ public class ZooKeeperDatabase extends Database { private static final Charset utf8 = StandardCharsets.UTF_8; private static final List<ACL> acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; - private final String zooKeeperRoot; + private final ZooKeeperPaths paths; private final Database.DatabaseListener listener; private final ZooKeeperWatcher watcher = new ZooKeeperWatcher(); private final ZooKeeper session; private boolean sessionOpen = true; private final FleetControllerContext context; - private final int nodeIndex; private final MasterDataGatherer masterDataGatherer; // Expected ZK znode versions. Note: these are _not_ -1 as that would match anything. // We expect the caller to invoke the load methods prior to calling any store methods. @@ -102,31 +100,30 @@ public class ZooKeeperDatabase extends Database { } } - public ZooKeeperDatabase(FleetControllerContext context, ContentCluster cluster, int nodeIndex, String address, int timeout, DatabaseListener zksl) throws IOException, KeeperException, InterruptedException { + public ZooKeeperDatabase(FleetControllerContext context, String address, int timeout, DatabaseListener zksl) throws IOException, KeeperException, InterruptedException { this.context = context; - this.nodeIndex = nodeIndex; - zooKeeperRoot = "/vespa/fleetcontroller/" + cluster.getName() + "/"; + this.paths = new ZooKeeperPaths(context.id()); session = new ZooKeeper(address, timeout, watcher, new ZkClientConfigBuilder().toConfig()); boolean completedOk = false; try{ this.listener = zksl; setupRoot(); context.log(log, Level.FINEST, "Asking for initial data on master election"); - masterDataGatherer = new MasterDataGatherer(session, zooKeeperRoot, listener, nodeIndex); + masterDataGatherer = new MasterDataGatherer(session, paths, listener, context.id().index()); completedOk = true; } finally { if (!completedOk) session.close(); } } - private void createNode(String prefix, String nodename, byte[] value) throws KeeperException, InterruptedException { + private void createNode(String path, byte[] value) throws KeeperException, InterruptedException { try{ - if (session.exists(prefix + nodename, false) != null) { - context.log(log, Level.FINE, () -> "Zookeeper node '" + prefix + nodename + "' already exists. Not creating it"); + if (session.exists(path, false) != null) { + context.log(log, Level.FINE, () -> "Zookeeper node '" + path + "' already exists. Not creating it"); return; } - session.create(prefix + nodename, value, acl, CreateMode.PERSISTENT); - context.log(log, Level.FINE, () -> "Created zookeeper node '" + prefix + nodename + "'"); + session.create(path, value, acl, CreateMode.PERSISTENT); + context.log(log, Level.FINE, () -> "Created zookeeper node '" + path + "'"); } catch (KeeperException.NodeExistsException e) { context.log(log, Level.FINE, "Node to create existed, but this is normal as other nodes " + "may create them at the same time."); @@ -134,21 +131,21 @@ public class ZooKeeperDatabase extends Database { } private void setupRoot() throws KeeperException, InterruptedException { - String[] pathElements = zooKeeperRoot.substring(1).split("/"); + String[] pathElements = paths.root().substring(1).split("/"); String path = ""; for (String elem : pathElements) { path += "/" + elem; - createNode("", path, new byte[0]); + createNode(path, new byte[0]); } - createNode(zooKeeperRoot, "indexes", new byte[0]); - createNode(zooKeeperRoot, "wantedstates", new byte[0]); - createNode(zooKeeperRoot, "starttimestamps", new byte[0]); - createNode(zooKeeperRoot, "latestversion", Integer.valueOf(0).toString().getBytes(utf8)); - createNode(zooKeeperRoot, "published_state_bundle", new byte[0]); // TODO dedupe string constants - byte[] val = String.valueOf(nodeIndex).getBytes(utf8); - deleteNodeIfExists(getMyIndexPath()); + createNode(paths.indexesRoot(), new byte[0]); + createNode(paths.wantedStates(), new byte[0]); + createNode(paths.startTimestamps(), new byte[0]); + createNode(paths.latestVersion(), Integer.valueOf(0).toString().getBytes(utf8)); + createNode(paths.publishedStateBundle(), new byte[0]); + byte[] val = String.valueOf(context.id().index()).getBytes(utf8); + deleteNodeIfExists(paths.indexOfMe()); context.log(log, Level.INFO, "Creating ephemeral master vote node with vote to self."); - session.create(getMyIndexPath(), val, acl, CreateMode.EPHEMERAL); + session.create(paths.indexOfMe(), val, acl, CreateMode.EPHEMERAL); } private void deleteNodeIfExists(String path) throws KeeperException, InterruptedException { @@ -158,10 +155,6 @@ public class ZooKeeperDatabase extends Database { } } - private String getMyIndexPath() { - return zooKeeperRoot + "indexes/" + nodeIndex; - } - /** * If this is called, we assume we're in shutdown situation, or we are doing it because we need a new session. * Thus we only need to free up resources, no need to notify anyone. @@ -192,8 +185,8 @@ public class ZooKeeperDatabase extends Database { public boolean storeMasterVote(int wantedMasterIndex) { byte[] val = String.valueOf(wantedMasterIndex).getBytes(utf8); try{ - session.setData(getMyIndexPath(), val, -1); - context.log(log, Level.INFO, "Stored new vote in ephemeral node. " + nodeIndex + " -> " + wantedMasterIndex); + session.setData(paths.indexOfMe(), val, -1); + context.log(log, Level.INFO, "Stored new vote in ephemeral node. " + context.id().index() + " -> " + wantedMasterIndex); return true; } catch (InterruptedException e) { throw new RuntimeException(e); @@ -206,7 +199,7 @@ public class ZooKeeperDatabase extends Database { byte[] data = Integer.toString(version).getBytes(utf8); try{ context.log(log, Level.INFO, "Storing new cluster state version in ZooKeeper: " + version); - var stat = session.setData(zooKeeperRoot + "latestversion", data, lastKnownStateVersionZNodeVersion); + var stat = session.setData(paths.latestVersion(), data, lastKnownStateVersionZNodeVersion); lastKnownStateVersionZNodeVersion = stat.getVersion(); return true; } catch (InterruptedException e) { @@ -222,17 +215,17 @@ public class ZooKeeperDatabase extends Database { public Integer retrieveLatestSystemStateVersion() { Stat stat = new Stat(); - context.log(log, Level.FINE, "Fetching latest cluster state at '%slatestversion'", zooKeeperRoot); + context.log(log, Level.FINE, "Fetching latest cluster state at '%s'", paths.latestVersion()); final byte[] data; try { - data = session.getData(zooKeeperRoot + "latestversion", false, stat); + data = session.getData(paths.latestVersion(), false, stat); } catch (KeeperException.NoNodeException e) { // Initial condition: No latest version has ever been written (or ZK state completely wiped!) lastKnownStateVersionZNodeVersion = 0; maybeLogExceptionWarning(e, "No latest system state found"); return null; } catch (InterruptedException | KeeperException e) { - throw new RuntimeException("Failed to get " + zooKeeperRoot + "latestversion", e); + throw new RuntimeException("Failed to get " + paths.latestVersion(), e); } lastKnownStateVersionZNodeVersion = stat.getVersion(); @@ -257,8 +250,8 @@ public class ZooKeeperDatabase extends Database { } byte[] val = sb.toString().getBytes(utf8); try{ - context.log(log, Level.FINE, () -> "Storing wanted states at '" + zooKeeperRoot + "wantedstates'"); - session.setData(zooKeeperRoot + "wantedstates", val, -1); + context.log(log, Level.FINE, () -> "Storing wanted states at '" + paths.wantedStates() + "'"); + session.setData(paths.wantedStates(), val, -1); return true; } catch (InterruptedException e) { throw new RuntimeException(e); @@ -270,9 +263,9 @@ public class ZooKeeperDatabase extends Database { public Map<Node, NodeState> retrieveWantedStates() { try{ - context.log(log, Level.FINE, () -> "Fetching wanted states at '" + zooKeeperRoot + "wantedstates'"); + context.log(log, Level.FINE, () -> "Fetching wanted states at '" + paths.wantedStates() + "'"); Stat stat = new Stat(); - byte[] data = session.getData(zooKeeperRoot + "wantedstates", false, stat); + byte[] data = session.getData(paths.wantedStates(), false, stat); Map<Node, NodeState> wanted = new TreeMap<>(); if (data != null && data.length > 0) { StringTokenizer st = new StringTokenizer(new String(data, utf8), "\n", false); @@ -308,8 +301,8 @@ public class ZooKeeperDatabase extends Database { } byte val[] = sb.toString().getBytes(utf8); try{ - context.log(log, Level.FINE, () -> "Storing start timestamps at '" + zooKeeperRoot + "starttimestamps"); - session.setData(zooKeeperRoot + "starttimestamps", val, -1); + context.log(log, Level.FINE, () -> "Storing start timestamps at '" + paths.startTimestamps() + "'"); + session.setData(paths.startTimestamps(), val, -1); return true; } catch (InterruptedException e) { throw new RuntimeException(e); @@ -322,9 +315,9 @@ public class ZooKeeperDatabase extends Database { @Override public Map<Node, Long> retrieveStartTimestamps() { try{ - context.log(log, Level.FINE, () -> "Fetching start timestamps at '" + zooKeeperRoot + "starttimestamps'"); + context.log(log, Level.FINE, () -> "Fetching start timestamps at '" + paths.startTimestamps() + "'"); Stat stat = new Stat(); - byte[] data = session.getData(zooKeeperRoot + "starttimestamps", false, stat); + byte[] data = session.getData(paths.startTimestamps(), false, stat); Map<Node, Long> wanted = new TreeMap<Node, Long>(); if (data != null && data.length > 0) { StringTokenizer st = new StringTokenizer(new String(data, utf8), "\n", false); @@ -357,10 +350,9 @@ public class ZooKeeperDatabase extends Database { try{ context.log(log, Level.FINE, - () -> String.format("Storing published state bundle %s at " + - "'%spublished_state_bundle' with expected znode version %d", - stateBundle, zooKeeperRoot, lastKnownStateBundleZNodeVersion)); - var stat = session.setData(zooKeeperRoot + "published_state_bundle", encodedBundle, lastKnownStateBundleZNodeVersion); + () -> String.format("Storing published state bundle %s at '%s' with expected znode version %d", + stateBundle, paths.publishedStateBundle(), lastKnownStateBundleZNodeVersion)); + var stat = session.setData(paths.publishedStateBundle(), encodedBundle, lastKnownStateBundleZNodeVersion); lastKnownStateBundleZNodeVersion = stat.getVersion(); } catch (InterruptedException e) { throw new RuntimeException(e); @@ -378,7 +370,7 @@ public class ZooKeeperDatabase extends Database { public ClusterStateBundle retrieveLastPublishedStateBundle() { Stat stat = new Stat(); try { - byte[] data = session.getData(zooKeeperRoot + "published_state_bundle", false, stat); + byte[] data = session.getData(paths.publishedStateBundle(), false, stat); lastKnownStateBundleZNodeVersion = stat.getVersion(); if (data != null && data.length != 0) { EnvelopedClusterStateBundleCodec envelopedBundleCodec = new SlimeClusterStateBundleCodec(); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabaseFactory.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabaseFactory.java index 71f39135609..3263c06a95c 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabaseFactory.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabaseFactory.java @@ -13,8 +13,7 @@ public class ZooKeeperDatabaseFactory implements DatabaseFactory { @Override public Database create(Params params) throws Exception { - return new ZooKeeperDatabase(context, params.cluster, params.nodeIndex, params.dbAddress, - params.dbSessionTimeout, params.listener); + return new ZooKeeperDatabase(context, params.dbAddress, params.dbSessionTimeout, params.listener); } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperPaths.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperPaths.java new file mode 100644 index 00000000000..06a9b240175 --- /dev/null +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperPaths.java @@ -0,0 +1,26 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.clustercontroller.core.database; + +import com.yahoo.vespa.clustercontroller.core.FleetControllerId; + +/** + * @author hakonhall + */ +public class ZooKeeperPaths { + private final String root; + private final int myIndex; + + public ZooKeeperPaths(FleetControllerId id) { + this.root = "/vespa/fleetcontroller/" + id.clusterName(); + this.myIndex = id.index(); + } + + public String root() { return root; } + public String indexesRoot() { return root + "/indexes"; } + public String indexOf(int index) { return indexesRoot() + "/" + index; } + public String indexOfMe() { return indexOf(myIndex); } + public String wantedStates() { return root + "/wantedstates"; } + public String publishedStateBundle() { return root + "/published_state_bundle"; } + public String latestVersion() { return root + "/latestversion"; } + public String startTimestamps() { return root + "/starttimestamps"; } +} diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/listeners/NodeListener.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/listeners/NodeListener.java new file mode 100644 index 00000000000..351b68b3b57 --- /dev/null +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/listeners/NodeListener.java @@ -0,0 +1,25 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.clustercontroller.core.listeners; + +import com.yahoo.vdslib.state.Node; +import com.yahoo.vdslib.state.NodeState; +import com.yahoo.vespa.clustercontroller.core.NodeInfo; +import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo; + +/** + * Implemented by classes wanting events when there are node changes. + */ +public interface NodeListener { + + default void handleNewNodeState(NodeInfo currentInfo, NodeState newState) {} + default void handleNewWantedNodeState(NodeInfo node, NodeState newState) {} + + /** Invoked after NodeInfo has been removed from the content cluster. */ + default void handleRemovedNode(Node node) {} + + /** + * For every getnodestate RPC call, handleUpdatedHostInfo() will be called with the host info JSON string. + */ + default void handleUpdatedHostInfo(NodeInfo node, HostInfo newHostInfo) {} + +} diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/listeners/NodeStateOrHostInfoChangeHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/listeners/NodeStateOrHostInfoChangeHandler.java deleted file mode 100644 index b76b46b216b..00000000000 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/listeners/NodeStateOrHostInfoChangeHandler.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.clustercontroller.core.listeners; - -import com.yahoo.vdslib.state.NodeState; -import com.yahoo.vespa.clustercontroller.core.NodeInfo; -import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo; - -/** - * Implemented by classes wanting events when node states changes. - */ -public interface NodeStateOrHostInfoChangeHandler { - - void handleNewNodeState(NodeInfo currentInfo, NodeState newState); - void handleNewWantedNodeState(NodeInfo node, NodeState newState); - - /** - * For every getnodestate RPC call, handleUpdatedHostInfo() will be called with the host info JSON string. - */ - void handleUpdatedHostInfo(NodeInfo node, HostInfo newHostInfo); - -} diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/listeners/NodeAddedOrRemovedListener.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/listeners/SlobrokListener.java index d811708c999..5a397cc4935 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/listeners/NodeAddedOrRemovedListener.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/listeners/SlobrokListener.java @@ -4,9 +4,9 @@ package com.yahoo.vespa.clustercontroller.core.listeners; import com.yahoo.vespa.clustercontroller.core.NodeInfo; /** - * Listeners for new nodes detected. + * Implemented by classes that wants to be notified of Slobrok events. */ -public interface NodeAddedOrRemovedListener { +public interface SlobrokListener { void handleNewNode(NodeInfo node); void handleMissingNode(NodeInfo node); void handleNewRpcAddress(NodeInfo node); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java index 431c207af5c..ddbda2bf776 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java @@ -11,7 +11,7 @@ import com.yahoo.vespa.clustercontroller.core.ContentCluster; import com.yahoo.vespa.clustercontroller.core.NodeInfo; import com.yahoo.vespa.clustercontroller.core.NodeStateChangeChecker; import com.yahoo.vespa.clustercontroller.core.RemoteClusterControllerTask; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeStateOrHostInfoChangeHandler; +import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; import com.yahoo.vespa.clustercontroller.core.restapiv2.Id; import com.yahoo.vespa.clustercontroller.core.restapiv2.MissingIdException; import com.yahoo.vespa.clustercontroller.core.restapiv2.Request; @@ -62,7 +62,7 @@ public class SetNodeStateRequest extends Request<SetResponse> { condition, newStates, id.getNode(), - context.nodeStateOrHostInfoChangeHandler, + context.nodeListener, context.currentConsolidatedState, context.masterInfo.inMasterMoratorium(), probe); @@ -112,7 +112,7 @@ public class SetNodeStateRequest extends Request<SetResponse> { SetUnitStateRequest.Condition condition, Map<String, UnitState> newStates, Node node, - NodeStateOrHostInfoChangeHandler stateListener, + NodeListener stateListener, ClusterState currentClusterState, boolean inMasterMoratorium, boolean probe) throws StateRestApiException { @@ -159,7 +159,7 @@ public class SetNodeStateRequest extends Request<SetResponse> { SetUnitStateRequest.Condition condition, NodeInfo nodeInfo, ContentCluster cluster, - NodeStateOrHostInfoChangeHandler stateListener, + NodeListener stateListener, boolean probe) { if (result.settingWantedStateIsAllowed()) { setNewWantedState(nodeInfo, newWantedState, stateListener, probe); @@ -186,7 +186,7 @@ public class SetNodeStateRequest extends Request<SetResponse> { private static void setDistributorWantedState(ContentCluster cluster, int index, NodeState newStorageWantedState, - NodeStateOrHostInfoChangeHandler stateListener, + NodeListener stateListener, boolean probe) { Node distributorNode = new Node(NodeType.DISTRIBUTOR, index); NodeInfo nodeInfo = cluster.getNodeInfo(distributorNode); @@ -224,7 +224,7 @@ public class SetNodeStateRequest extends Request<SetResponse> { private static void setNewWantedState(NodeInfo nodeInfo, NodeState newWantedState, - NodeStateOrHostInfoChangeHandler stateListener, + NodeListener stateListener, boolean probe) { if (probe) return; nodeInfo.setWantedState(newWantedState); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStatesForClusterRequest.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStatesForClusterRequest.java index 55ac75957bc..1cc8f2860c6 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStatesForClusterRequest.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStatesForClusterRequest.java @@ -70,7 +70,7 @@ public class SetNodeStatesForClusterRequest extends Request<SetResponse> { condition, newStates, node, - context.nodeStateOrHostInfoChangeHandler, + context.nodeListener, context.currentConsolidatedState, context.masterInfo.inMasterMoratorium(), probe); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/WantedStateSetter.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/WantedStateSetter.java index 8095d37d641..51b2f1cfe4f 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/WantedStateSetter.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/WantedStateSetter.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.clustercontroller.core.restapiv2.requests; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.Node; import com.yahoo.vespa.clustercontroller.core.ContentCluster; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeStateOrHostInfoChangeHandler; +import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.StateRestApiException; import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest; import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.SetResponse; @@ -21,7 +21,7 @@ public interface WantedStateSetter { SetUnitStateRequest.Condition condition, Map<String, UnitState> newStates, Node node, - NodeStateOrHostInfoChangeHandler stateListener, + NodeListener stateListener, ClusterState currentClusterState, boolean inMasterMoratorium, boolean probe) throws StateRestApiException; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java index 65c8a9df28e..6e416ce4906 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java @@ -27,8 +27,8 @@ import com.yahoo.vespa.clustercontroller.core.ContentCluster; import com.yahoo.vespa.clustercontroller.core.MasterElectionHandler; import com.yahoo.vespa.clustercontroller.core.NodeInfo; import com.yahoo.vespa.clustercontroller.core.Timer; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeAddedOrRemovedListener; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeStateOrHostInfoChangeHandler; +import com.yahoo.vespa.clustercontroller.core.listeners.SlobrokListener; +import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; import java.io.PrintWriter; import java.io.StringWriter; @@ -186,8 +186,8 @@ public class RpcServer { } public boolean handleRpcRequests(ContentCluster cluster, ClusterState systemState, - NodeStateOrHostInfoChangeHandler changeListener, - NodeAddedOrRemovedListener addedListener) + NodeListener changeListener, + SlobrokListener addedListener) { boolean handledAnyRequests = false; if (!isConnected()) { @@ -234,7 +234,7 @@ public class RpcServer { log.log(Level.FINE, "Resolving RPC getNodeList request"); List<String> slobrok = new ArrayList<String>(); List<String> rpc = new ArrayList<String>(); - for(NodeInfo node : cluster.getNodeInfo()) { + for(NodeInfo node : cluster.getNodeInfos()) { String s1 = node.getSlobrokAddress(); String s2 = node.getRpcAddress(); assert(s1 != null); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java index 7487f9546b7..c88bf71af09 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java @@ -13,7 +13,7 @@ import com.yahoo.vespa.clustercontroller.core.FleetControllerContext; import com.yahoo.vespa.clustercontroller.core.NodeInfo; import com.yahoo.vespa.clustercontroller.core.NodeLookup; import com.yahoo.vespa.clustercontroller.core.Timer; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeAddedOrRemovedListener; +import com.yahoo.vespa.clustercontroller.core.listeners.SlobrokListener; import java.util.Iterator; import java.util.LinkedList; @@ -78,7 +78,7 @@ public class SlobrokClient implements NodeLookup { } @Override - public boolean updateCluster(ContentCluster cluster, NodeAddedOrRemovedListener listener) { + public boolean updateCluster(ContentCluster cluster, SlobrokListener listener) { if (mirror == null) return false; int mirrorVersion = mirror.updates(); if (freshMirror) { @@ -149,7 +149,7 @@ public class SlobrokClient implements NodeLookup { } } cluster.setSlobrokGenerationCount(mirrorVersion); - for (NodeInfo nodeInfo : cluster.getNodeInfo()) { + for (NodeInfo nodeInfo : cluster.getNodeInfos()) { if (slobrokNodes.containsKey(nodeInfo.getNode()) && nodeInfo.isRpcAddressOutdated()) { context.log(log, Level.WARNING, @@ -174,7 +174,7 @@ public class SlobrokClient implements NodeLookup { List<SlobrokData> alteredRpcAddress, List<NodeInfo> returningRpcAddressNodeInfos) { - Iterator<NodeInfo> oldIt = oldCluster.getNodeInfo().iterator(); + Iterator<NodeInfo> oldIt = oldCluster.getNodeInfos().iterator(); Iterator<SlobrokData> newIt = slobrokNodes.values().iterator(); NodeInfo oldNext = null; SlobrokData newNext = null; diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFixture.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFixture.java index 4ce32484098..f8d41405e85 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFixture.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFixture.java @@ -8,7 +8,7 @@ import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeStateOrHostInfoChangeHandler; +import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; import java.util.Collection; import java.util.HashSet; @@ -57,7 +57,7 @@ public class ClusterFixture { private void doReportNodeState(final Node node, final NodeState nodeState) { final ClusterState stateBefore = rawGeneratedClusterState(); - NodeStateOrHostInfoChangeHandler handler = mock(NodeStateOrHostInfoChangeHandler.class); + NodeListener handler = mock(NodeListener.class); NodeInfo nodeInfo = cluster.getNodeInfo(node); nodeStateChangeHandler.handleNewReportedNodeState(stateBefore, nodeInfo, nodeState, handler); @@ -142,7 +142,7 @@ public class ClusterFixture { } public ClusterFixture assignDummyRpcAddresses() { - cluster.getNodeInfo().forEach(ni -> { + cluster.getNodeInfos().forEach(ni -> { ni.setRpcAddress(String.format("tcp/%s.%d.local:0", ni.isStorage() ? "storage" : "distributor", ni.getNodeIndex())); @@ -169,7 +169,7 @@ public class ClusterFixture { Set<ConfiguredNode> configuredNodes = new HashSet<>(cluster.getConfiguredNodes().values()); configuredNodes.remove(new ConfiguredNode(nodeIndex, false)); configuredNodes.add(new ConfiguredNode(nodeIndex, true)); - cluster.setNodes(configuredNodes); + cluster.setNodes(configuredNodes, new NodeListener() {}); return this; } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseHandlerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseHandlerTest.java index a621b0f565a..1f7b9293960 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseHandlerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseHandlerTest.java @@ -1,23 +1,42 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; +import com.yahoo.vdslib.state.Node; +import com.yahoo.vdslib.state.NodeState; +import com.yahoo.vdslib.state.NodeType; +import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.database.Database; import com.yahoo.vespa.clustercontroller.core.database.DatabaseFactory; import com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeAddedOrRemovedListener; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeStateOrHostInfoChangeHandler; +import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; +import com.yahoo.vespa.clustercontroller.core.listeners.SlobrokListener; +import org.junit.After; +import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.MockitoAnnotations; + +import java.util.Map; +import java.util.TreeMap; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class DatabaseHandlerTest { + private AutoCloseable openMock = null; + + @Captor + ArgumentCaptor<TreeMap<Node, NodeState>> wantedStatesArgument; + static class Fixture { final ClusterFixture clusterFixture = ClusterFixture.forFlatCluster(10); final FleetController mockController = mock(FleetController.class); @@ -52,12 +71,12 @@ public class DatabaseHandlerTest { } @Override - public NodeAddedOrRemovedListener getNodeAddedOrRemovedListener() { + public SlobrokListener getNodeAddedOrRemovedListener() { return null; } @Override - public NodeStateOrHostInfoChangeHandler getNodeStateUpdateListener() { + public NodeListener getNodeStateUpdateListener() { return null; } }; @@ -70,6 +89,16 @@ public class DatabaseHandlerTest { } } + @Before + public void setUp() { + openMock = MockitoAnnotations.openMocks(this); + } + + @After + public void tearDown() throws Exception { + openMock.close(); + } + @Test public void can_store_latest_cluster_state_bundle() throws Exception { Fixture f = new Fixture(); @@ -104,4 +133,29 @@ public class DatabaseHandlerTest { assertEquals(ClusterStateBundle.empty(), retrievedBundle); } + @Test + public void save_wanted_state_of_configured_nodes() throws Exception { + var fixture = new Fixture(); + DatabaseHandler handler = fixture.createHandler(); + DatabaseHandler.DatabaseContext databaseContext = fixture.createMockContext(); + + // The test fixture contains 10 nodes with indices 1-10. A wanted state for + // an existing node (5) should be preserved. Note that it is not possible to set a + // wanted state outside the existing nodes. + Node storageNode5 = Node.ofStorage(5); + NodeState maintenance = new NodeState(NodeType.STORAGE, State.MAINTENANCE); + databaseContext.getCluster().getNodeInfo(storageNode5).setWantedState(maintenance); + var expectedWantedStates = new TreeMap<>(Map.of(storageNode5, maintenance)); + + // Ensure database is connected to ZooKeeper + assertTrue(handler.doNextZooKeeperTask(databaseContext)); + + // Verify ZooKeeperDatabase::storeWantedStates is invoked once + verify(fixture.mockDatabase, times(0)).storeWantedStates(any()); + assertTrue(handler.saveWantedStates(databaseContext)); + verify(fixture.mockDatabase, times(1)).storeWantedStates(wantedStatesArgument.capture()); + + // Verify ZooKeeperDatabase::storeWantedStates only saves states for existing nodes + assertEquals(expectedWantedStates, wantedStatesArgument.getValue()); + } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java index 37fe3e28a5d..2ff48e00d0f 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseTest.java @@ -26,8 +26,6 @@ public class DatabaseTest extends FleetControllerTest { private static final Logger log = Logger.getLogger(DatabaseTest.class.getName()); - // These tests work in isolation but causes other tests to hang - @Ignore @Test public void testWantedStatesInZooKeeper() throws Exception { startingTest("DatabaseTest::testWantedStatesInZooKeeper"); @@ -82,8 +80,6 @@ public class DatabaseTest extends FleetControllerTest { assertWantedStates(wantedStates); } - // These tests work in isolation but causes other tests to hang - @Ignore @Test public void testWantedStateOfUnknownNode() throws Exception { startingTest("DatabaseTest::testWantedStatesOfUnknownNode"); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyCommunicator.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyCommunicator.java index 3c232a7c52b..3127201a342 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyCommunicator.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyCommunicator.java @@ -6,7 +6,7 @@ import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.State; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeAddedOrRemovedListener; +import com.yahoo.vespa.clustercontroller.core.listeners.SlobrokListener; import java.util.ArrayList; import java.util.List; @@ -133,14 +133,14 @@ public class DummyCommunicator implements Communicator, NodeLookup { } @Override - public boolean updateCluster(ContentCluster cluster, NodeAddedOrRemovedListener listener) { + public boolean updateCluster(ContentCluster cluster, SlobrokListener listener) { if (newNodes != null) { List<Node> tmp = newNodes; for (Node node : tmp) cluster.clusterInfo().setRpcAddress(node, "foo"); - for (NodeInfo info : cluster.getNodeInfo()) { + for (NodeInfo info : cluster.getNodeInfos()) { if (!tmp.contains(info.getNode())) { info.markRpcAddressOutdated(timer); listener.handleMissingNode(info); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java index c56b3bbdc69..a5eeb69e126 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java @@ -84,7 +84,7 @@ public abstract class FleetControllerTest implements Waiter { static { LogSetup.initVespaLogging("fleetcontroller"); - timeoutS = 120; + timeoutS = 30; timeoutMS = timeoutS * 1000; } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownTest.java index 254f863e9ea..1d4b2a73560 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownTest.java @@ -7,20 +7,19 @@ import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeStateOrHostInfoChangeHandler; - -import static com.yahoo.vespa.clustercontroller.core.matchers.EventForNode.eventForNode; -import static com.yahoo.vespa.clustercontroller.core.matchers.NodeEventWithDescription.nodeEventWithDescription; +import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; import org.junit.Test; import java.util.HashSet; import java.util.List; import java.util.Set; +import static com.yahoo.vespa.clustercontroller.core.matchers.EventForNode.eventForNode; +import static com.yahoo.vespa.clustercontroller.core.matchers.NodeEventWithDescription.nodeEventWithDescription; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.AllOf.allOf; import static org.hamcrest.core.IsCollectionContaining.hasItem; import static org.junit.Assert.assertEquals; -import static org.hamcrest.MatcherAssert.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -179,7 +178,7 @@ public class GroupAutoTakedownTest { // However, once grace period expires the group should be taken down. fixture.timer.advanceTime(1001); - NodeStateOrHostInfoChangeHandler changeListener = mock(NodeStateOrHostInfoChangeHandler.class); + NodeListener changeListener = mock(NodeListener.class); fixture.nodeStateChangeHandler.watchTimers( fixture.cluster, fixture.annotatedGeneratedClusterState().getClusterState(), changeListener); @@ -253,7 +252,7 @@ public class GroupAutoTakedownTest { nodes.add(new ConfiguredNode(5, true)); // TODO this should ideally also set the retired flag in the distribution // config, but only the ConfiguredNodes are actually looked at currently. - fixture.cluster.setNodes(nodes); + fixture.cluster.setNodes(nodes, new NodeListener() {}); assertEquals("distributor:6 storage:6 .4.s:d .5.s:r", stateAfterStorageTransition(fixture, 5, State.UP)); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java index db86df88fc5..257bb54047c 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java @@ -12,7 +12,6 @@ import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.status.StatusHandler; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestRule; @@ -79,7 +78,6 @@ public class MasterElectionTest extends FleetControllerTest { long maxTime = System.currentTimeMillis() + timeoutMS; for (FleetController f : fleetControllers) { while (f.hasZookeeperConnection()) { - timer.advanceTime(1000); try { Thread.sleep(1); } catch (InterruptedException e) { /* ignore */ } if (System.currentTimeMillis() > maxTime) throw new TimeoutException("Failed to notice zookeeper down within timeout of " + timeoutMS + " ms"); @@ -117,14 +115,12 @@ public class MasterElectionTest extends FleetControllerTest { super.tearDown(); } - /** Ignored for unknown reasons */ @Test - @Ignore public void testMasterElection() throws Exception { startingTest("MasterElectionTest::testMasterElection"); log.log(Level.INFO, "STARTING TEST: MasterElectionTest::testMasterElection()"); FleetControllerOptions options = defaultOptions("mycluster"); - options.masterZooKeeperCooldownPeriod = 1; + options.masterZooKeeperCooldownPeriod = 100; setUpFleetController(5, false, options); waitForMaster(0); log.log(Level.INFO, "SHUTTING DOWN FLEET CONTROLLER 0"); @@ -137,7 +133,7 @@ public class MasterElectionTest extends FleetControllerTest { fleetControllers.get(2).shutdown(); // Too few for there to be a master at this point - for (int i=0; i<fleetControllers.size(); ++i) { + for (int i = 0; i < fleetControllers.size(); ++i) { if (fleetControllers.get(i).isRunning()) waitForCompleteCycle(i); assertFalse("Fleet controller " + i, fleetControllers.get(i).isMaster()); } @@ -163,7 +159,7 @@ public class MasterElectionTest extends FleetControllerTest { fleetControllers.get(2).shutdown(); // Too few for there to be a master at this point - for (int i=0; i<fleetControllers.size(); ++i) { + for (int i = 0; i < fleetControllers.size(); ++i) { if (fleetControllers.get(i).isRunning()) waitForCompleteCycle(i); assertFalse(fleetControllers.get(i).isMaster()); } @@ -172,15 +168,15 @@ public class MasterElectionTest extends FleetControllerTest { private void waitForMaster(int master) { log.log(Level.INFO, "Entering waitForMaster"); boolean isOnlyMaster = false; - for (int i=0; i < FleetControllerTest.timeoutMS; i+=100) { + for (int i = 0; i < FleetControllerTest.timeoutMS; i += 100) { if (!fleetControllers.get(master).isMaster()) { log.log(Level.INFO, "Node " + master + " is not master yet, sleeping more"); timer.advanceTime(100); waitForCompleteCycle(master); } else { - log.log(Level.INFO, "Node " + master + " is master. Checking that noone else is master"); + log.log(Level.INFO, "Node " + master + " is master. Checking that no one else is master"); isOnlyMaster = true; - for (int j=0; j<fleetControllers.size(); ++j) { + for (int j = 0; j < fleetControllers.size(); ++j) { if (j != master && fleetControllers.get(j).isMaster()) { isOnlyMaster = false; log.log(Level.INFO, "Node " + j + " also says it is master."); @@ -192,7 +188,7 @@ public class MasterElectionTest extends FleetControllerTest { } } // Have to wait to get zookeeper communication chance to happen. - try{ Thread.sleep(100); } catch (InterruptedException e) { /* ignore */ } + try { Thread.sleep(100); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } if (!isOnlyMaster) { @@ -258,11 +254,9 @@ public class MasterElectionTest extends FleetControllerTest { zooKeeperServer.shutdown(true); waitForCompleteCycles(); - timer.advanceTime(options.zooKeeperSessionTimeout); waitForZookeeperDisconnected(); zooKeeperServer = ZooKeeperTestServer.createWithFixedPort(18342); - timer.advanceTime(10 * 1000); // Wait long enough for fleetcontroller wanting to retry zookeeper connection log.log(Level.INFO, "WAITING FOR 0 TO BE MASTER"); waitForMaster(0); @@ -281,11 +275,10 @@ public class MasterElectionTest extends FleetControllerTest { log.log(Level.INFO, "STOPPING ZOOKEEPER SERVER AT " + zooKeeperServer.getAddress()); zooKeeperServer.shutdown(true); waitForCompleteCycles(); - timer.advanceTime(options.zooKeeperSessionTimeout); waitForZookeeperDisconnected(); // No one can be master if server is unavailable log.log(Level.INFO, "Checking master status"); - for (int i=0; i<fleetControllers.size(); ++i) { + for (int i = 0; i < fleetControllers.size(); ++i) { assertFalse("Index " + i, fleetControllers.get(i).isMaster()); } @@ -297,19 +290,16 @@ public class MasterElectionTest extends FleetControllerTest { fc.updateOptions(myoptions); log.log(Level.INFO, "Should now have sent out new zookeeper server address " + myoptions.zooKeeperServerAddress + " to fleetcontroller " + myoptions.fleetControllerIndex); } - timer.advanceTime(10 * 1000); // Wait long enough for fleetcontroller wanting to retry zookeeper connection waitForMaster(0); log.log(Level.INFO, "SHUTTING DOWN"); } - /** Ignored for unknown reasons */ @Test - @Ignore public void testMasterZooKeeperCooldown() throws Exception { startingTest("MasterElectionTest::testMasterZooKeeperCooldown"); FleetControllerOptions options = defaultOptions("mycluster"); options.masterZooKeeperCooldownPeriod = 3600 * 1000; // An hour - setUpFleetController(3, false, options); + setUpFleetController(3, true, options); waitForMaster(0); timer.advanceTime(24 * 3600 * 1000); // A day waitForCompleteCycle(1); @@ -352,14 +342,12 @@ public class MasterElectionTest extends FleetControllerTest { + "' within timeout of " + timeoutMS + " ms"); } - /** Ignored for unknown reasons */ @Test - @Ignore public void testGetMaster() throws Exception { startingTest("MasterElectionTest::testGetMaster"); FleetControllerOptions options = defaultOptions("mycluster"); options.masterZooKeeperCooldownPeriod = 3600 * 1000; // An hour - setUpFleetController(3, false, options); + setUpFleetController(3, true, options); waitForMaster(0); supervisor = new Supervisor(new Transport()); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculatorTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculatorTest.java index 7e02f63d56e..e136ddfa72d 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculatorTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceExhaustionCalculatorTest.java @@ -23,7 +23,7 @@ public class ResourceExhaustionCalculatorTest { var calc = new ResourceExhaustionCalculator(true, mapOf(usage("disk", 0.5), usage("memory", 0.8))); var cf = createFixtureWithReportedUsages(forNode(1, usage("disk", 0.49), usage("memory", 0.79)), forNode(2, usage("disk", 0.4), usage("memory", 0.6))); - var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); + var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfos()); assertNull(feedBlock); } @@ -32,7 +32,7 @@ public class ResourceExhaustionCalculatorTest { var calc = new ResourceExhaustionCalculator(true, mapOf(usage("disk", 0.5), usage("memory", 0.8))); var cf = createFixtureWithReportedUsages(forNode(1, usage("disk", 0.51), usage("memory", 0.79)), forNode(2, usage("disk", 0.4), usage("memory", 0.6))); - var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); + var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfos()); assertNotNull(feedBlock); assertTrue(feedBlock.blockFeedInCluster()); assertEquals("disk on node 1 [storage.1.local] (0.510 > 0.500)", feedBlock.getDescription()); @@ -43,7 +43,7 @@ public class ResourceExhaustionCalculatorTest { var calc = new ResourceExhaustionCalculator(true, mapOf(usage("disk", 0.5), usage("memory", 0.8))); var cf = createFixtureWithReportedUsages(forNode(1, usage("disk", "a-fancy-disk", 0.51), usage("memory", 0.79)), forNode(2, usage("disk", 0.4), usage("memory", 0.6))); - var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); + var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfos()); assertNotNull(feedBlock); assertTrue(feedBlock.blockFeedInCluster()); assertEquals("disk:a-fancy-disk on node 1 [storage.1.local] (0.510 > 0.500)", feedBlock.getDescription()); @@ -56,7 +56,7 @@ public class ResourceExhaustionCalculatorTest { forNode(2, usage("disk", 0.4), usage("memory", 0.85))); cf.cluster().getNodeInfo(storageNode(1)).setRpcAddress(null); cf.cluster().getNodeInfo(storageNode(2)).setRpcAddress("max mekker"); - var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); + var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfos()); assertNotNull(feedBlock); assertTrue(feedBlock.blockFeedInCluster()); assertEquals("disk on node 1 [unknown hostname] (0.510 > 0.500), " + @@ -68,7 +68,7 @@ public class ResourceExhaustionCalculatorTest { var calc = new ResourceExhaustionCalculator(true, mapOf(usage("disk", 0.4), usage("memory", 0.8))); var cf = createFixtureWithReportedUsages(forNode(1, usage("disk", 0.51), usage("memory", 0.85)), forNode(2, usage("disk", 0.45), usage("memory", 0.6))); - var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); + var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfos()); assertNotNull(feedBlock); assertTrue(feedBlock.blockFeedInCluster()); assertEquals("disk on node 1 [storage.1.local] (0.510 > 0.400), " + @@ -83,7 +83,7 @@ public class ResourceExhaustionCalculatorTest { var cf = createFixtureWithReportedUsages(forNode(1, usage("disk", 0.51), usage("memory", 0.85)), forNode(2, usage("disk", 0.45), usage("memory", 0.6)), forNode(3, usage("disk", 0.6), usage("memory", 0.9))); - var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); + var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfos()); assertNotNull(feedBlock); assertTrue(feedBlock.blockFeedInCluster()); assertEquals("disk on node 1 [storage.1.local] (0.510 > 0.400), " + @@ -97,7 +97,7 @@ public class ResourceExhaustionCalculatorTest { var calc = new ResourceExhaustionCalculator(false, mapOf(usage("disk", 0.5), usage("memory", 0.8))); var cf = createFixtureWithReportedUsages(forNode(1, usage("disk", 0.51), usage("memory", 0.79)), forNode(2, usage("disk", 0.4), usage("memory", 0.6))); - var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); + var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfos()); assertNull(feedBlock); } @@ -109,7 +109,7 @@ public class ResourceExhaustionCalculatorTest { // Node 2 is at 0.49 but was not previously blocked and should not be blocked now either. var cf = createFixtureWithReportedUsages(forNode(1, usage("disk", 0.3), usage("memory", 0.49)), forNode(2, usage("disk", 0.3), usage("memory", 0.49))); - var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); + var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfos()); assertNotNull(feedBlock); // TODO should we not change the limits themselves? Explicit mention of hysteresis state? assertEquals("memory on node 1 [storage.1.local] (0.490 > 0.400)", @@ -124,7 +124,7 @@ public class ResourceExhaustionCalculatorTest { // Node 2 is at 0.49 but was not previously blocked and should not be blocked now either. var cf = createFixtureWithReportedUsages(forNode(1, usage("disk", 0.3), usage("memory", 0.48)), forNode(2, usage("disk", 0.3), usage("memory", 0.49))); - var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); + var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfos()); assertNotNull(feedBlock); assertEquals("memory on node 1 [storage.1.local] (0.480 > 0.400)", feedBlock.getDescription()); @@ -138,7 +138,7 @@ public class ResourceExhaustionCalculatorTest { // Node 2 is at 0.49 but was not previously blocked and should not be blocked now either. var cf = createFixtureWithReportedUsages(forNode(1, usage("disk", 0.3), usage("memory", 0.39)), forNode(2, usage("disk", 0.3), usage("memory", 0.49))); - var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); + var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfos()); assertNull(feedBlock); } @@ -149,7 +149,7 @@ public class ResourceExhaustionCalculatorTest { forNode(2, usage("disk", 0.6), usage("memory", 0.6))); cf.reportStorageNodeState(1, State.DOWN); cf.reportStorageNodeState(2, State.DOWN); - var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); + var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfos()); assertNull(feedBlock); } @@ -160,7 +160,7 @@ public class ResourceExhaustionCalculatorTest { forNode(2, usage("disk", 0.6), usage("memory", 0.6))); cf.proposeStorageNodeWantedState(1, State.DOWN); cf.proposeStorageNodeWantedState(2, State.MAINTENANCE); - var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfo()); + var feedBlock = calc.inferContentClusterFeedBlockOrNull(cf.cluster().getNodeInfos()); assertNull(feedBlock); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceUsageStatsTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceUsageStatsTest.java index 3a5f9954a20..2eeaf7658ff 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceUsageStatsTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceUsageStatsTest.java @@ -66,7 +66,7 @@ public class ResourceUsageStatsTest { } private static Collection<NodeInfo> createNodeInfo(FeedBlockUtil.NodeAndUsages... nodeAndUsages) { - return createFixtureWithReportedUsages(nodeAndUsages).cluster().getNodeInfo(); + return createFixtureWithReportedUsages(nodeAndUsages).cluster().getNodeInfos(); } private static Map<String, Double> createFeedBlockLimits(double diskLimit, double memoryLimit) { diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java index 5395048cad9..47ba7e1cb77 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java @@ -99,7 +99,7 @@ public class SlobrokTest extends FleetControllerTest { private boolean clusterAvailable() { boolean ok = true; ContentCluster cluster = fleetController.getCluster(); - for (NodeInfo info : cluster.getNodeInfo()) { + for (NodeInfo info : cluster.getNodeInfos()) { if (info.getConnectionAttemptCount() > 0) ok = false; if (info.getLatestNodeStateRequestTime() == null) ok = false; } @@ -107,7 +107,7 @@ public class SlobrokTest extends FleetControllerTest { } private void assertClusterAvailable() { ContentCluster cluster = fleetController.getCluster(); - for (NodeInfo info : cluster.getNodeInfo()) { + for (NodeInfo info : cluster.getNodeInfos()) { assertEquals("Node " + info + " connection attempts.", 0, info.getConnectionAttemptCount()); assertTrue("Node " + info + " has no last request time.", info.getLatestNodeStateRequestTime() != 0); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandlerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandlerTest.java index 95c097c5920..699a35a190c 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandlerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandlerTest.java @@ -9,7 +9,7 @@ import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeStateOrHostInfoChangeHandler; +import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; import com.yahoo.vespa.clustercontroller.core.mocks.TestEventLog; import com.yahoo.vespa.clustercontroller.core.testutils.LogFormatter; import org.junit.Before; @@ -34,7 +34,7 @@ public class StateChangeHandlerTest { int maxPrematureCrashes = 3; } - private static class TestNodeStateOrHostInfoChangeHandler implements NodeStateOrHostInfoChangeHandler { + private static class TestNodeListener implements NodeListener { LinkedList<String> events = new LinkedList<>(); @@ -49,6 +49,11 @@ public class StateChangeHandlerTest { } @Override + public void handleRemovedNode(Node node) { + events.add("removed: " + node); + } + + @Override public void handleUpdatedHostInfo(NodeInfo node, HostInfo newHostInfo) { events.add(node + " - " + newHostInfo); } @@ -68,7 +73,7 @@ public class StateChangeHandlerTest { private Config config; private ContentCluster cluster; private StateChangeHandler nodeStateChangeHandler; - private TestNodeStateOrHostInfoChangeHandler nodeStateUpdateListener; + private TestNodeListener nodeStateUpdateListener; private final ClusterStateGenerator.Params params = new ClusterStateGenerator.Params(); @Before @@ -88,7 +93,7 @@ public class StateChangeHandlerTest { .maxPrematureCrashes(config.maxPrematureCrashes) .transitionTimes(5000) .cluster(cluster); - nodeStateUpdateListener = new TestNodeStateOrHostInfoChangeHandler(); + nodeStateUpdateListener = new TestNodeListener(); } private ClusterState currentClusterState() { diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java index a5bb65e11d0..5a33414c955 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateChangeTest.java @@ -1294,7 +1294,7 @@ public class StateChangeTest extends FleetControllerTest { NodeState newNodeState = new NodeState(NodeType.STORAGE, State.MAINTENANCE); NodeInfo nodeInfo = ctx.cluster.getNodeInfo(new Node(NodeType.STORAGE, 0)); nodeInfo.setWantedState(newNodeState); - ctx.nodeStateOrHostInfoChangeHandler.handleNewWantedNodeState(nodeInfo, newNodeState); + ctx.nodeListener.handleNewWantedNodeState(nodeInfo, newNodeState); invoked = true; } } @@ -1312,7 +1312,7 @@ public class StateChangeTest extends FleetControllerTest { NodeState newNodeState = new NodeState(NodeType.STORAGE, State.DOWN); NodeInfo nodeInfo = ctx.cluster.getNodeInfo(new Node(NodeType.STORAGE, 0)); nodeInfo.setWantedState(newNodeState); - ctx.nodeStateOrHostInfoChangeHandler.handleNewWantedNodeState(nodeInfo, newNodeState); + ctx.nodeListener.handleNewWantedNodeState(nodeInfo, newNodeState); invoked = true; } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcasterTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcasterTest.java index 45593375c0b..1832f1132ac 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcasterTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcasterTest.java @@ -6,8 +6,8 @@ import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeAddedOrRemovedListener; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeStateOrHostInfoChangeHandler; +import com.yahoo.vespa.clustercontroller.core.listeners.SlobrokListener; +import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; import org.junit.Test; import org.mockito.ArgumentCaptor; @@ -69,12 +69,12 @@ public class SystemStateBroadcasterTest { } @Override - public NodeAddedOrRemovedListener getNodeAddedOrRemovedListener() { + public SlobrokListener getNodeAddedOrRemovedListener() { return null; } @Override - public NodeStateOrHostInfoChangeHandler getNodeStateUpdateListener() { + public NodeListener getNodeStateUpdateListener() { return null; } }; @@ -91,7 +91,7 @@ public class SystemStateBroadcasterTest { ClusterFixture cf = ClusterFixture.forFlatCluster(2).bringEntireClusterUp().assignDummyRpcAddresses(); f.broadcaster.handleNewClusterStates(stateBundle); f.broadcaster.broadcastNewStateBundleIfRequired(dbContextFrom(cf.cluster()), f.mockCommunicator, 3); - cf.cluster().getNodeInfo().forEach(nodeInfo -> verify(f.mockCommunicator).setSystemState(eq(stateBundle), eq(nodeInfo), any())); + cf.cluster().getNodeInfos().forEach(nodeInfo -> verify(f.mockCommunicator).setSystemState(eq(stateBundle), eq(nodeInfo), any())); } @Test @@ -121,7 +121,7 @@ public class SystemStateBroadcasterTest { f.broadcaster.handleNewClusterStates(stateBundle); f.broadcaster.broadcastNewStateBundleIfRequired(dbContextFrom(cf.cluster()), f.mockCommunicator, 3); - cf.cluster().getNodeInfo().forEach(nodeInfo -> verify(f.mockCommunicator).setSystemState(eq(stateBundle), eq(nodeInfo), any())); + cf.cluster().getNodeInfos().forEach(nodeInfo -> verify(f.mockCommunicator).setSystemState(eq(stateBundle), eq(nodeInfo), any())); } @Test @@ -153,7 +153,7 @@ public class SystemStateBroadcasterTest { f.broadcaster.handleNewClusterStates(stateBundle); f.broadcaster.broadcastNewStateBundleIfRequired(dbContextFrom(cf.cluster()), f.mockCommunicator, 99); - cf.cluster().getNodeInfo().forEach(nodeInfo -> { + cf.cluster().getNodeInfos().forEach(nodeInfo -> { verify(f.mockCommunicator, times(0)).setSystemState(any(), eq(nodeInfo), any()); }); } @@ -166,7 +166,7 @@ public class SystemStateBroadcasterTest { f.broadcaster.handleNewClusterStates(stateBundle); f.broadcaster.broadcastNewStateBundleIfRequired(dbContextFrom(cf.cluster()), f.mockCommunicator, 100); - cf.cluster().getNodeInfo().forEach(nodeInfo -> { + cf.cluster().getNodeInfos().forEach(nodeInfo -> { verify(f.mockCommunicator, times(1)).setSystemState(any(), eq(nodeInfo), any()); }); } @@ -276,7 +276,7 @@ public class SystemStateBroadcasterTest { f.simulateBroadcastTick(cf, 123); // No activations should be sent yet - cf.cluster().getNodeInfo().forEach(nodeInfo -> { + cf.cluster().getNodeInfos().forEach(nodeInfo -> { verify(f.mockCommunicator, times(0)).activateClusterStateVersion(eq(123), eq(nodeInfo), any()); }); assertNull(f.broadcaster.getLastClusterStateBundleConverged()); @@ -285,7 +285,7 @@ public class SystemStateBroadcasterTest { f.simulateBroadcastTick(cf, 123); // Activation should now be sent to _all_ nodes (distributor and storage) - cf.cluster().getNodeInfo().forEach(nodeInfo -> { + cf.cluster().getNodeInfos().forEach(nodeInfo -> { verify(f.mockCommunicator).activateClusterStateVersion(eq(123), eq(nodeInfo), any()); }); // But not converged yet, as activations have not been ACKed diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ZooKeeperDatabaseTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ZooKeeperDatabaseTest.java index 7d64a8f8878..1ce7586adea 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ZooKeeperDatabaseTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ZooKeeperDatabaseTest.java @@ -37,7 +37,7 @@ public class ZooKeeperDatabaseTest { closeDatabaseIfOpen(); var id = new FleetControllerId(clusterFixture.cluster.getName(), nodeIndex); var context = new TestFleetControllerContext(id); - zkDatabase = new ZooKeeperDatabase(context, clusterFixture.cluster(), nodeIndex, zkServer.getAddress(), + zkDatabase = new ZooKeeperDatabase(context, zkServer.getAddress(), (int)sessionTimeout.toMillis(), mockListener); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterControllerMock.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterControllerMock.java index 95071931a75..f53b2898145 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterControllerMock.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/ClusterControllerMock.java @@ -2,11 +2,12 @@ package com.yahoo.vespa.clustercontroller.core.restapiv2; import com.yahoo.vdslib.state.ClusterState; +import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vespa.clustercontroller.core.*; import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeAddedOrRemovedListener; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeStateOrHostInfoChangeHandler; +import com.yahoo.vespa.clustercontroller.core.listeners.SlobrokListener; +import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; public class ClusterControllerMock implements RemoteClusterControllerTaskScheduler { public RemoteClusterControllerTask.Context context = new RemoteClusterControllerTask.Context(); @@ -40,16 +41,21 @@ public class ClusterControllerMock implements RemoteClusterControllerTaskSchedul return fleetControllerMaster; } }; - context.nodeStateOrHostInfoChangeHandler = new NodeStateOrHostInfoChangeHandler() { + context.nodeListener = new NodeListener() { @Override public void handleNewNodeState(NodeInfo currentInfo, NodeState newState) { - events.append("newNodeState(").append(currentInfo.getNode()).append(": ").append(newState).append("\n"); + events.append("newNodeState(").append(currentInfo.getNode()).append(": ").append(newState).append('\n'); } @Override public void handleNewWantedNodeState(NodeInfo node, NodeState newState) { - events.append("newWantedNodeState(").append(node.getNode()).append(": ").append(newState).append("\n"); + events.append("newWantedNodeState(").append(node.getNode()).append(": ").append(newState).append('\n'); + } + + @Override + public void handleRemovedNode(Node node) { + events.append("handleRemovedNode(").append(node).append(")\n"); } @Override @@ -59,7 +65,7 @@ public class ClusterControllerMock implements RemoteClusterControllerTaskSchedul } }; - context.nodeAddedOrRemovedListener = new NodeAddedOrRemovedListener() { + context.slobrokListener = new SlobrokListener() { @Override public void handleNewNode(NodeInfo node) { @@ -68,12 +74,12 @@ public class ClusterControllerMock implements RemoteClusterControllerTaskSchedul @Override public void handleMissingNode(NodeInfo node) { - events.append("newMissingNode(").append(node.getNode()).append("\n"); + events.append("newMissingNode(").append(node.getNode()).append('\n'); } @Override public void handleNewRpcAddress(NodeInfo node) { - events.append("newRpcAddress(").append(node.getNode()).append("\n"); + events.append("newRpcAddress(").append(node.getNode()).append('\n'); } @Override diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequestTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequestTest.java index 090e80361e5..9f14b2e71d2 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequestTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequestTest.java @@ -10,7 +10,7 @@ import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.ContentCluster; import com.yahoo.vespa.clustercontroller.core.NodeInfo; import com.yahoo.vespa.clustercontroller.core.NodeStateChangeChecker; -import com.yahoo.vespa.clustercontroller.core.listeners.NodeStateOrHostInfoChangeHandler; +import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.StateRestApiException; import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest; import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.SetResponse; @@ -38,7 +38,7 @@ public class SetNodeStateRequestTest { private final UnitState unitState = mock(UnitState.class); private final int NODE_INDEX = 2; private final Node storageNode = new Node(NodeType.STORAGE, NODE_INDEX); - private final NodeStateOrHostInfoChangeHandler stateListener = mock(NodeStateOrHostInfoChangeHandler.class); + private final NodeListener stateListener = mock(NodeListener.class); private final ClusterState currentClusterState = mock(ClusterState.class); private boolean inMasterMoratorium = false; private boolean probe = false; diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/testutils/WaitTask.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/testutils/WaitTask.java index 6362d6fe9a7..d9967381e75 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/testutils/WaitTask.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/testutils/WaitTask.java @@ -25,7 +25,7 @@ public abstract class WaitTask { public boolean performWaitTask() { boolean didWork = false; synchronized (fleetController.getMonitor()) { - for (NodeInfo info : fleetController.getCluster().getNodeInfo()) { + for (NodeInfo info : fleetController.getCluster().getNodeInfos()) { if (info.getTimeForNextStateRequestAttempt() != 0) didWork = true; info.setNextGetStateAttemptTime(0); } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 0579aebe771..78d195821bf 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -60,6 +60,8 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.BiFunction; +import java.util.function.Function; import java.util.logging.Level; import java.util.logging.Logger; @@ -114,7 +116,7 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { * @return an Application package instance */ public static FilesApplicationPackage fromFile(File appDir, boolean includeSourceFiles) { - return new Builder(appDir).preprocessedDir(new File(appDir, preprocessed)) + return new Builder(appDir).preprocessedDir(applicationFile(appDir, preprocessed)) .includeSourceFiles(includeSourceFiles) .build(); } @@ -156,7 +158,7 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { this.appDir = appDir; this.preprocessedDir = preprocessedDir; appSubDirs = new AppSubDirs(appDir); - configDefsDir = new File(appDir, CONFIG_DEFINITIONS_DIR); + configDefsDir = applicationFile(appDir, CONFIG_DEFINITIONS_DIR); addUserIncludeDirs(); this.metaData = metaData; transformerFactory = TransformerFactory.newInstance(); @@ -178,7 +180,7 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { @Override public ApplicationFile getFile(Path path) { - File file = (path.isRoot() ? appDir : new File(appDir, path.getRelative())); + File file = (path.isRoot() ? appDir : applicationFile(appDir, path.getRelative())); return new FilesApplicationFile(path, file); } @@ -190,7 +192,7 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { private List<NamedReader> getFiles(Path relativePath, String namePrefix, String suffix, boolean recurse) { try { List<NamedReader> readers=new ArrayList<>(); - File dir = new File(appDir, relativePath.getRelative()); + File dir = applicationFile(appDir, relativePath); if ( ! dir.isDirectory()) return readers; File[] files = dir.listFiles(); @@ -242,7 +244,7 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { } private File getHostsFile() { - return new File(appDir, HOSTS); + return applicationFile(appDir, HOSTS); } @Override @@ -251,7 +253,7 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { } private File getServicesFile() { - return new File(appDir, SERVICES); + return applicationFile(appDir, SERVICES); } @Override @@ -430,11 +432,11 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { static List<File> getSearchDefinitionFiles(File appDir) { List<File> schemaFiles = new ArrayList<>(); - File sdDir = new File(appDir, SEARCH_DEFINITIONS_DIR.getRelative()); + File sdDir = applicationFile(appDir, SEARCH_DEFINITIONS_DIR.getRelative()); if (sdDir.isDirectory()) schemaFiles.addAll(Arrays.asList(sdDir.listFiles((dir, name) -> name.matches(".*\\" + SD_NAME_SUFFIX)))); - sdDir = new File(appDir, SCHEMAS_DIR.getRelative()); + sdDir = applicationFile(appDir, SCHEMAS_DIR.getRelative()); if (sdDir.isDirectory()) schemaFiles.addAll(Arrays.asList(sdDir.listFiles((dir, name) -> name.matches(".*\\" + SD_NAME_SUFFIX)))); @@ -447,17 +449,17 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { // Only for use by deploy processor public static List<Component> getComponents(File appDir) { - List<Component> components = new ArrayList<>(); - for (Bundle bundle : Bundle.getBundles(new File(appDir, COMPONENT_DIR))) { - components.add(new Component(bundle, new ComponentInfo(new File(COMPONENT_DIR, bundle.getFile().getName()).getPath()))); - } - return components; + return components(appDir, Component::new); } private static List<ComponentInfo> getComponentsInfo(File appDir) { - List<ComponentInfo> components = new ArrayList<>(); - for (Bundle bundle : Bundle.getBundles(new File(appDir, COMPONENT_DIR))) { - components.add(new ComponentInfo(new File(COMPONENT_DIR, bundle.getFile().getName()).getPath())); + return components(appDir, (__, info) -> info); + } + + private static <T> List<T> components(File appDir, BiFunction<Bundle, ComponentInfo, T> toValue) { + List<T> components = new ArrayList<>(); + for (Bundle bundle : Bundle.getBundles(applicationFile(appDir, COMPONENT_DIR))) { + components.add(toValue.apply(bundle, new ComponentInfo(Path.fromString(COMPONENT_DIR).append(bundle.getFile().getName()).getRelative()))); } return components; } @@ -492,7 +494,7 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { "", 0L, 0L); - File metaFile = new File(appDir, META_FILE_NAME); + File metaFile = applicationFile(appDir, META_FILE_NAME); if ( ! metaFile.exists()) { return defaultMetaData; } @@ -550,18 +552,16 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { throw new IllegalArgumentException("Absolute path to ranking expression file is not allowed: " + name); Path path = Path.fromString(name); - File sdDir = new File(appDir, SCHEMAS_DIR.getRelative()); - File expressionFile = new File(sdDir, path.getRelative()); + File expressionFile = applicationFile(appDir, SCHEMAS_DIR.append(path)); if ( ! expressionFile.exists()) { - sdDir = new File(appDir, SEARCH_DEFINITIONS_DIR.getRelative()); - expressionFile = new File(sdDir, path.getRelative()); + expressionFile = applicationFile(appDir, SEARCH_DEFINITIONS_DIR.append(path)); } return expressionFile; } @Override public File getFileReference(Path pathRelativeToAppDir) { - return new File(appDir, pathRelativeToAppDir.getRelative()); + return applicationFile(appDir, pathRelativeToAppDir.getRelative()); } @Override @@ -579,7 +579,7 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { @Override public void writeMetaData() throws IOException { - File metaFile = new File(appDir, META_FILE_NAME); + File metaFile = applicationFile(appDir, META_FILE_NAME); IOUtils.writeFile(metaFile, metaData.asJsonBytes()); } @@ -604,13 +604,11 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { @Override public ApplicationPackage preprocess(Zone zone, DeployLogger logger) throws IOException { IOUtils.recursiveDeleteDir(preprocessedDir); - IOUtils.copyDirectory(appDir, preprocessedDir, -1, (dir, name) -> ! name.equals(preprocessed) && - ! name.equals(SERVICES) && - ! name.equals(HOSTS) && - ! name.equals(CONFIG_DEFINITIONS_DIR)); + IOUtils.copyDirectory(appDir, preprocessedDir, -1, + (__, name) -> ! List.of(preprocessed, SERVICES, HOSTS, CONFIG_DEFINITIONS_DIR).contains(name)); File servicesFile = validateServicesFile(); - preprocessXML(new File(preprocessedDir, SERVICES), servicesFile, zone); - preprocessXML(new File(preprocessedDir, HOSTS), getHostsFile(), zone); + preprocessXML(applicationFile(preprocessedDir, SERVICES), servicesFile, zone); + preprocessXML(applicationFile(preprocessedDir, HOSTS), getHostsFile(), zone); FilesApplicationPackage preprocessed = fromFile(preprocessedDir, includeSourceFiles); preprocessed.copyUserDefsIntoApplication(); return preprocessed; @@ -733,10 +731,22 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { } public FilesApplicationPackage build() { - return new FilesApplicationPackage(appDir, preprocessedDir.orElse(new File(appDir, preprocessed)), + return new FilesApplicationPackage(appDir, preprocessedDir.orElse(applicationFile(appDir, preprocessed)), metaData.orElse(readMetaData(appDir)), includeSourceFiles); } } + static File applicationFile(File parent, String path) { + return applicationFile(parent, Path.fromString(path)); + } + + static File applicationFile(File parent, Path path) { + File file = new File(parent, path.getRelative()); + if ( ! file.getAbsolutePath().startsWith(parent.getAbsolutePath())) + throw new IllegalArgumentException(file + " is not a child of " + parent); + + return file; + } + } diff --git a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java index ae6f9373e16..fd5437c27de 100644 --- a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java @@ -16,8 +16,10 @@ import java.io.FileReader; import java.io.IOException; import java.nio.file.Files; +import static com.yahoo.config.model.application.provider.FilesApplicationPackage.applicationFile; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -126,4 +128,19 @@ public class FilesApplicationPackageTest { } } + @Test + public void testApplicationFile() { + applicationFile(new File("foo"), ""); + applicationFile(new File("foo"), "bar"); + applicationFile(new File(new File(""), ""), ""); + assertEquals("/ is not a child of ", + assertThrows(IllegalArgumentException.class, + () -> applicationFile(new File(""), "")) + .getMessage()); + assertEquals("'..' is not allowed in path", + assertThrows(IllegalArgumentException.class, + () -> applicationFile(new File("foo"), "..")) + .getMessage()); + } + } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java index 50556e3bad7..0424b1dae16 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java @@ -14,7 +14,6 @@ import java.util.Objects; * @author jonmv */ public class ZoneId { - // TODO: Replace usages of environment + region with usages of this. private final Environment environment; private final RegionName region; diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 4dcc56ed2d8..67bc553c478 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -6635,11 +6635,13 @@ ], "methods": [ "public void <init>()", - "public java.lang.Object get(com.yahoo.processing.request.CompoundName, java.util.Map, com.yahoo.processing.request.Properties)" + "public java.lang.Object get(com.yahoo.processing.request.CompoundName, java.util.Map, com.yahoo.processing.request.Properties)", + "public static void requireNotPresentIn(java.util.Map)" ], "fields": [ "public static final com.yahoo.processing.request.CompoundName MAX_OFFSET", "public static final com.yahoo.processing.request.CompoundName MAX_HITS", + "public static final com.yahoo.processing.request.CompoundName MAX_QUERY_ITEMS", "public static final com.yahoo.search.query.profile.types.QueryProfileType argumentType" ] }, @@ -8268,6 +8270,18 @@ "public static final com.yahoo.processing.request.CompoundName dryRunKey" ] }, + "com.yahoo.search.searchers.ValidateFuzzySearcher": { + "superClass": "com.yahoo.search.Searcher", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.vespa.config.search.AttributesConfig)", + "public com.yahoo.search.Result search(com.yahoo.search.Query, com.yahoo.search.searchchain.Execution)" + ], + "fields": [] + }, "com.yahoo.search.searchers.ValidateMatchPhaseSearcher": { "superClass": "com.yahoo.search.Searcher", "interfaces": [], diff --git a/container-search/src/main/java/com/yahoo/prelude/query/FuzzyItem.java b/container-search/src/main/java/com/yahoo/prelude/query/FuzzyItem.java index b26205b74e9..ea2a7752809 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/FuzzyItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/FuzzyItem.java @@ -28,14 +28,10 @@ public class FuzzyItem extends TermItem { } public void setMaxEditDistance(int maxEditDistance) { - if (maxEditDistance < 0) - throw new IllegalArgumentException("Can not use negative maxEditDistance " + maxEditDistance); this.maxEditDistance = maxEditDistance; } public void setPrefixLength(int prefixLength) { - if (prefixLength < 0) - throw new IllegalArgumentException("Can not use negative prefixLength " + prefixLength); this.prefixLength = prefixLength; } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java b/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java index a93dd1b9de4..916f23bd768 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java @@ -4,6 +4,7 @@ package com.yahoo.prelude.query; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.query.QueryTree; +import com.yahoo.search.query.properties.DefaultProperties; import java.util.HashSet; import java.util.ListIterator; @@ -20,8 +21,6 @@ public class QueryCanonicalizer { /** The name of the operation performed by this, for use in search chain ordering */ public static final String queryCanonicalization = "queryCanonicalization"; - private static final CompoundName MAX_QUERY_ITEMS = new CompoundName("maxQueryItems"); - /** * Validates this query and carries out possible operations on this query * which simplifies it without changing its semantics. @@ -29,8 +28,8 @@ public class QueryCanonicalizer { * @return null if the query is valid, an error message if it is invalid */ public static String canonicalize(Query query) { - Integer maxQueryItems = query.properties().getInteger(MAX_QUERY_ITEMS, Integer.MAX_VALUE); - return canonicalize(query.getModel().getQueryTree(), maxQueryItems); + return canonicalize(query.getModel().getQueryTree(), + query.properties().getInteger(DefaultProperties.MAX_QUERY_ITEMS)); } /** @@ -52,7 +51,8 @@ public class QueryCanonicalizer { CanonicalizationResult result = recursivelyCanonicalize(rootItemIterator.next(), rootItemIterator); if (query.isEmpty() && ! result.isError()) result = CanonicalizationResult.error("No query"); int itemCount = query.treeSize(); - if (itemCount > maxQueryItems) result = CanonicalizationResult.error(String.format("Query tree exceeds allowed item count. Configured limit: %d - Item count: %d", maxQueryItems, itemCount)); + if (itemCount > maxQueryItems) + result = CanonicalizationResult.error(String.format("Query tree exceeds allowed item count. Configured limit: %d - Item count: %d", maxQueryItems, itemCount)); return result.error().orElse(null); // preserve old API, unfortunately } diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index af6374ba245..54d8ac40556 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -580,11 +580,7 @@ public class SearchHandler extends LoggingRequestHandler { } private Result validateQuery(Query query) { - if (query.getHttpRequest().getProperty(DefaultProperties.MAX_HITS.toString()) != null) - throw new RuntimeException(DefaultProperties.MAX_HITS + " must be specified in a query profile."); - - if (query.getHttpRequest().getProperty(DefaultProperties.MAX_OFFSET.toString()) != null) - throw new RuntimeException(DefaultProperties.MAX_OFFSET + " must be specified in a query profile."); + DefaultProperties.requireNotPresentIn(query.getHttpRequest().propertyMap()); int maxHits = query.properties().getInteger(DefaultProperties.MAX_HITS); int maxOffset = query.properties().getInteger(DefaultProperties.MAX_OFFSET); diff --git a/container-search/src/main/java/com/yahoo/search/query/QueryTree.java b/container-search/src/main/java/com/yahoo/search/query/QueryTree.java index 3dac5648660..0655727b46b 100644 --- a/container-search/src/main/java/com/yahoo/search/query/QueryTree.java +++ b/container-search/src/main/java/com/yahoo/search/query/QueryTree.java @@ -185,7 +185,7 @@ public class QueryTree extends CompositeItem { */ public int treeSize() { if (isEmpty()) return 0; - return(countItemsRecursively(getItemIterator().next())); + return countItemsRecursively(getItemIterator().next()); } private int countItemsRecursively(Item item) { diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/DefaultProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/DefaultProperties.java index b94ddde4733..221368afeb6 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/DefaultProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/DefaultProperties.java @@ -6,6 +6,7 @@ import com.yahoo.search.query.Properties; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileType; +import java.util.List; import java.util.Map; /** @@ -17,26 +18,30 @@ public final class DefaultProperties extends Properties { public static final CompoundName MAX_OFFSET = new CompoundName("maxOffset"); public static final CompoundName MAX_HITS = new CompoundName("maxHits"); + public static final CompoundName MAX_QUERY_ITEMS = new CompoundName("maxQueryItems"); public static final QueryProfileType argumentType = new QueryProfileType("DefaultProperties"); + private static final List<CompoundName> properties = List.of(MAX_OFFSET, MAX_HITS, MAX_QUERY_ITEMS); + static { argumentType.setBuiltin(true); - - argumentType.addField(new FieldDescription(MAX_OFFSET.toString(), "integer")); - argumentType.addField(new FieldDescription(MAX_HITS.toString(), "integer")); - + properties.forEach(property -> argumentType.addField(new FieldDescription(property.toString(), "integer"))); argumentType.freeze(); } @Override public Object get(CompoundName name, Map<String, String> context, com.yahoo.processing.request.Properties substitution) { - if (MAX_OFFSET.equals(name)) { - return 1000; - } else if (MAX_HITS.equals(name)) { - return 400; - } else { - return super.get(name, context, substitution); + if (name.equals(MAX_OFFSET)) return 1000; + if (name.equals(MAX_HITS)) return 400; + if (name.equals(MAX_QUERY_ITEMS)) return 10000; + return super.get(name, context, substitution); + } + + public static void requireNotPresentIn(Map<String, String> map) { + for (var property : properties) { + if (map.containsKey(property.toString())) + throw new IllegalArgumentException(property + " must be specified in a query profile."); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterFeatures.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterFeatures.java index bb5792d81b7..c3905b8200b 100644 --- a/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterFeatures.java +++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterFeatures.java @@ -68,7 +68,7 @@ public class RewriterFeatures { oldRoot.equals(origQueryItem)) { PhraseItem phrase = convertAndToPhrase((AndItem)oldRoot); - if(!keepOriginalQuery) { + if (!keepOriginalQuery) { qTree.setRoot(phrase); } else { OrItem newRoot = new OrItem(); @@ -145,12 +145,12 @@ public class RewriterFeatures { } StringTokenizer rewrite_list = new StringTokenizer(rewrites, "\t"); - Item rI = null; + Item rI; // Convert matching string to query tree item Item matchingStrItem = convertStringToQTree(query, matchingStr); PhraseItem matchingStrPhraseItem = null; - if(matchingStrItem instanceof AndItem) { + if (matchingStrItem instanceof AndItem) { matchingStrPhraseItem = convertAndToPhrase(((AndItem)matchingStrItem)); } @@ -166,30 +166,32 @@ public class RewriterFeatures { // - matchingStr: (AND aa bb) // - for this case, should use getNonOverlappingMatches instead OrItem newRoot; - if(oldRoot instanceof OrItem) { - if(((OrItem)oldRoot).getItemIndex(matchingStrItem)==-1) { + if (oldRoot instanceof OrItem) { + if (((OrItem)oldRoot).getItemIndex(matchingStrItem)==-1) { RewriterUtils.log(logger, query, "Whole query matching is used, skipping rewrite"); return query; } newRoot = (OrItem)oldRoot; - } else if(oldRoot.equals(matchingStrItem) || oldRoot.equals(matchingStrPhraseItem)) { + } + else if(oldRoot.equals(matchingStrItem) || oldRoot.equals(matchingStrPhraseItem)) { newRoot = new OrItem(); newRoot.addItem(oldRoot); - } else { + } + else { RewriterUtils.log(logger, query, "Whole query matching is used, skipping rewrite"); return query; } int numRewrites = 0; - while(rewrite_list.hasMoreTokens() && - (maxNumRewrites==0 || numRewrites < maxNumRewrites)) { + while (rewrite_list.hasMoreTokens() && (maxNumRewrites == 0 || numRewrites < maxNumRewrites)) { rI = convertStringToQTree(query, rewrite_list.nextToken()); - if(addUnitToRewrites && rI instanceof AndItem) { + if (addUnitToRewrites && rI instanceof AndItem) { rI = convertAndToPhrase((AndItem)rI); } - if(newRoot.getItemIndex(rI)==-1) { + if(newRoot.getItemIndex(rI) == -1) { newRoot.addItem(rI); numRewrites++; - } else { + } + else { RewriterUtils.log(logger, query, "Rewrite already exist, skipping"); } } @@ -229,19 +231,19 @@ public class RewriterFeatures { Query query) throws RuntimeException { RewriterUtils.log(logger, query, "Retrieving longest non-overlapping full phrase matches"); - if(phraseMatcher==null) + if (phraseMatcher == null) return null; Item root = query.getModel().getQueryTree().getRoot(); List<PhraseMatcher.Phrase> matches = phraseMatcher.matchPhrases(root); - if (matches==null || matches.isEmpty()) + if (matches == null || matches.isEmpty()) return null; Set<PhraseMatcher.Phrase> resultMatches = new HashSet<>(); ListIterator<Phrase> matchesIter = matches.listIterator(); // Iterate through all matches - while(matchesIter.hasNext()) { + while (matchesIter.hasNext()) { PhraseMatcher.Phrase phrase = matchesIter.next(); RewriterUtils.log(logger, query, "Working on phrase: " + phrase); CompositeItem currOwner = phrase.getOwner(); @@ -250,11 +252,11 @@ public class RewriterFeatures { // If phrase is not an AND item, only keep those that are single word // in order to eliminate cases such as (new RANK york) from being treated // as match if only new york but not new or york is in the dictionary - if((currOwner!=null && + if((currOwner != null && ((phrase.isComplete() && currOwner instanceof AndItem) || - (phrase.getLength()==1 && currOwner instanceof OrItem) || - (phrase.getLength()==1 && currOwner instanceof RankItem && phrase.getStartIndex()==0))) || - (currOwner==null && phrase.getLength()==1)) { + (phrase.getLength() == 1 && currOwner instanceof OrItem) || + (phrase.getLength() == 1 && currOwner instanceof RankItem && phrase.getStartIndex() == 0))) || + (currOwner == null && phrase.getLength() == 1)) { resultMatches.add(phrase); RewriterUtils.log(logger, query, "Keeping phrase: " + phrase); } @@ -298,12 +300,12 @@ public class RewriterFeatures { Query query) throws RuntimeException { RewriterUtils.log(logger, query, "Retrieving longest non-overlapping partial phrase matches"); - if(phraseMatcher==null) + if (phraseMatcher == null) return null; Item root = query.getModel().getQueryTree().getRoot(); List<PhraseMatcher.Phrase> matches = phraseMatcher.matchPhrases(root); - if (matches==null || matches.isEmpty()) + if (matches == null || matches.isEmpty()) return null; Set<PhraseMatcher.Phrase> resultMatches = new HashSet<>(); @@ -312,14 +314,14 @@ public class RewriterFeatures { ListIterator<PhraseMatcher.Phrase> matchesIter = matches.listIterator(); // Iterate through all matches - while(matchesIter.hasNext()) { + while (matchesIter.hasNext()) { PhraseMatcher.Phrase phrase = matchesIter.next(); RewriterUtils.log(logger, query, "Working on phrase: " + phrase); CompositeItem currOwner = phrase.getOwner(); // Check if previous is AND item and this phrase is in a different item // If so, work on the previous set to eliminate overlapping matches - if(!phrasesInSubTree.isEmpty() && currOwner!=null && + if (!phrasesInSubTree.isEmpty() && currOwner!=null && prevOwner!=null && !currOwner.equals(prevOwner)) { RewriterUtils.log(logger, query, "Previous phrase is in different AND item"); List<PhraseMatcher.Phrase> subTreeMatches @@ -333,13 +335,13 @@ public class RewriterFeatures { } // Check if this is an AND item - if(currOwner!=null && currOwner instanceof AndItem) { + if (currOwner instanceof AndItem) { phrasesInSubTree.add(phrase); - // If phrase is not an AND item, only keep those that are single word - // in order to eliminate cases such as (new RANK york) from being treated - // as match if only new york but not new or york is in the dictionary - } else if (phrase.getLength()==1 && - !(currOwner!=null && currOwner instanceof RankItem && phrase.getStartIndex()!=0)) { + // If phrase is not an AND item, only keep those that are single word + // in order to eliminate cases such as (new RANK york) from being treated + // as match if only new york but not new or york is in the dictionary + } + else if (phrase.getLength() == 1 && !(currOwner instanceof RankItem && phrase.getStartIndex() != 0)) { resultMatches.add(phrase); } @@ -476,7 +478,7 @@ public class RewriterFeatures { boolean removeOriginal, boolean addUnitToRewrites) throws RuntimeException { - if(matches==null) { + if(matches == null) { RewriterUtils.log(logger, query, "No expansions to be added"); return query; } @@ -494,7 +496,7 @@ public class RewriterFeatures { // Retrieve expansion phrases String expansionStr = match.getData(); - if(expansionStr.equalsIgnoreCase("n/a") && expandIndex==null) { + if (expansionStr.equalsIgnoreCase("n/a") && expandIndex == null) { continue; } StringTokenizer expansions = new StringTokenizer(expansionStr,"\t"); @@ -509,17 +511,17 @@ public class RewriterFeatures { (maxNumRewrites==0 || numRewrites < maxNumRewrites)) { String expansion = expansions.nextToken(); RewriterUtils.log(logger, query, "Working on expansion: " + expansion); - if(expansion.equalsIgnoreCase("n/a")) { + if (expansion.equalsIgnoreCase("n/a")) { expansion = matchStr; } // (AND expansion) or "expansion" Item expansionItem = convertStringToQTree(query, expansion); - if(addUnitToRewrites && expansionItem instanceof AndItem) { + if (addUnitToRewrites && expansionItem instanceof AndItem) { expansionItem = convertAndToPhrase((AndItem)expansionItem); } expansionGrp.addItem(expansionItem); - if(expandIndex!=null) { + if (expandIndex!=null) { // indexName:expansion WordItem expansionIndexItem = new WordItem(expansion, expandIndex); expansionGrp.addItem(expansionIndexItem); @@ -528,19 +530,19 @@ public class RewriterFeatures { RewriterUtils.log(logger, query, "Adding expansion: " + expansion); } - if(!removeOriginal) { + if (!removeOriginal) { //(AND original) Item matchItem = convertStringToQTree(query, matchStr); - if(expansionGrp.getItemIndex(matchItem)==-1) { + if (expansionGrp.getItemIndex(matchItem)==-1) { expansionGrp.addItem(matchItem); } } parent = match.getOwner(); int matchIndex = match.getStartIndex(); - if(parent!=null) { + if (parent!=null) { // Remove matching phrase from original query - for(int i=0; i<match.getLength(); i++) { + for (int i=0; i<match.getLength(); i++) { parent.removeItem(matchIndex); } // Adding back expansions @@ -554,11 +556,11 @@ public class RewriterFeatures { } // Not root single item - if(parent!=null) { + if (parent != null) { // Cleaning up the query after rewrite to remove redundant tags // e.g. (AND (OR (AND a b) c)) => (OR (AND a b) c) String cleanupError = QueryCanonicalizer.canonicalize(qTree); - if(cleanupError!=null) { + if (cleanupError!=null) { RewriterUtils.error(logger, query, "Error canonicalizing query tree"); throw new RuntimeException("Error canonicalizing query tree"); } @@ -595,7 +597,7 @@ public class RewriterFeatures { */ static Item convertStringToQTree(Query query, String stringToParse) { RewriterUtils.log(logger, query, "Converting string [" + stringToParse + "] to query tree"); - if(stringToParse==null) { + if (stringToParse == null) { return new NullItem(); } Model model = query.getModel(); @@ -621,7 +623,7 @@ public class RewriterFeatures { Iterator<Item> subItems = andItem.getItemIterator(); while(subItems.hasNext()) { Item curr = (subItems.next()); - if(curr instanceof IntItem) { + if (curr instanceof IntItem) { WordItem numItem = new WordItem(((IntItem)curr).stringValue()); result.addItem(numItem); } else { @@ -639,7 +641,7 @@ public class RewriterFeatures { */ private static class PhraseLength implements Comparator<PhraseMatcher.Phrase> { public int compare(PhraseMatcher.Phrase phrase1, PhraseMatcher.Phrase phrase2) { - if((phrase2.getLength()>phrase1.getLength()) || + if ((phrase2.getLength()>phrase1.getLength()) || (phrase2.getLength()==phrase1.getLength() && phrase2.getStartIndex()<=phrase1.getStartIndex())) { return 1; @@ -648,4 +650,5 @@ public class RewriterFeatures { } } } + } diff --git a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/LocalProviderSpec.java b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/LocalProviderSpec.java index b8d6a050691..a12456f5354 100644 --- a/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/LocalProviderSpec.java +++ b/container-search/src/main/java/com/yahoo/search/searchchain/model/federation/LocalProviderSpec.java @@ -44,6 +44,7 @@ public class LocalProviderSpec { com.yahoo.prelude.searcher.ValidatePredicateSearcher.class, com.yahoo.search.searchers.ValidateNearestNeighborSearcher.class, com.yahoo.search.searchers.ValidateMatchPhaseSearcher.class, + com.yahoo.search.searchers.ValidateFuzzySearcher.class, com.yahoo.search.yql.FieldFiller.class, com.yahoo.search.searchers.InputCheckingSearcher.class, com.yahoo.search.searchers.ContainerLatencySearcher.class); diff --git a/container-search/src/main/java/com/yahoo/search/searchers/ValidateFuzzySearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/ValidateFuzzySearcher.java new file mode 100644 index 00000000000..249a6342da6 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/searchers/ValidateFuzzySearcher.java @@ -0,0 +1,95 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchers; + +import com.yahoo.prelude.query.FuzzyItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.ToolBox; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.Searcher; +import com.yahoo.search.grouping.vespa.GroupingExecutor; +import com.yahoo.search.query.ranking.RankProperties; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.yolean.chain.Before; + +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; + +/** + * Validates any FuzzyItem query items. + * + * @author alexeyche + */ +@Before(GroupingExecutor.COMPONENT_NAME) // Must happen before query.prepare() +public class ValidateFuzzySearcher extends Searcher { + + private final Set<String> validAttributes = new HashSet<>(); + + public ValidateFuzzySearcher(AttributesConfig attributesConfig) { + for (AttributesConfig.Attribute a : attributesConfig.attribute()) { + if (a.datatype() == AttributesConfig.Attribute.Datatype.STRING) { + validAttributes.add(a.name()); + } + } + } + + @Override + public Result search(Query query, Execution execution) { + Optional<ErrorMessage> e = validate(query); + return e.isEmpty() ? execution.search(query) : new Result(query, e.get()); + } + + private Optional<ErrorMessage> validate(Query query) { + FuzzyVisitor visitor = new FuzzyVisitor(query.getRanking().getProperties(), validAttributes, query); + ToolBox.visit(visitor, query.getModel().getQueryTree().getRoot()); + return visitor.errorMessage; + } + + private static class FuzzyVisitor extends ToolBox.QueryVisitor { + + public Optional<ErrorMessage> errorMessage = Optional.empty(); + + private final Set<String> validAttributes; + private final Query query; + + public FuzzyVisitor(RankProperties rankProperties, Set<String> validAttributes, Query query) { + this.validAttributes = validAttributes; + this.query = query; + } + + @Override + public boolean visit(Item item) { + if (item instanceof FuzzyItem) { + String error = validate((FuzzyItem)item); + if (error != null) + errorMessage = Optional.of(ErrorMessage.createIllegalQuery(error)); + } + return true; + } + + /** Returns an error message if this is invalid, or null if it is valid */ + private String validate(FuzzyItem item) { + if (!validAttributes.contains(item.getIndexName())) { + return item + " field is not a string attribute"; + } + if (item.getPrefixLength() < 0) { + return item + " has invalid prefixLength " + item.getPrefixLength() + ": Must be >= 0"; + } + if (item.getMaxEditDistance() < 0) { + return item + " has invalid maxEditDistance " + item.getMaxEditDistance() + ": Must be >= 0"; + } + if (item.stringValue().isEmpty()) { + return item + " fuzzy query must be non-empty"; + } + return null; + } + + @Override + public void onExit() {} + + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/searchers/ValidateFuzzySearcherTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/ValidateFuzzySearcherTestCase.java new file mode 100644 index 00000000000..587b40dfd03 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/searchers/ValidateFuzzySearcherTestCase.java @@ -0,0 +1,133 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.searchers; + +import com.yahoo.config.subscription.ConfigGetter; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexModel; +import com.yahoo.prelude.SearchDefinition; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.query.QueryTree; +import com.yahoo.search.query.parser.Parsable; +import com.yahoo.search.query.parser.ParserEnvironment; +import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.yql.YqlParser; +import com.yahoo.vespa.config.search.AttributesConfig.Attribute; +import com.yahoo.vespa.config.search.AttributesConfig; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * @author alexeyche + */ +public class ValidateFuzzySearcherTestCase { + ValidateFuzzySearcher searcher; + + List<String> attributes; + + public ValidateFuzzySearcherTestCase() { + int i = 0; + attributes = new ArrayList<>(); + StringBuilder attributeConfig = new StringBuilder(); + for (Attribute.Datatype.Enum attr: Attribute.Datatype.Enum.values()) { + for (Attribute.Collectiontype.Enum ctype: Attribute.Collectiontype.Enum.values()) { + String attributeName = attr.name().toLowerCase() + "_" + ctype.name().toLowerCase(); + + attributeConfig.append("attribute[" + i + "].name "); + attributeConfig.append(attributeName); + attributeConfig.append("\n"); + + attributeConfig.append("attribute[" + i + "].datatype "); + attributeConfig.append(attr.name()); + attributeConfig.append("\n"); + + attributeConfig.append("attribute[" + i + "].collectiontype "); + attributeConfig.append(ctype.name()); + attributeConfig.append("\n"); + + i += 1; + attributes.add(attributeName); + } + } + + searcher = new ValidateFuzzySearcher(ConfigGetter.getConfig( + AttributesConfig.class, + "raw: " + + "attribute[" + attributes.size() + "]\n" + + attributeConfig)); + } + + private String makeQuery(String attribute, String query, int maxEditDistance, int prefixLength) { + return "select * from sources * where " + attribute + + " contains ({maxEditDistance:" + maxEditDistance + ", prefixLength:" + prefixLength +"}" + + "fuzzy(\"" + query + "\"))"; + } + + private String makeQuery(String attribute, String query) { + return makeQuery(attribute, query, 2, 0); + } + + + @Test + public void testQueriesToAllAttributes() { + final Set<String> validAttributes = Set.of("string_single", "string_array", "string_weightedset"); + + for (String attribute: attributes) { + String q = makeQuery(attribute, "fuzzy"); + Result r = doSearch(searcher, q); + if (validAttributes.contains(attribute)) { + assertNull(r.hits().getError()); + } else { + assertErrMsg("FUZZY(fuzzy,2,0) " + attribute + ":fuzzy field is not a string attribute", r); + } + } + } + + @Test + public void testInvalidEmptyStringQuery() { + String q = makeQuery("string_single", ""); + Result r = doSearch(searcher, q); + assertErrMsg("FUZZY(,2,0) string_single: fuzzy query must be non-empty", r); + } + + @Test + public void testInvalidQueryWrongMaxEditDistance() { + String q = makeQuery("string_single", "fuzzy", -1, 0); + Result r = doSearch(searcher, q); + assertErrMsg("FUZZY(fuzzy,-1,0) string_single:fuzzy has invalid maxEditDistance -1: Must be >= 0", r); + } + + @Test + public void testInvalidQueryWrongPrefixLength() { + String q = makeQuery("string_single", "fuzzy", 2, -1); + Result r = doSearch(searcher, q); + assertErrMsg("FUZZY(fuzzy,2,-1) string_single:fuzzy has invalid prefixLength -1: Must be >= 0", r); + } + + @Test + public void testInvalidQueryWrongAttributeName() { + String q = makeQuery("wrong_name", "fuzzy"); + Result r = doSearch(searcher, q); + assertErrMsg("FUZZY(fuzzy,2,0) wrong_name:fuzzy field is not a string attribute", r); + } + + private static void assertErrMsg(String message, Result r) { + assertEquals(ErrorMessage.createIllegalQuery(message), r.hits().getError()); + } + + private static Result doSearch(ValidateFuzzySearcher searcher, String yqlQuery) { + QueryTree queryTree = new YqlParser(new ParserEnvironment()).parse(new Parsable().setQuery(yqlQuery)); + Query query = new Query(); + query.getModel().getQueryTree().setRoot(queryTree.getRoot()); + SearchDefinition searchDefinition = new SearchDefinition("document"); + IndexFacts indexFacts = new IndexFacts(new IndexModel(searchDefinition)); + return new Execution(searcher, Execution.Context.createContextStub(indexFacts)).search(query); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java index 5f567e8b84a..0415b33b29d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.athenz.api.AthenzRole; import com.yahoo.vespa.athenz.api.AthenzRoleInformation; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.OAuthCredentials; +import com.yahoo.vespa.athenz.client.zms.QuotaUsage; import com.yahoo.vespa.athenz.client.zms.RoleAction; import com.yahoo.vespa.athenz.client.zms.ZmsClient; import com.yahoo.vespa.athenz.client.zms.ZmsClientException; @@ -262,6 +263,11 @@ public class ZmsClientMock implements ZmsClient { } @Override + public QuotaUsage getQuotaUsage() { + return new QuotaUsage(0.1, 0.2, 0.3, 0.4, 0.5); + } + + @Override public void close() {} private static AthenzDomain getTenantDomain(AthenzResourceName resource) { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java index 4505fe3ceb5..9e14c4ae8dd 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java @@ -3,214 +3,40 @@ package com.yahoo.vespa.hosted.controller.api.integration.deployment; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; -import java.util.Arrays; import java.util.Comparator; import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; import static ai.vespa.validation.Validation.require; +import static com.yahoo.config.provision.Environment.dev; +import static com.yahoo.config.provision.Environment.perf; import static com.yahoo.config.provision.Environment.prod; import static com.yahoo.config.provision.Environment.staging; import static com.yahoo.config.provision.Environment.test; -import static com.yahoo.config.provision.SystemName.Public; -import static com.yahoo.config.provision.SystemName.PublicCd; -import static com.yahoo.config.provision.SystemName.cd; -import static com.yahoo.config.provision.SystemName.main; - -/** Job types that exist in the build system */ +import static java.util.Comparator.naturalOrder; +import static java.util.stream.Collectors.toUnmodifiableList; + +/** + * Specification for a deployment and/or test job to run: what zone, and whether it is a production test. + * + * @author jonmv + */ public final class JobType implements Comparable<JobType> { -// | enum name ------------| job name ------------------| Zone in main system ---------------------------------------| Zone in CD system ------------------------------------------- - public static final JobType systemTest = of("system-test", - Map.of(main , ZoneId.from("test", "us-east-1"), - cd , ZoneId.from("test", "cd-us-west-1"), - PublicCd, ZoneId.from("test", "aws-us-east-1c"), - Public , ZoneId.from("test", "aws-us-east-1c"))); - - public static final JobType stagingTest = of("staging-test", - Map.of(main , ZoneId.from("staging", "us-east-3"), - cd , ZoneId.from("staging", "cd-us-west-1"), - PublicCd, ZoneId.from("staging", "aws-us-east-1c"), - Public , ZoneId.from("staging", "aws-us-east-1c"))); - - public static final JobType productionUsEast3 = prod("us-east-3"); - - public static final JobType testUsEast3 = test("us-east-3"); - - public static final JobType productionUsWest1 = prod("us-west-1"); - - public static final JobType testUsWest1 = test("us-west-1"); - - public static final JobType productionUsCentral1 = prod("us-central-1"); - - public static final JobType testUsCentral1 = test("us-central-1"); - - public static final JobType productionApNortheast1 = prod("ap-northeast-1"); - - public static final JobType testApNortheast1 = test("ap-northeast-1"); - - public static final JobType productionApNortheast2 = prod("ap-northeast-2"); - - public static final JobType testApNortheast2 = test("ap-northeast-2"); - - public static final JobType productionApSoutheast1 = prod("ap-southeast-1"); - - public static final JobType testApSoutheast1 = test("ap-southeast-1"); - - public static final JobType productionEuWest1 = prod("eu-west-1"); - - public static final JobType testEuWest1 = test("eu-west-1"); - - public static final JobType productionAwsUsEast1a= prod("aws-us-east-1a"); - - public static final JobType testAwsUsEast1a = test("aws-us-east-1a"); - - public static final JobType productionAwsUsEast1c= prod("aws-us-east-1c"); - - public static final JobType testAwsUsEast1c = test("aws-us-east-1c"); - - public static final JobType productionAwsApNortheast1a= prod("aws-ap-northeast-1a"); - - public static final JobType testAwsApNortheast1a = test("aws-ap-northeast-1a"); - - public static final JobType productionAwsEuWest1a= prod("aws-eu-west-1a"); - - public static final JobType testAwsEuWest1a = test("aws-eu-west-1a"); - - public static final JobType productionAwsUsWest2a= prod("aws-us-west-2a"); - - public static final JobType testAwsUsWest2a = test("aws-us-west-2a"); - - public static final JobType productionAwsUsEast1b= prod("aws-us-east-1b"); - - public static final JobType testAwsUsEast1b = test("aws-us-east-1b"); - - public static final JobType devUsEast1 = dev("us-east-1"); - - public static final JobType devAwsUsEast2a = dev("aws-us-east-2a"); - - public static final JobType productionCdAwsUsEast1a = prod("cd-aws-us-east-1a"); - - public static final JobType testCdAwsUsEast1a = test("cd-aws-us-east-1a"); - - public static final JobType productionCdUsCentral1 = prod("cd-us-central-1"); - - public static final JobType testCdUsCentral1 = test("cd-us-central-1"); - - public static final JobType productionCdUsCentral2 = prod("cd-us-central-2"); - - public static final JobType testCdUsCentral2 = test("cd-us-central-2"); - - public static final JobType productionCdUsEast1= prod("cd-us-east-1"); - - public static final JobType testCdUsEast1 = test("cd-us-east-1"); - - public static final JobType productionCdUsWest1= prod("cd-us-west-1"); - - public static final JobType testCdUsWest1 = test("cd-us-west-1"); - - public static final JobType devCdUsCentral1 = dev("cd-us-central-1"); - - public static final JobType devCdUsWest1 = dev("cd-us-west-1"); - - public static final JobType devAwsUsEast1c = dev("aws-us-east-1c"); - - public static final JobType perfAwsUsEast1c = perf("aws-us-east-1c"); - - public static final JobType perfUsEast3 = perf("us-east-3"); - - private static final JobType[] values = new JobType[] { - systemTest, - stagingTest, - productionUsEast3, - testUsEast3, - productionUsWest1, - testUsWest1, - productionUsCentral1, - testUsCentral1, - productionApNortheast1, - testApNortheast1, - productionApNortheast2, - testApNortheast2, - productionApSoutheast1, - testApSoutheast1, - productionEuWest1, - testEuWest1, - productionAwsUsEast1a, - testAwsUsEast1a, - productionAwsUsEast1c, - testAwsUsEast1c, - productionAwsApNortheast1a, - testAwsApNortheast1a, - productionAwsEuWest1a, - testAwsEuWest1a, - productionAwsUsWest2a, - testAwsUsWest2a, - productionAwsUsEast1b, - testAwsUsEast1b, - devUsEast1, - devAwsUsEast2a, - productionCdAwsUsEast1a, - testCdAwsUsEast1a, - productionCdUsCentral1, - testCdUsCentral1, - productionCdUsCentral2, - testCdUsCentral2, - productionCdUsEast1, - testCdUsEast1, - productionCdUsWest1, - testCdUsWest1, - devCdUsCentral1, - devCdUsWest1, - devAwsUsEast1c, - perfAwsUsEast1c, - perfUsEast3 - }; private final String jobName; - final Map<SystemName, ZoneId> zones; + private final ZoneId zone; private final boolean isProductionTest; - private JobType(String jobName, Map<SystemName, ZoneId> zones, boolean isProductionTest) { - if (zones.values().stream().map(ZoneId::environment).distinct().count() > 1) - throw new IllegalArgumentException("All zones of a job must be in the same environment"); - + private JobType(String jobName, ZoneId zone, boolean isProductionTest) { this.jobName = jobName; - this.zones = zones; + this.zone = zone; this.isProductionTest = isProductionTest; } - private static JobType of(String jobName, Map<SystemName, ZoneId> zones, boolean isProductionTest) { - return new JobType(jobName, zones, isProductionTest); - } - - private static JobType of(String jobName, Map<SystemName, ZoneId> zones) { - return of(jobName, zones, false); - } - - public String jobName() { return jobName; } - - /** Returns the zone for this job in the given system. */ - public ZoneId zone() { - throw new UnsupportedOperationException(); - } - - /** Returns the zone for this job in the given system, or throws if this job does not have a zone */ - public ZoneId zone(SystemName system) { - if ( ! zones.containsKey(system)) - throw new IllegalArgumentException(this + " does not have any zones in " + system); - - return zones.get(system); - } - /** A system test in a test zone, or throws if no test zones are present.. */ public static JobType systemTest(ZoneRegistry zones) { return testIn(test, zones); @@ -227,25 +53,46 @@ public final class JobType implements Comparable<JobType> { } /** A deployment to the given dev region. */ + public static JobType dev(RegionName region) { + return deploymentTo(ZoneId.from(dev, region)); + } + + /** A deployment to the given dev region. */ public static JobType dev(String region) { return deploymentTo(ZoneId.from("dev", region)); } /** A deployment to the given perf region. */ + public static JobType perf(RegionName region) { + return deploymentTo(ZoneId.from(perf, region)); + } + + /** A deployment to the given perf region. */ public static JobType perf(String region) { return deploymentTo(ZoneId.from("perf", region)); } /** A deployment to the given prod region. */ + public static JobType prod(RegionName region) { + return deploymentTo(ZoneId.from(prod, region)); + } + + /** A deployment to the given prod region. */ public static JobType prod(String region) { return deploymentTo(ZoneId.from("prod", region)); } /** A production test in the given region. */ + public static JobType test(RegionName region) { + return productionTestOf(ZoneId.from(prod, region)); + } + + /** A production test in the given region. */ public static JobType test(String region) { return productionTestOf(ZoneId.from("prod", region)); } + /** A deployment to the given zone; this may be a zone in the {@code test} or {@code staging} environments. */ public static JobType deploymentTo(ZoneId zone) { String name; switch (zone.environment()) { @@ -254,18 +101,16 @@ public final class JobType implements Comparable<JobType> { case staging: name = "staging-test"; break; default: name = zone.environment().value() + "-" + zone.region().value(); } - return of(name, dummy(zone), false); + return new JobType(name, zone, false); } + /** A production test in the given production zone. */ public static JobType productionTestOf(ZoneId zone) { - return of("test-" + require(zone.environment() == prod, zone, "must be prod zone").region().value(), dummy(zone), true); + String name = "test-" + require(zone.environment() == prod, zone, "must be prod zone").region().value(); + return new JobType(name, zone, true); } - private static Map<SystemName, ZoneId> dummy(ZoneId zone) { - return Stream.of(SystemName.values()).collect(Collectors.toMap(Function.identity(), __ -> zone)); - } - - // TODO jonmv: use for serialisation + /** Creates a new job type from serialized zone data, and whether it is a production test; the inverse of {@link #serialized()} */ public static JobType ofSerialized(String raw) { String[] parts = raw.split("\\."); if (parts.length == 2) return deploymentTo(ZoneId.from(parts[0], parts[1])); @@ -273,20 +118,41 @@ public final class JobType implements Comparable<JobType> { throw new IllegalArgumentException("illegal serialized job type '" + raw + "'"); } - public String serialized(SystemName system) { - ZoneId zone = zone(system); - return zone.environment().value() + "." + zone.region().value() + (isProductionTest ? ".test" : ""); + /** Creates a new job type from a job name, and a zone registry for looking up zones for the special system and staging test types. */ + public static JobType fromJobName(String jobName, ZoneRegistry zones) { + String[] parts = jobName.split("-", 2); + if (parts.length != 2) throw new IllegalArgumentException("job names must be 'system-test', 'staging-test', or environment and region parts, separated by '-', but got: " + jobName); + switch (parts[0]) { + case "system": return systemTest(zones); + case "staging": return stagingTest(zones); + case "production": return prod(parts[1]); + case "test": return test(parts[1]); + case "dev": return dev(parts[1]); + case "perf": return perf(parts[1]); + default: throw new IllegalArgumentException("job names must begin with one of: system, staging, production, test, dev, perf; but got: " + jobName); + } } public static List<JobType> allIn(ZoneRegistry zones) { return zones.zones().controllerUpgraded().zones().stream() .flatMap(zone -> zone.getEnvironment().isProduction() ? Stream.of(deploymentTo(zone.getId()), productionTestOf(zone.getId())) : Stream.of(deploymentTo(zone.getId()))) - .collect(Collectors.toUnmodifiableList()); + .sorted(naturalOrder()) + .collect(toUnmodifiableList()); } - static JobType[] values() { - return Arrays.copyOf(values, values.length); + /** A serialized form of this: {@code <environment>.<region>[.test]}; the inverse of {@link #ofSerialized(String)} */ + public String serialized() { + return zone.environment().value() + "." + zone.region().value() + (isProductionTest ? ".test" : ""); + } + + public String jobName() { + return jobName; + } + + /** Returns the zone for this job. */ + public ZoneId zone() { + return zone; } public boolean isSystemTest() { @@ -298,81 +164,31 @@ public final class JobType implements Comparable<JobType> { } /** Returns whether this is a production job */ - public boolean isProduction() { return environment() == prod; } + public boolean isProduction() { + return environment() == prod; + } /** Returns whether this job runs tests */ - public boolean isTest() { return isProductionTest || environment().isTest(); } + public boolean isTest() { + return isProductionTest || environment().isTest(); + } /** Returns whether this job deploys to a zone */ - public boolean isDeployment() { return ! (isProduction() && isProductionTest); } + public boolean isDeployment() { + return ! isProductionTest; + } /** Returns the environment of this job type */ public Environment environment() { - return zones.values().iterator().next().environment(); - } - - // TODO jonmv: require zones - public static Optional<JobType> fromOptionalJobName(String jobName) { - if (jobName.contains(".")) return Optional.of(ofSerialized(jobName)); // TODO jonmv: remove - return Stream.of(values) - .filter(jobType -> jobType.jobName.equals(jobName)) - .findAny(); - } - - // TODO jonmv: require zones - public static JobType fromJobName(String jobName) { - return fromOptionalJobName(jobName) - .orElseThrow(() -> new IllegalArgumentException("Unknown job name '" + jobName + "'")); - } - - // TODO jonmv: require zones - public static JobType fromJobName(String jobName, ZoneRegistry zones) { - String[] parts = jobName.split("-", 2); - if (parts.length != 2) throw new IllegalArgumentException("job names must be 'system-test', 'staging-test', or environment and region parts, separated by '-', but got: " + jobName); - switch (parts[0]) { - case "system": return systemTest(zones); - case "staging": return stagingTest(zones); - case "production": return prod(parts[1]); - case "test": return test(parts[1]); - case "dev": return dev(parts[1]); - case "perf": return perf(parts[1]); - default: throw new IllegalArgumentException("job names must begin with one of: system, staging, production, test, dev, perf; but got: " + jobName); - } - } - - /** Returns the job type for the given zone */ - public static Optional<JobType> from(SystemName system, ZoneId zone, boolean isTest) { - return Stream.of(values) - .filter(job -> zone.equals(job.zones.get(system)) && job.isTest() == isTest) - .findAny(); + return zone.environment(); } - /** Returns the job type for the given zone */ - public static Optional<JobType> from(SystemName system, ZoneId zone) { - return from(system, zone, zone.environment().isTest()); - } - - /** Returns the production test job type for the given environment and region or null if none */ - public static Optional<JobType> testFrom(SystemName system, RegionName region) { - return from(system, ZoneId.from(prod, region), true); - } - - /** Returns the job job type for the given environment and region or null if none */ - public static Optional<JobType> from(SystemName system, Environment environment, RegionName region) { - switch (environment) { - case test: return Optional.of(systemTest); - case staging: return Optional.of(stagingTest); - } - return from(system, ZoneId.from(environment, region)); - } - - - private static final Comparator<JobType> comparator = Comparator.comparing(JobType::environment) - .thenComparing(JobType::isDeployment) - .thenComparing(JobType::jobName); @Override public int compareTo(JobType other) { - return comparator.compare(this, other); + int result; + if (0 != (result = environment().compareTo(other.environment()))) return -result; + if (0 != (result = zone.region().compareTo(other.zone.region()))) return result; + return Boolean.compare(isProductionTest, other.isProductionTest); } @Override diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentFailureMails.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentFailureMails.java index b35bf7c68c2..561fa21fde6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentFailureMails.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentFailureMails.java @@ -75,8 +75,7 @@ public class DeploymentFailureMails { return "System test"; if (type.isStagingTest()) return "Staging test"; - return (type.isDeployment() ? "Deployment to " : "Verification test of ") + - type.zone(registry.system()).region(); + return (type.isDeployment() ? "Deployment to " : "Verification test of ") + type.zone().region(); } } diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobTypeTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobTypeTest.java index 64f4d6150dd..6ff52bd5f03 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobTypeTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobTypeTest.java @@ -1,10 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.deployment; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -18,20 +15,8 @@ public class JobTypeTest { @Test public void test() { - for (JobType type : JobType.values()) { - if (type.isProduction()) { - boolean match = false; - for (JobType other : JobType.values()) - match |= type != other - && type.isTest() == other.isDeployment() - && type.zones.equals(other.zones); - - assertTrue(type + " should have matching job", match); - } - } - - assertEquals(JobType.testUsEast3, JobType.ofSerialized("prod.us-east-3.test")); - assertEquals(JobType.devAwsUsEast1c, JobType.ofSerialized("dev.aws-us-east-1c")); + assertEquals(JobType.test("us-east-3"), JobType.ofSerialized("prod.us-east-3.test")); + assertEquals(JobType.dev("aws-us-east-1c"), JobType.ofSerialized("dev.aws-us-east-1c")); assertEquals(JobType.fromJobName("production-my-zone", null), JobType.prod("my-zone")); assertEquals(JobType.fromJobName("test-my-zone", null), JobType.test("my-zone")); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 0f7cbcee4ab..613e3413ae8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -296,7 +296,7 @@ public class ApplicationController { /** Reads the oldest installed platform for the given application and zone from the node repo of that zone. */ private Optional<Version> oldestInstalledPlatform(JobId job) { - return configServer.nodeRepository().list(job.type().zone(controller.system()), + return configServer.nodeRepository().list(job.type().zone(), NodeFilter.all() .applications(job.application()) .states(active, reserved)) @@ -454,7 +454,7 @@ public class ApplicationController { throw new IllegalArgumentException("'" + job.application() + "' is a tester application!"); TenantAndApplicationId applicationId = TenantAndApplicationId.from(job.application()); - ZoneId zone = job.type().zone(controller.system()); + ZoneId zone = job.type().zone(); DeploymentId deployment = new DeploymentId(job.application(), zone); try (Mutex deploymentLock = lockForDeployment(job.application(), zone)) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java index 9ab5096ec8f..950eaea904a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java @@ -62,7 +62,7 @@ public class ApplicationPackageValidator { /** Verify that each of the production zones listed in the deployment spec exist in this system */ private void validateSteps(DeploymentSpec deploymentSpec) { for (var spec : deploymentSpec.instances()) { - new DeploymentSteps(spec, controller::system).jobs(); + new DeploymentSteps(spec, controller.zoneRegistry()).jobs(); spec.zones().stream() .filter(zone -> zone.environment() == Environment.prod) .forEach(zone -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index e13175806bf..cc7031bab5a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -12,7 +12,6 @@ import com.yahoo.config.application.api.DeploymentSpec.UpgradeRollout; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Instance; @@ -20,6 +19,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationV import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -45,8 +45,6 @@ import static com.yahoo.config.application.api.DeploymentSpec.RevisionTarget.nex import static com.yahoo.config.provision.Environment.prod; import static com.yahoo.config.provision.Environment.staging; import static com.yahoo.config.provision.Environment.test; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; import static java.util.Comparator.comparing; import static java.util.Comparator.naturalOrder; import static java.util.Objects.requireNonNull; @@ -69,17 +67,19 @@ public class DeploymentStatus { private final Application application; private final JobList allJobs; - private final SystemName system; + private final JobType systemTest; + private final JobType stagingTest; private final Version systemVersion; private final Function<InstanceName, VersionCompatibility> versionCompatibility; private final Instant now; private final Map<JobId, StepStatus> jobSteps; private final List<StepStatus> allSteps; - public DeploymentStatus(Application application, Function<JobId, JobStatus> allJobs, SystemName system, + public DeploymentStatus(Application application, Function<JobId, JobStatus> allJobs, ZoneRegistry zones, Version systemVersion, Function<InstanceName, VersionCompatibility> versionCompatibility, Instant now) { this.application = requireNonNull(application); - this.system = requireNonNull(system); + this.systemTest = JobType.systemTest(zones); + this.stagingTest = JobType.stagingTest(zones); this.systemVersion = requireNonNull(systemVersion); this.versionCompatibility = versionCompatibility; this.now = requireNonNull(now); @@ -244,7 +244,7 @@ public class DeploymentStatus { public Optional<Deployment> deploymentFor(JobId job) { return Optional.ofNullable(application.require(job.application().instance()) - .deployments().get(job.type().zone(system))); + .deployments().get(job.type().zone())); } /** @@ -388,7 +388,7 @@ public class DeploymentStatus { // For a dual change, where both targets remain, we determine what to run by looking at when the two parts became ready: // for deployments, we look at dependencies; for production tests, this may be overridden by what is already deployed. - JobId deployment = new JobId(job.application(), JobType.from(system, job.type().zone(system)).get()); + JobId deployment = new JobId(job.application(), JobType.deploymentTo(job.type().zone())); UpgradeRollout rollout = application.deploymentSpec().requireInstance(job.application().instance()).upgradeRollout(); if (job.type().isTest()) { Optional<Instant> platformDeployedAt = jobSteps.get(deployment).completedAt(change.withoutApplication(), Optional.of(deployment)); @@ -556,23 +556,20 @@ public class DeploymentStatus { JobId jobId; StepStatus stepStatus; if (step.concerns(test) || step.concerns(staging)) { - jobType = JobType.from(system, ((DeclaredZone) step).environment(), null) - .orElseThrow(() -> new IllegalStateException(application + " specifies " + step + ", but this has no job in " + system)); + jobType = step.concerns(test) ? systemTest : stagingTest; jobId = new JobId(application.id().instance(instance), jobType); stepStatus = JobStepStatus.ofTestDeployment((DeclaredZone) step, List.of(), this, jobs.apply(jobId), true); previous = new ArrayList<>(previous); previous.add(stepStatus); } else if (step.isTest()) { - jobType = JobType.testFrom(system, ((DeclaredTest) step).region()) - .orElseThrow(() -> new IllegalStateException(application + " specifies " + step + ", but this has no job in " + system)); + jobType = JobType.test(((DeclaredTest) step).region()); jobId = new JobId(application.id().instance(instance), jobType); stepStatus = JobStepStatus.ofProductionTest((DeclaredTest) step, previous, this, jobs.apply(jobId)); previous = List.of(stepStatus); } else if (step.concerns(prod)) { - jobType = JobType.from(system, ((DeclaredZone) step).environment(), ((DeclaredZone) step).region().get()) - .orElseThrow(() -> new IllegalStateException(application + " specifies " + step + ", but this has no job in " + system)); + jobType = JobType.prod(((DeclaredZone) step).region().get()); jobId = new JobId(application.id().instance(instance), jobType); stepStatus = JobStepStatus.ofProductionDeployment((DeclaredZone) step, previous, this, jobs.apply(jobId)); previous = List.of(stepStatus); @@ -872,7 +869,7 @@ public class DeploymentStatus { private static JobStepStatus ofProductionTest(DeclaredTest step, List<StepStatus> dependencies, DeploymentStatus status, JobStatus job) { - JobId prodId = new JobId(job.id().application(), JobType.from(status.system, job.id().type().zone(status.system)).get()); + JobId prodId = new JobId(job.id().application(), JobType.deploymentTo(job.id().type().zone())); return new JobStepStatus(StepType.test, step, dependencies, job, status) { @Override Optional<Instant> readyAt(Change change, Optional<JobId> dependent) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java index 7ab895654f3..44079a90097 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentSteps.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.application.Deployment; import java.util.Collection; @@ -29,16 +30,16 @@ import static java.util.stream.Collectors.collectingAndThen; public class DeploymentSteps { private final DeploymentInstanceSpec spec; - private final Supplier<SystemName> system; + private final ZoneRegistry zones; - public DeploymentSteps(DeploymentInstanceSpec spec, Supplier<SystemName> system) { + public DeploymentSteps(DeploymentInstanceSpec spec, ZoneRegistry zones) { this.spec = Objects.requireNonNull(spec, "spec cannot be null"); - this.system = Objects.requireNonNull(system, "system cannot be null"); + this.zones = Objects.requireNonNull(zones, "system cannot be null"); } /** Returns jobs for this, in the order they should run */ public List<JobType> jobs() { - return Stream.concat(production().isEmpty() ? Stream.of() : Stream.of(JobType.systemTest, JobType.stagingTest), + return Stream.concat(production().isEmpty() ? Stream.of() : Stream.of(JobType.systemTest(zones), JobType.stagingTest(zones)), spec.steps().stream().flatMap(step -> toJobs(step).stream())) .distinct() .collect(Collectors.toUnmodifiableList()); @@ -67,7 +68,6 @@ public class DeploymentSteps { public List<JobType> toJobs(DeploymentSpec.Step step) { return step.zones().stream() .map(this::toJob) - .flatMap(Optional::stream) .collect(Collectors.toUnmodifiableList()); } @@ -93,8 +93,13 @@ public class DeploymentSteps { } /** Resolve job from deployment zone */ - private Optional<JobType> toJob(DeploymentSpec.DeclaredZone zone) { - return JobType.from(system.get(), zone.environment(), zone.region().orElse(null)); + private JobType toJob(DeploymentSpec.DeclaredZone zone) { + switch (zone.environment()) { + case prod: return JobType.prod(zone.region().get()); + case test: return JobType.systemTest(zones); + case staging: return JobType.stagingTest(zones); + default: throw new IllegalArgumentException("region must be one with automated deployments, but got: " + zone.environment()); + } } /** Resolve jobs from steps */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index c98b3b76292..be07a2b0cb1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -74,10 +74,7 @@ public class DeploymentTrigger { } public DeploymentSteps steps(DeploymentInstanceSpec spec) { - return new DeploymentSteps(spec, controller::system); - } - - public void notifyOfSubmission(TenantAndApplicationId id, ApplicationVersion version, long projectId) { + return new DeploymentSteps(spec, controller.zoneRegistry()); } /** @@ -268,8 +265,7 @@ public class DeploymentTrigger { /** Retrigger job. If the job is already running, it will be canceled, and retrigger enqueued. */ public Optional<JobId> reTriggerOrAddToQueue(DeploymentId deployment, String reason) { - JobType jobType = JobType.from(controller.system(), deployment.zoneId()) - .orElseThrow(() -> new IllegalArgumentException(Text.format("No job to trigger for (system/zone): %s/%s", controller.system().value(), deployment.zoneId().value()))); + JobType jobType = JobType.deploymentTo(deployment.zoneId()); Optional<Run> existingRun = controller.jobController().active(deployment.applicationId()).stream() .filter(run -> run.id().type().equals(jobType)) .findFirst(); @@ -389,7 +385,7 @@ public class DeploymentTrigger { /** Returns whether the application is healthy in all other production zones. */ private boolean isUnhealthyInAnotherZone(Application application, JobId job) { for (Deployment deployment : application.require(job.application().instance()).productionDeployments().values()) { - if ( ! deployment.zone().equals(job.type().zone(controller.system())) + if ( ! deployment.zone().equals(job.type().zone()) && ! controller.applications().isHealthy(new DeploymentId(job.application(), deployment.zone()))) return true; } @@ -418,9 +414,7 @@ public class DeploymentTrigger { boolean blocked = status.jobs().get(job).get().isRunning(); if ( ! job.type().isTest()) { - Optional<JobStatus> productionTest = JobType.testFrom(controller.system(), job.type().zone(controller.system()).region()) - .map(type -> new JobId(job.application(), type)) - .flatMap(status.jobs()::get); + Optional<JobStatus> productionTest = status.jobs().get(new JobId(job.application(), JobType.productionTestOf(job.type().zone()))); if (productionTest.isPresent()) { abortIfOutdated(status, jobs, productionTest.get().id()); // Production deployments are also blocked by their declared tests, if the next versions to run diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 28d9439b457..9e551c7ce78 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -217,7 +217,7 @@ public class InternalStepRunner implements StepRunner { logger.log("Deploying the tester container on platform " + platform + " ..."); return deploy(() -> controller.applications().deployTester(id.tester(), testerPackage(id), - id.type().zone(controller.system()), + id.type().zone(), platform), controller.jobController().run(id).get() .stepInfo(deployTester).get() @@ -314,19 +314,19 @@ public class InternalStepRunner implements StepRunner { Version platform = setTheStage ? versions.sourcePlatform().orElse(versions.targetPlatform()) : versions.targetPlatform(); Run run = controller.jobController().run(id).get(); - Optional<ServiceConvergence> services = controller.serviceRegistry().configServer().serviceConvergence(new DeploymentId(id.application(), id.type().zone(controller.system())), + Optional<ServiceConvergence> services = controller.serviceRegistry().configServer().serviceConvergence(new DeploymentId(id.application(), id.type().zone()), Optional.of(platform)); if (services.isEmpty()) { logger.log("Config status not currently available -- will retry."); return Optional.empty(); } - List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(controller.system()), + List<Node> nodes = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(), NodeFilter.all() .applications(id.application()) .states(active)); Set<HostName> parentHostnames = nodes.stream().map(node -> node.parentHostname().get()).collect(toSet()); - List<Node> parents = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(controller.system()), + List<Node> parents = controller.serviceRegistry().configServer().nodeRepository().list(id.type().zone(), NodeFilter.all() .hostnames(parentHostnames)); boolean firstTick = run.convergenceSummary().isEmpty(); @@ -357,8 +357,8 @@ public class InternalStepRunner implements StepRunner { } if (summary.converged()) { controller.jobController().locked(id, lockedRun -> lockedRun.withSummary(null)); - if (endpointsAvailable(id.application(), id.type().zone(controller.system()), logger)) { - if (containersAreUp(id.application(), id.type().zone(controller.system()), logger)) { + if (endpointsAvailable(id.application(), id.type().zone(), logger)) { + if (containersAreUp(id.application(), id.type().zone(), logger)) { logger.log("Installation succeeded!"); return Optional.of(running); } @@ -440,7 +440,7 @@ public class InternalStepRunner implements StepRunner { private Optional<RunStatus> installTester(RunId id, DualLogger logger) { Run run = controller.jobController().run(id).get(); Version platform = testerPlatformVersion(id); - ZoneId zone = id.type().zone(controller.system()); + ZoneId zone = id.type().zone(); ApplicationId testerId = id.tester().id(); Optional<ServiceConvergence> services = controller.serviceRegistry().configServer().serviceConvergence(new DeploymentId(testerId, zone), @@ -609,7 +609,7 @@ public class InternalStepRunner implements StepRunner { .productionDeployments().keySet().stream() .map(zone -> new DeploymentId(id.application(), zone)) .collect(Collectors.toSet()); - ZoneId zoneId = id.type().zone(controller.system()); + ZoneId zoneId = id.type().zone(); deployments.add(new DeploymentId(id.application(), zoneId)); logger.log("Attempting to find endpoints ..."); @@ -722,8 +722,8 @@ public class InternalStepRunner implements StepRunner { private Optional<RunStatus> deactivateReal(RunId id, DualLogger logger) { try { - logger.log("Deactivating deployment of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); - controller.applications().deactivate(id.application(), id.type().zone(controller.system())); + logger.log("Deactivating deployment of " + id.application() + " in " + id.type().zone() + " ..."); + controller.applications().deactivate(id.application(), id.type().zone()); return Optional.of(running); } catch (RuntimeException e) { @@ -737,7 +737,7 @@ public class InternalStepRunner implements StepRunner { private Optional<RunStatus> deactivateTester(RunId id, DualLogger logger) { try { - logger.log("Deactivating tester of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); + logger.log("Deactivating tester of " + id.application() + " in " + id.type().zone() + " ..."); controller.jobController().deactivateTester(id.tester(), id.type()); return Optional.of(running); } @@ -870,7 +870,7 @@ public class InternalStepRunner implements StepRunner { /** Returns the deployment of the real application in the zone of the given job, if it exists. */ private Optional<Deployment> deployment(ApplicationId id, JobType type) { - return Optional.ofNullable(application(id).deployments().get(type.zone(controller.system()))); + return Optional.ofNullable(application(id).deployments().get(type.zone())); } /** Returns the real application with the given id. */ @@ -908,7 +908,7 @@ public class InternalStepRunner implements StepRunner { RevisionId revision = controller.jobController().run(id).get().versions().targetRevision(); DeploymentSpec spec = controller.applications().requireApplication(TenantAndApplicationId.from(id.application())).deploymentSpec(); - ZoneId zone = id.type().zone(controller.system()); + ZoneId zone = id.type().zone(); boolean useTesterCertificate = useTesterCertificate(id); byte[] servicesXml = servicesXml( ! controller.system().isPublic(), @@ -952,7 +952,7 @@ public class InternalStepRunner implements StepRunner { } private DeploymentId getTesterDeploymentId(RunId runId) { - ZoneId zoneId = runId.type().zone(controller.system()); + ZoneId zoneId = runId.type().zone(); return new DeploymentId(runId.tester().id(), zoneId); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 6a6ed6e3b5d..69d9ba504a5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -30,7 +30,6 @@ import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageDiff; import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import java.security.cert.X509Certificate; @@ -51,6 +50,7 @@ import java.util.TreeMap; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.function.UnaryOperator; import java.util.logging.Level; import java.util.logging.Logger; @@ -169,7 +169,7 @@ public class JobController { if ( ! run.hasStep(copyVespaLogs)) return run; - ZoneId zone = id.type().zone(controller.system()); + ZoneId zone = id.type().zone(); Optional<Deployment> deployment = Optional.ofNullable(controller.applications().requireInstance(id.application()) .deployments().get(zone)); if (deployment.isEmpty() || deployment.get().at().isBefore(run.start())) @@ -197,7 +197,7 @@ public class JobController { if (step.isEmpty()) return run; - List<LogEntry> entries = cloud.getLog(new DeploymentId(id.tester().id(), id.type().zone(controller.system())), + List<LogEntry> entries = cloud.getLog(new DeploymentId(id.tester().id(), id.type().zone()), run.lastTestLogEntry()); if (entries.isEmpty()) return run; @@ -209,7 +209,7 @@ public class JobController { public void updateTestReport(RunId id) { locked(id, run -> { - Optional<TestReport> report = cloud.getTestReport(new DeploymentId(id.tester().id(), id.type().zone(controller.system()))); + Optional<TestReport> report = cloud.getTestReport(new DeploymentId(id.tester().id(), id.type().zone())); if (report.isEmpty()) { return run; } @@ -257,10 +257,9 @@ public class JobController { /** Returns when given deployment last started deploying, falling back to time of deployment if it cannot be determined from job runs */ public Instant lastDeploymentStart(ApplicationId instanceId, Deployment deployment) { - return jobStarts(new JobId(instanceId, JobType.from(controller.system(), - deployment.zone()).get())).stream() - .findFirst() - .orElseGet(deployment::at); + return jobStarts(new JobId(instanceId, JobType.deploymentTo(deployment.zone()))).stream() + .findFirst() + .orElseGet(deployment::at); } /** Returns an immutable map of all known runs for the given application and job type. */ @@ -353,7 +352,7 @@ public class JobController { private DeploymentStatus deploymentStatus(Application application, Version systemVersion) { return new DeploymentStatus(application, this::jobStatus, - controller.system(), + controller.zoneRegistry(), systemVersion, instance -> controller.applications().versionCompatibility(application.id().instance(instance)), controller.clock().instant()); @@ -526,7 +525,7 @@ public class JobController { controller.applications().store(application.withRevisions(revisions -> revisions.withoutOlderThan(oldestRevision))); } else { - controller.applications().applicationStore().pruneDevDiffs(new DeploymentId(run.id().application(), run.id().job().type().zone(controller.system())), oldestRevision.number()); + controller.applications().applicationStore().pruneDevDiffs(new DeploymentId(run.id().application(), run.id().job().type().zone()), oldestRevision.number()); controller.applications().store(application.withRevisions(revisions -> revisions.withoutOlderThan(oldestRevision, run.id().job()))); } }); @@ -566,6 +565,9 @@ public class JobController { /** Stores the given package and starts a deployment of it, after aborting any such ongoing deployment.*/ public void deploy(ApplicationId id, JobType type, Optional<Version> platform, ApplicationPackage applicationPackage, boolean dryRun) { + if ( ! controller.zoneRegistry().hasZone(type.zone())) + throw new IllegalArgumentException(type.zone() + " is not present in this system"); + controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { if ( ! application.get().instances().containsKey(id.instance())) application = controller.applications().withNewInstance(application, id); @@ -573,7 +575,7 @@ public class JobController { controller.applications().store(application); }); - DeploymentId deploymentId = new DeploymentId(id, type.zone(controller.system())); + DeploymentId deploymentId = new DeploymentId(id, type.zone()); Optional<Run> lastRun = last(id, type); lastRun.filter(run -> ! run.hasEnded()).ifPresent(run -> abortAndWait(run.id())); @@ -585,7 +587,7 @@ public class JobController { controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { controller.applications().applicationStore().putDev(deploymentId, version.id(), applicationPackage.zippedContent(), diff); - Version targetPlatform = platform.orElseGet(() -> findTargetPlatform(applicationPackage, lastRun, id)); + Version targetPlatform = platform.orElseGet(() -> findTargetPlatform(applicationPackage, deploymentId, application.get().get(id.instance()))); controller.applications().store(application.withRevisions(revisions -> revisions.with(version))); start(id, type, @@ -615,24 +617,27 @@ public class JobController { .orElseGet(() -> ApplicationPackageDiff.diffAgainstEmpty(applicationPackage)); } - private Version findTargetPlatform(ApplicationPackage applicationPackage, Optional<Run> lastRun, ApplicationId id) { + private Version findTargetPlatform(ApplicationPackage applicationPackage, DeploymentId id, Optional<Instance> instance) { Optional<Integer> major = applicationPackage.deploymentSpec().majorVersion(); if (major.isPresent()) return controller.applications().lastCompatibleVersion(major.get()) .orElseThrow(() -> new IllegalArgumentException("major " + major.get() + " specified in deployment.xml, " + "but no version on this major was found")); - // Prefer previous platform if possible. - VersionStatus versionStatus = controller.readVersionStatus(); - VersionCompatibility compatibility = controller.applications().versionCompatibility(id); - Optional<Version> target = lastRun.map(run -> run.versions().targetPlatform()).filter(versionStatus::isActive); - if (target.isPresent() && compatibility.accept(target.get(), applicationPackage.compileVersion().orElse(target.get()))) - return target.get(); + VersionCompatibility compatibility = controller.applications().versionCompatibility(id.applicationId()); + + // Prefer previous platform if possible. Candidates are all deployable, ascending, with existing version appended; then reversed. + List<Version> versions = controller.readVersionStatus().deployableVersions().stream() + .map(VespaVersion::versionNumber) + .collect(toList()); + instance.map(Instance::deployments) + .map(deployments -> deployments.get(id.zoneId())) + .map(Deployment::version) + .ifPresent(versions::add); - // Otherwise, use newest, compatible version. - for (VespaVersion platform : reversed(versionStatus.deployableVersions())) - if (compatibility.accept(platform.versionNumber(), applicationPackage.compileVersion().orElse(platform.versionNumber()))) - return platform.versionNumber(); + for (Version target : reversed(versions)) + if (applicationPackage.compileVersion().isEmpty() || compatibility.accept(target, applicationPackage.compileVersion().get())) + return target; throw new IllegalArgumentException("no suitable platform version found" + applicationPackage.compileVersion() @@ -686,7 +691,7 @@ public class JobController { } public void deactivateTester(TesterId id, JobType type) { - controller.serviceRegistry().configServer().deactivate(new DeploymentId(id.id(), type.zone(controller.system()))); + controller.serviceRegistry().configServer().deactivate(new DeploymentId(id.id(), type.zone())); } /** Locks all runs and modifies the list of historic runs for the given application and job type. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java index e95e515685f..14fce806152 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java @@ -47,7 +47,7 @@ public class JobMetrics { "tenantName", id.application().tenant().value(), "app", id.application().application().value() + "." + id.application().instance().value(), "test", Boolean.toString(id.type().isTest()), - "zone", id.type().zone(system.get()).value()); + "zone", id.type().zone().value()); } static String valueOf(RunStatus status) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java index 063167647d5..e0c1fef91b3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RetriggerEntrySerializer.java @@ -24,12 +24,6 @@ public class RetriggerEntrySerializer { private static final String JOB_TYPE_KEY = "jobType"; private static final String MIN_REQUIRED_RUN_ID_KEY = "minimumRunId"; - private final SystemName system; - - public RetriggerEntrySerializer(SystemName system) { - this.system = system; - } - public List<RetriggerEntry> fromSlime(Slime slime) { return SlimeUtils.entriesStream(slime.get().field("entries")) .map(this::deserializeEntry) @@ -48,14 +42,14 @@ public class RetriggerEntrySerializer { Cursor root = array.addObject(); Cursor jobid = root.setObject(JOB_ID_KEY); jobid.setString(APPLICATION_ID_KEY, entry.jobId().application().serializedForm()); - jobid.setString(JOB_TYPE_KEY, entry.jobId().type().serialized(system)); + jobid.setString(JOB_TYPE_KEY, entry.jobId().type().serialized()); root.setLong(MIN_REQUIRED_RUN_ID_KEY, entry.requiredRun()); } private RetriggerEntry deserializeEntry(Inspector inspector) { Inspector jobid = inspector.field(JOB_ID_KEY); ApplicationId applicationId = ApplicationId.fromSerializedForm(require(jobid, APPLICATION_ID_KEY).asString()); - JobType jobType = JobType.fromJobName(require(jobid, JOB_TYPE_KEY).asString()); + JobType jobType = JobType.ofSerialized(require(jobid, JOB_TYPE_KEY).asString()); long minRequiredRunId = require(inspector, MIN_REQUIRED_RUN_ID_KEY).asLong(); return new RetriggerEntry(new JobId(applicationId, jobType), minRequiredRunId); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java index da42a00cd44..1680e064234 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java @@ -37,7 +37,7 @@ public class TestConfigSerializer { Cursor root = slime.setObject(); root.setString("application", id.serializedForm()); - root.setString("zone", type.zone(system).value()); + root.setString("zone", type.zone().value()); root.setString("system", system.value()); root.setBool("isCI", isCI); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index deafcd35e9b..041d0694ca9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement; import java.time.Duration; @@ -36,7 +37,7 @@ public class ControllerMaintenance extends AbstractComponent { @Inject @SuppressWarnings("unused") // instantiated by Dependency Injection - public ControllerMaintenance(Controller controller, Metric metric, UserManagement userManagement) { + public ControllerMaintenance(Controller controller, Metric metric, UserManagement userManagement, AthenzClientFactory athenzClientFactory) { Intervals intervals = new Intervals(controller.system()); upgrader = new Upgrader(controller, intervals.defaultInterval); maintainers.add(upgrader); @@ -44,7 +45,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new DeploymentExpirer(controller, intervals.defaultInterval)); maintainers.add(new DeploymentUpgrader(controller, intervals.defaultInterval)); maintainers.add(new DeploymentIssueReporter(controller, controller.serviceRegistry().deploymentIssues(), intervals.defaultInterval)); - maintainers.add(new MetricsReporter(controller, metric)); + maintainers.add(new MetricsReporter(controller, metric, athenzClientFactory.createZmsClient())); maintainers.add(new OutstandingChangeDeployer(controller, intervals.outstandingChangeDeployer)); maintainers.add(new VersionStatusUpdater(controller, intervals.versionStatusUpdater)); maintainers.add(new ReadyJobsTrigger(controller, intervals.readyJobsTrigger)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java index d33603243e2..97f3f955a20 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java @@ -57,10 +57,6 @@ public class DeploymentExpirer extends ControllerMaintainer { Optional<Duration> ttl = controller().zoneRegistry().getDeploymentTimeToLive(deployment.zone()); if (ttl.isEmpty()) return false; - Optional<JobId> jobId = JobType.from(controller().system(), deployment.zone()) - .map(type -> new JobId(instance, type)); - if (jobId.isEmpty()) return false; - return controller().jobController().lastDeploymentStart(instance, deployment) .plus(ttl.get()).isBefore(controller().clock().instant()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java index 144c27b9e5e..c86f79ce188 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java @@ -47,7 +47,7 @@ public class DeploymentUpgrader extends ControllerMaintainer { for (Instance instance : application.instances().values()) for (Deployment deployment : instance.deployments().values()) try { - JobId job = new JobId(instance.id(), JobType.from(controller().system(), deployment.zone()).get()); + JobId job = new JobId(instance.id(), JobType.deploymentTo(deployment.zone())); if ( ! deployment.zone().environment().isManuallyDeployed()) continue; Run last = controller().jobController().last(job).get(); @@ -60,7 +60,7 @@ public class DeploymentUpgrader extends ControllerMaintainer { log.log(Level.FINE, "Upgrading deployment of " + instance.id() + " in " + deployment.zone()); attempts.incrementAndGet(); - controller().jobController().start(instance.id(), JobType.from(controller().system(), deployment.zone()).get(), target, true, Optional.of("automated upgrade")); + controller().jobController().start(instance.id(), JobType.deploymentTo(deployment.zone()), target, true, Optional.of("automated upgrade")); } catch (Exception e) { failures.incrementAndGet(); log.log(Level.WARNING, "Failed upgrading " + deployment + " of " + instance + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index faa42e5caef..193fb89eb99 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -104,7 +104,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { controller().applications().getInstance(applicationId) .ifPresent(instance -> instance.productionDeployments().forEach((zone, deployment) -> { if (deployment.at().isBefore(refreshTime)) { - JobType job = JobType.from(controller().system(), zone).orElseThrow(); + JobType job = JobType.deploymentTo(zone); deploymentTrigger.reTrigger(applicationId, job, "re-triggered by EndpointCertificateMaintainer"); log.info("Re-triggering deployment job " + job.jobName() + " for instance " + applicationId.serializedForm() + " to roll out refreshed endpoint certificate"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 74bbf906653..294d5bad42d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.athenz.client.zms.ZmsClient; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -62,17 +63,20 @@ public class MetricsReporter extends ControllerMaintainer { public static final String REMAINING_ROTATIONS = "remaining_rotations"; public static final String NAME_SERVICE_REQUESTS_QUEUED = "dns.queuedRequests"; public static final String OPERATION_PREFIX = "operation."; + public static final String ZMS_QUOTA_USAGE = "zms.quota.usage"; private final Metric metric; private final Clock clock; + private final ZmsClient zmsClient; // Keep track of reported node counts for each version private final ConcurrentHashMap<NodeCountKey, Long> nodeCounts = new ConcurrentHashMap<>(); - public MetricsReporter(Controller controller, Metric metric) { + public MetricsReporter(Controller controller, Metric metric, ZmsClient zmsClient) { super(controller, Duration.ofMinutes(1)); // use fixed rate for metrics this.metric = metric; this.clock = controller.clock(); + this.zmsClient = zmsClient; } @Override @@ -85,6 +89,7 @@ public class MetricsReporter extends ControllerMaintainer { reportAuditLog(); reportBrokenSystemVersion(versionStatus); reportTenantMetrics(); + reportZmsQuotaMetrics(); return 1.0; } @@ -252,6 +257,20 @@ public class MetricsReporter extends ControllerMaintainer { }); } + private void reportZmsQuotaMetrics() { + var quota = zmsClient.getQuotaUsage(); + reportZmsQuota("subdomains", quota.getSubdomainUsage()); + reportZmsQuota("services", quota.getServiceUsage()); + reportZmsQuota("policies", quota.getPolicyUsage()); + reportZmsQuota("roles", quota.getRoleUsage()); + reportZmsQuota("groups", quota.getGroupUsage()); + } + + private void reportZmsQuota(String resourceType, double usage) { + var context = metric.createContext(Map.of("resourceType", resourceType)); + metric.set(ZMS_QUOTA_USAGE, usage, context); + } + private Map<NodeVersion, Duration> platformChangeDurations(VersionStatus versionStatus) { return changeDurations(versionStatus.versions(), VespaVersion::nodeVersions); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index e853dbc0d5a..48d8627f407 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -157,12 +157,6 @@ public class ApplicationSerializer { private static final String deploymentCostField = "cost"; - private final SystemName system; - - public ApplicationSerializer(SystemName system) { - this.system = system; - } - // ------------------ Serialization public Slime toSlime(Application application) { @@ -244,7 +238,7 @@ public class ApplicationSerializer { revisions.development().forEach((job, devRevisions) -> { Cursor devRevisionsObject = devRevisionsArray.addObject(); devRevisionsObject.setString(instanceNameField, job.application().instance().value()); - devRevisionsObject.setString(jobTypeField, job.type().serialized(system)); + devRevisionsObject.setString(jobTypeField, job.type().serialized()); revisionsToSlime(devRevisions, devRevisionsObject.setArray(versionsField)); }); } @@ -284,7 +278,7 @@ public class ApplicationSerializer { Cursor jobStatusArray = cursor.setArray(jobStatusField); jobPauses.forEach((type, until) -> { Cursor jobPauseObject = jobStatusArray.addObject(); - jobPauseObject.setString(jobTypeField, type.serialized(system)); + jobPauseObject.setString(jobTypeField, type.serialized()); jobPauseObject.setLong(pausedUntilField, until.toEpochMilli()); }); } @@ -371,7 +365,7 @@ public class ApplicationSerializer { private JobId jobIdFromSlime(TenantAndApplicationId base, Inspector idObject) { return new JobId(base.instance(idObject.field(instanceNameField).asString()), - JobType.fromJobName(idObject.field(jobTypeField).asString())); + JobType.ofSerialized(idObject.field(jobTypeField).asString())); } private List<ApplicationVersion> revisionsFromSlime(Inspector versionsArray, JobId job) { @@ -414,7 +408,7 @@ public class ApplicationSerializer { private Deployment deploymentFromSlime(Inspector deploymentObject, ApplicationId id) { ZoneId zone = zoneIdFromSlime(deploymentObject.field(zoneField)); return new Deployment(zone, - revisionFromSlime(deploymentObject.field(applicationPackageRevisionField), new JobId(id, JobType.from(system, zone).get())), + revisionFromSlime(deploymentObject.field(applicationPackageRevisionField), new JobId(id, JobType.deploymentTo(zone))), Version.fromString(deploymentObject.field(versionField).asString()), SlimeUtils.instant(deploymentObject.field(deployTimeField)), deploymentMetricsFromSlime(deploymentObject.field(deploymentMetricsField)), @@ -507,9 +501,8 @@ public class ApplicationSerializer { private Map<JobType, Instant> jobPausesFromSlime(Inspector object) { Map<JobType, Instant> jobPauses = new HashMap<>(); object.field(jobStatusField).traverse((ArrayTraverser) (__, jobPauseObject) -> - JobType.fromOptionalJobName(jobPauseObject.field(jobTypeField).asString()) - .ifPresent(jobType -> jobPauses.put(jobType, - SlimeUtils.instant(jobPauseObject.field(pausedUntilField))))); + jobPauses.put(JobType.ofSerialized(jobPauseObject.field(jobTypeField).asString()), + SlimeUtils.instant(jobPauseObject.field(pausedUntilField)))); return jobPauses; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 45b762b1b9c..16398fb1137 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -114,10 +114,10 @@ public class CuratorDb { private final ZoneRoutingPolicySerializer zoneRoutingPolicySerializer = new ZoneRoutingPolicySerializer(routingPolicySerializer); private final AuditLogSerializer auditLogSerializer = new AuditLogSerializer(); private final NameServiceQueueSerializer nameServiceQueueSerializer = new NameServiceQueueSerializer(); - private final ApplicationSerializer applicationSerializer; - private final RunSerializer runSerializer; - private final RetriggerEntrySerializer retriggerEntrySerializer; - private final NotificationsSerializer notificationsSerializer; + private final ApplicationSerializer applicationSerializer = new ApplicationSerializer(); + private final RunSerializer runSerializer = new RunSerializer(); + private final RetriggerEntrySerializer retriggerEntrySerializer = new RetriggerEntrySerializer(); + private final NotificationsSerializer notificationsSerializer = new NotificationsSerializer(); private final Curator curator; private final Duration tryLockTimeout; @@ -139,10 +139,6 @@ public class CuratorDb { this.curator = curator; this.tryLockTimeout = tryLockTimeout; this.lockScheme = Flags.CONTROLLER_LOCK_SCHEME.bindTo(flagSource); - this.applicationSerializer = new ApplicationSerializer(system); - this.runSerializer = new RunSerializer(system); - this.retriggerEntrySerializer = new RetriggerEntrySerializer(system); - this.notificationsSerializer = new NotificationsSerializer(system); } /** Returns all hostnames configured to be part of this ZooKeeper cluster */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java index 1d5f6d70ca5..c882681632e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java @@ -44,12 +44,6 @@ public class NotificationsSerializer { private static final String jobTypeField = "jobId"; private static final String runNumberField = "runNumber"; - private final SystemName system; - - NotificationsSerializer(SystemName system) { - this.system = system; - } - public Slime toSlime(List<Notification> notifications) { Slime slime = new Slime(); Cursor notificationsArray = slime.setObject().setArray(notificationsFieldName); @@ -66,7 +60,7 @@ public class NotificationsSerializer { notification.source().instance().ifPresent(instance -> notificationObject.setString(instanceField, instance.value())); notification.source().zoneId().ifPresent(zoneId -> notificationObject.setString(zoneField, zoneId.value())); notification.source().clusterId().ifPresent(clusterId -> notificationObject.setString(clusterIdField, clusterId.value())); - notification.source().jobType().ifPresent(jobType -> notificationObject.setString(jobTypeField, jobType.serialized(system))); + notification.source().jobType().ifPresent(jobType -> notificationObject.setString(jobTypeField, jobType.serialized())); notification.source().runNumber().ifPresent(runNumber -> notificationObject.setLong(runNumberField, runNumber)); } @@ -90,7 +84,7 @@ public class NotificationsSerializer { SlimeUtils.optionalString(inspector.field(instanceField)).map(InstanceName::from), SlimeUtils.optionalString(inspector.field(zoneField)).map(ZoneId::from), SlimeUtils.optionalString(inspector.field(clusterIdField)).map(ClusterSpec.Id::from), - SlimeUtils.optionalString(inspector.field(jobTypeField)).map(JobType::fromJobName), + SlimeUtils.optionalString(inspector.field(jobTypeField)).map(jobName -> JobType.ofSerialized(jobName)), SlimeUtils.optionalLong(inspector.field(runNumberField))), SlimeUtils.entriesStream(inspector.field(messagesField)).map(Inspector::asString).collect(Collectors.toUnmodifiableList())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index 143aaaeabb8..dd28978d948 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -98,12 +98,6 @@ class RunSerializer { private static final String isDryRunField = "isDryRun"; private static final String reasonField = "reason"; - private final SystemName system; - - RunSerializer(SystemName system) { - this.system = system; - } - Run runFromSlime(Slime slime) { return runFromSlime(slime.get()); } @@ -132,7 +126,7 @@ class RunSerializer { steps.put(typedStep, new StepInfo(typedStep, stepStatusOf(status.asString()), startTime)); }); RunId id = new RunId(ApplicationId.fromSerializedForm(runObject.field(applicationField).asString()), - JobType.fromJobName(runObject.field(jobTypeField).asString()), + JobType.ofSerialized(runObject.field(jobTypeField).asString()), runObject.field(numberField).asLong()); return new Run(id, steps, @@ -212,7 +206,7 @@ class RunSerializer { private void toSlime(Run run, Cursor runObject) { runObject.setString(applicationField, run.id().application().serializedForm()); - runObject.setString(jobTypeField, run.id().type().serialized(system)); + runObject.setString(jobTypeField, run.id().type().serialized()); runObject.setBool(isRedeploymentField, run.isRedeployment()); runObject.setLong(numberField, run.id().number()); runObject.setLong(startField, run.start().toEpochMilli()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index c11a5adf3b9..c07794ea39c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -797,14 +797,14 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } private HttpResponse devApplicationPackage(ApplicationId id, JobType type) { - ZoneId zone = type.zone(controller.system()); + ZoneId zone = type.zone(); RevisionId revision = controller.jobController().last(id, type).get().versions().targetRevision(); byte[] applicationPackage = controller.applications().applicationStore().get(new DeploymentId(id, zone), revision); return new ZipResponse(id.toFullString() + "." + zone.value() + ".zip", applicationPackage); } private HttpResponse devApplicationPackageDiff(RunId runId) { - DeploymentId deploymentId = new DeploymentId(runId.application(), runId.job().type().zone(controller.system())); + DeploymentId deploymentId = new DeploymentId(runId.application(), runId.job().type().zone()); return controller.applications().applicationStore().getDevDiff(deploymentId, runId.number()) .map(ByteArrayResponse::new) .orElseThrow(() -> new NotExistsException("No application package diff found for " + runId)); @@ -1102,12 +1102,6 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { Slime slime = new Slime(); Cursor nodesArray = slime.setObject().setArray("nodes"); for (Node node : nodes) { - Optional<Instant> downAt = node.history().stream() - .filter(event -> "down".equals(event.name())) - .map(Node.Event::at) - .findFirst(); - boolean isUp = downAt.isEmpty() || node.history().stream() - .anyMatch(event -> "up".equals(event.name()) && event.at().isAfter(downAt.get())); Cursor nodeObject = nodesArray.addObject(); nodeObject.setString("hostname", node.hostname().value()); nodeObject.setString("state", valueOf(node.state())); @@ -1118,8 +1112,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { toSlime(node.resources(), nodeObject); nodeObject.setString("clusterId", node.clusterId()); nodeObject.setString("clusterType", valueOf(node.clusterType())); - nodeObject.setBool("down", !isUp); -// nodeObject.setBool("down", node.down()); // TODO (valerijf): Enable when all configservers expose this + nodeObject.setBool("down", node.down()); nodeObject.setBool("retired", node.retired() || node.wantToRetire()); nodeObject.setBool("restarting", node.wantedRestartGeneration() > node.restartGeneration()); nodeObject.setBool("rebooting", node.wantedRebootGeneration() > node.rebootGeneration()); @@ -1393,7 +1386,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { // Deployments sorted according to deployment spec List<Deployment> deployments = deploymentSpec.instance(instance.name()) - .map(spec -> new DeploymentSteps(spec, controller::system)) + .map(spec -> new DeploymentSteps(spec, controller.zoneRegistry())) .map(steps -> steps.sortedDeployments(instance.deployments().values())) .orElse(List.copyOf(instance.deployments().values())); @@ -1481,7 +1474,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { // Deployments sorted according to deployment spec List<Deployment> deployments = application.deploymentSpec().instance(instance.name()) - .map(spec -> new DeploymentSteps(spec, controller::system)) + .map(spec -> new DeploymentSteps(spec, controller.zoneRegistry())) .map(steps -> steps.sortedDeployments(instance.deployments().values())) .orElse(List.copyOf(instance.deployments().values())); Cursor instancesArray = object.setArray("instances"); @@ -1523,7 +1516,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { controller.jobController().active(instance.id()).stream() .map(run -> run.id().job()) .filter(job -> job.type().environment().isManuallyDeployed())) - .map(job -> job.type().zone(controller.system())) + .map(job -> job.type().zone()) .filter(zone -> ! instance.deployments().containsKey(zone)) .forEach(zone -> { Cursor deploymentObject = instancesArray.addObject(); @@ -1635,9 +1628,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (!deployment.zone().environment().isManuallyDeployed()) { DeploymentStatus status = controller.jobController().deploymentStatus(application); - JobType.from(controller.system(), deployment.zone()) - .map(type -> new JobId(instance.id(), type)) - .map(status.jobSteps()::get) + JobId jobId = new JobId(instance.id(), JobType.deploymentTo(deployment.zone())); + Optional.ofNullable(status.jobSteps().get(jobId)) .ifPresent(stepStatus -> { JobControllerApiHandlerHelper.toSlime(response.setObject("applicationVersion"), application.revisions().get(deployment.revision())); if ( ! status.jobsToRun().containsKey(stepStatus.job().get())) @@ -1647,9 +1639,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { else response.setString("status", "running"); }); } else { - var deploymentRun = JobType.from(controller.system(), deploymentId.zoneId()) - .flatMap(jobType -> controller.jobController().last(deploymentId.applicationId(), jobType)); - + var deploymentRun = controller.jobController().last(deploymentId.applicationId(), JobType.deploymentTo(deploymentId.zoneId())); deploymentRun.ifPresent(run -> { response.setString("status", run.hasEnded() ? "complete" : "running"); }); @@ -2066,7 +2056,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get(EnvironmentResource.APPLICATION_ZIP)); controller.applications().verifyApplicationIdentityConfiguration(id.tenant(), Optional.of(id.instance()), - Optional.of(type.zone(controller.system())), + Optional.of(type.zone()), applicationPackage, Optional.of(requireUserPrincipal(request))); @@ -2179,7 +2169,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { .flatMap(instance -> instance.productionDeployments().keySet().stream()) .map(zone -> new DeploymentId(prodInstanceId, zone)) .collect(Collectors.toCollection(HashSet::new)); - ZoneId testedZone = type.zone(controller.system()); + ZoneId testedZone = type.zone(); // If a production job is specified, the production deployment of the orchestrated instance is the relevant one, // as user instances should not exist in prod. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 0bbfaea3e6f..fac883299e3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -341,7 +341,7 @@ class JobControllerApiHandlerHelper { "/job/" + job.type().jobName()).normalize(); stepObject.setString("url", baseUriForJob.toString()); stepObject.setString("environment", job.type().environment().value()); - stepObject.setString("region", job.type().zone(controller.system()).value()); + stepObject.setString("region", job.type().zone().value()); if (job.type().isProduction() && job.type().isDeployment()) { status.deploymentFor(job).ifPresent(deployment -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java index 4cfcadd5c41..9b400fdfb78 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiHandler.java @@ -81,9 +81,7 @@ public class BadgeApiHandler extends ThreadedHttpRequestHandler { () -> { DeploymentStatus status = controller.jobController().deploymentStatus(controller.applications().requireApplication(TenantAndApplicationId.from(id))); Predicate<JobStatus> isDeclaredJob = job -> status.jobSteps().get(job.id()) != null && status.jobSteps().get(job.id()).isDeclared(); - return Badges.overviewBadge(id, - status.jobs().instance(id.instance()).matching(isDeclaredJob), - controller.system()); + return Badges.overviewBadge(id, status.jobs().instance(id.instance()).matching(isDeclaredJob)); }); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java index 7b4f2fec853..26a5da45bdb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/Badges.java @@ -144,7 +144,7 @@ public class Badges { return badge(sections, texts, x); } - static String overviewBadge(ApplicationId id, JobList jobs, SystemName system) { + static String overviewBadge(ApplicationId id, JobList jobs) { // Put production tests right after their deployments, for a more compact rendering. List<Run> runs = new ArrayList<>(jobs.lastTriggered().asList()); boolean anyTest = false; @@ -153,7 +153,7 @@ public class Badges { if (run.id().type().isProduction() && run.id().type().isTest()) { anyTest = true; int j = i; - while ( ! runs.get(j - 1).id().type().zone(system).equals(run.id().type().zone(system))) + while ( ! runs.get(j - 1).id().type().zone().equals(run.id().type().zone())) runs.set(j, runs.get(--j)); runs.set(j, run); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java index 43037322f22..01bd02fdc13 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java @@ -111,7 +111,7 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { } else if(path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/{*}")) { zone = Optional.of(ZoneId.from(path.get("environment"), path.get("region"))); } else if(path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploy/{jobname}")) { - zone = Optional.of(JobType.fromJobName(path.get("jobname"), zones).zone(systemName)); + zone = Optional.of(JobType.fromJobName(path.get("jobname"), zones).zone()); } else { zone = Optional.empty(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 687820d15cc..e9619297a2f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -23,7 +23,6 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.dns.LatencyAliasTarget; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; @@ -61,11 +60,10 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import static com.yahoo.config.provision.SystemName.main; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.devUsEast1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsEast3; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsEast3; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -172,7 +170,7 @@ public class ControllerTest { e.getMessage()); } assertNotNull("Zone was not removed", - context.instance().deployments().get(productionUsWest1.zone(main))); + context.instance().deployments().get(productionUsWest1.zone())); // prod zone removal is allowed with override applicationPackage = new ApplicationPackageBuilder() @@ -182,7 +180,7 @@ public class ControllerTest { .build(); context.submit(applicationPackage); assertNull("Zone was removed", - context.instance().deployments().get(productionUsWest1.zone(main))); + context.instance().deployments().get(productionUsWest1.zone())); assertNull("Deployment job was removed", context.instanceJobs().get(productionUsWest1)); // Submission has stored application meta. @@ -203,7 +201,7 @@ public class ControllerTest { .get(tester.clock().instant())); assertNull(tester.controllerTester().serviceRegistry().applicationStore() - .getMeta(context.deploymentIdIn(productionUsWest1.zone(main)))); + .getMeta(context.deploymentIdIn(productionUsWest1.zone()))); } @Test @@ -875,6 +873,8 @@ public class ControllerTest { @Test public void testDeployWithGlobalEndpointsInMultipleClouds() { tester.controllerTester().zoneRegistry().setZones( + ZoneApiMock.fromId("test.us-west-1"), + ZoneApiMock.fromId("staging.us-west-1"), ZoneApiMock.fromId("prod.us-west-1"), ZoneApiMock.newBuilder().with(CloudName.from("aws")).withId("prod.aws-us-east-1").build() ); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java index 84af9f4af85..df31883b1d5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java @@ -146,10 +146,10 @@ public class EndpointCertificatesTest { "*.default.default.g.vespa-app.cloud", "default.default.aws-us-east-1a.z.vespa-app.cloud", "*.default.default.aws-us-east-1a.z.vespa-app.cloud", - "default.default.aws-us-east-1c.test.z.vespa-app.cloud", - "*.default.default.aws-us-east-1c.test.z.vespa-app.cloud", - "default.default.aws-us-east-1c.staging.z.vespa-app.cloud", - "*.default.default.aws-us-east-1c.staging.z.vespa-app.cloud" + "default.default.us-east-1.test.z.vespa-app.cloud", + "*.default.default.us-east-1.test.z.vespa-app.cloud", + "default.default.us-east-3.staging.z.vespa-app.cloud", + "*.default.default.us-east-3.staging.z.vespa-app.cloud" ); Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificates.getMetadata(testInstance, testZone, DeploymentSpec.empty); assertTrue(endpointCertificateMetadata.isPresent()); @@ -261,10 +261,10 @@ public class EndpointCertificatesTest { "*.a1.t1.aws-us-east-1c.r.vespa-app.cloud", "a1.t1.aws-us-east-1c.z.vespa-app.cloud", "*.a1.t1.aws-us-east-1c.z.vespa-app.cloud", - "a1.t1.aws-us-east-1c.test.z.vespa-app.cloud", - "*.a1.t1.aws-us-east-1c.test.z.vespa-app.cloud", - "a1.t1.aws-us-east-1c.staging.z.vespa-app.cloud", - "*.a1.t1.aws-us-east-1c.staging.z.vespa-app.cloud" + "a1.t1.us-east-1.test.z.vespa-app.cloud", + "*.a1.t1.us-east-1.test.z.vespa-app.cloud", + "a1.t1.us-east-3.staging.z.vespa-app.cloud", + "*.a1.t1.us-east-3.staging.z.vespa-app.cloud" ); Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificates.getMetadata(instance, zone1, applicationPackage.deploymentSpec()); assertTrue(endpointCertificateMetadata.isPresent()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 3765f815e49..ad6a987d42b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -21,7 +21,6 @@ import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; @@ -78,6 +77,28 @@ import static org.junit.Assert.assertTrue; */ public class DeploymentContext { + public static final JobType systemTest = JobType.deploymentTo(ZoneId.from("test", "us-east-1")); + public static final JobType stagingTest = JobType.deploymentTo(ZoneId.from("staging", "us-east-3")); + public static final JobType productionUsEast3 = JobType.prod("us-east-3"); + public static final JobType testUsEast3 = JobType.test("us-east-3"); + public static final JobType productionUsWest1 = JobType.prod("us-west-1"); + public static final JobType testUsWest1 = JobType.test("us-west-1"); + public static final JobType productionUsCentral1 = JobType.prod("us-central-1"); + public static final JobType testUsCentral1 = JobType.test("us-central-1"); + public static final JobType productionApNortheast1 = JobType.prod("ap-northeast-1"); + public static final JobType testApNortheast1 = JobType.test("ap-northeast-1"); + public static final JobType productionApNortheast2 = JobType.prod("ap-northeast-2"); + public static final JobType testApNortheast2 = JobType.test("ap-northeast-2"); + public static final JobType productionApSoutheast1 = JobType.prod("ap-southeast-1"); + public static final JobType testApSoutheast1 = JobType.test("ap-southeast-1"); + public static final JobType productionEuWest1 = JobType.prod("eu-west-1"); + public static final JobType testEuWest1 = JobType.test("eu-west-1"); + public static final JobType productionAwsUsEast1a = JobType.prod("aws-us-east-1a"); + public static final JobType testAwsUsEast1a = JobType.test("aws-us-east-1a"); + public static final JobType devUsEast1 = JobType.dev("us-east-1"); + public static final JobType devAwsUsEast2a = JobType.dev("aws-us-east-2a"); + public static final JobType perfUsEast3 = JobType.perf("us-east-3"); + private final AtomicLong salt = new AtomicLong(); // Application packages are expensive to construct, and a given test typically only needs to the test in the context @@ -198,10 +219,10 @@ public class DeploymentContext { .allMatch(deployment -> deployment.version().equals(version)))); for (var spec : application().deploymentSpec().instances()) - for (JobType type : new DeploymentSteps(spec, tester.controller()::system).productionJobs()) + for (JobType type : new DeploymentSteps(spec, tester.controller().zoneRegistry()).productionJobs()) assertTrue(tester.configServer().nodeRepository() - .list(type.zone(tester.controller().system()), - NodeFilter.all().applications(applicationId.defaultInstance())).stream() // TODO jonmv: support more + .list(type.zone(), + NodeFilter.all().applications(applicationId.defaultInstance())).stream() .allMatch(node -> node.currentVersion().equals(version))); assertFalse(instance().change().hasTargets()); @@ -395,7 +416,7 @@ public class DeploymentContext { /** Runs a deployment of the given package to the given manually deployable zone. */ public DeploymentContext runJob(ZoneId zone, ApplicationPackage applicationPackage) { - return runJob(JobType.from(tester.controller().system(), zone).get(), applicationPackage, null); + return runJob(JobType.deploymentTo(zone), applicationPackage, null); } /** Pulls the ready job trigger, and then runs the whole of the given job in the instance of this, successfully. */ @@ -417,14 +438,13 @@ public class DeploymentContext { } RunId id = currentRun(job).id(); - ZoneId zone = zone(job); assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.endTests)); tester.cloud().set(noTests ? Status.NO_TESTS : Status.FAILURE); runner.advance(currentRun(job)); assertTrue(jobs.run(id).get().hasEnded()); assertEquals(noTests, jobs.run(id).get().hasSucceeded()); - assertTrue(configServer().nodeRepository().list(zone, NodeFilter.all().applications(TesterId.of(instanceId).id())).isEmpty()); + assertTrue(configServer().nodeRepository().list(job.type().zone(), NodeFilter.all().applications(TesterId.of(instanceId).id())).isEmpty()); return this; } @@ -497,8 +517,8 @@ public class DeploymentContext { tester.readyJobsTrigger().maintain(); if (type.isProduction()) { - runJob(JobType.systemTest); - runJob(JobType.stagingTest); + runJob(systemTest); + runJob(stagingTest); tester.readyJobsTrigger().maintain(); } @@ -511,8 +531,8 @@ public class DeploymentContext { /** Start tests in system test stage */ public RunId startSystemTestTests() { - var id = newRun(JobType.systemTest); - var testZone = JobType.systemTest.zone(tester.controller().system()); + var id = newRun(systemTest); + var testZone = systemTest.zone(); runner.run(); if ( ! deferDnsUpdates) flushDnsUpdates(); @@ -537,7 +557,7 @@ public class DeploymentContext { /** Deploys tester and real app, and completes tester and initial staging installation first if needed. */ private void doDeploy(JobId job) { RunId id = currentRun(job).id(); - ZoneId zone = zone(job); + ZoneId zone = job.type().zone(); DeploymentId deployment = new DeploymentId(job.application(), zone); // First step is always a deployment. @@ -549,7 +569,7 @@ public class DeploymentContext { if (job.type().isTest()) doInstallTester(job); - if (job.type().equals(JobType.stagingTest)) { // Do the initial deployment and installation of the real application. + if (job.type().equals(stagingTest)) { // Do the initial deployment and installation of the real application. assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installInitialReal)); tester.configServer().nodeRepository().doUpgrade(deployment, Optional.empty(), tester.configServer().application(job.application(), zone).get().version().get()); configServer().convergeServices(id.application(), zone); @@ -571,7 +591,7 @@ public class DeploymentContext { /** Upgrades nodes to target version. */ private void doUpgrade(JobId job) { RunId id = currentRun(job).id(); - ZoneId zone = zone(job); + ZoneId zone = job.type().zone(); DeploymentId deployment = new DeploymentId(job.application(), zone); assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installReal)); @@ -592,7 +612,7 @@ public class DeploymentContext { /** Lets nodes converge on new application version. */ private void doConverge(JobId job) { RunId id = currentRun(job).id(); - ZoneId zone = zone(job); + ZoneId zone = job.type().zone(); assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installReal)); configServer().convergeServices(id.application(), zone); @@ -608,7 +628,7 @@ public class DeploymentContext { /** Installs tester and starts tests. */ private void doInstallTester(JobId job) { RunId id = currentRun(job).id(); - ZoneId zone = zone(job); + ZoneId zone = job.type().zone(); assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installTester)); configServer().nodeRepository().doUpgrade(new DeploymentId(TesterId.of(job.application()).id(), zone), Optional.empty(), tester.configServer().application(id.tester().id(), zone).get().version().get()); @@ -623,7 +643,7 @@ public class DeploymentContext { /** Completes tests with success. */ private void doTests(JobId job) { RunId id = currentRun(job).id(); - ZoneId zone = zone(job); + ZoneId zone = job.type().zone(); // All installation is complete and endpoints are ready, so tests may begin. if (job.type().isDeployment()) @@ -645,10 +665,6 @@ public class DeploymentContext { return new JobId(instanceId, type); } - private ZoneId zone(JobId job) { - return job.type().zone(tester.controller().system()); - } - private ConfigServerMock configServer() { return tester.configServer(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 5a330b00809..4d4d94f9e1f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -3,19 +3,15 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import org.junit.Assert; import org.junit.Test; @@ -23,7 +19,6 @@ import org.junit.Test; import java.time.Duration; import java.time.Instant; import java.util.Collection; -import java.util.EnumSet; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -34,26 +29,23 @@ import java.util.stream.Collectors; import static ai.vespa.validation.Validation.require; import static com.yahoo.config.provision.SystemName.cd; -import static com.yahoo.config.provision.SystemName.main; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionApNortheast1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionApNortheast2; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionApSoutheast1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionAwsUsEast1a; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionCdAwsUsEast1a; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionCdUsEast1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionEuWest1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsCentral1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsEast3; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.testApNortheast1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.testApNortheast2; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.testAwsUsEast1a; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.testEuWest1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.testUsCentral1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.testUsEast3; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.testUsWest1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionApNortheast1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionApNortheast2; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionApSoutheast1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionAwsUsEast1a; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionEuWest1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsCentral1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsEast3; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.testApNortheast1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.testApNortheast2; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.testAwsUsEast1a; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.testEuWest1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.testUsCentral1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.testUsEast3; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.testUsWest1; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.ALL; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.PLATFORM; import static java.util.Collections.emptyList; @@ -572,7 +564,7 @@ public class DeploymentTriggerTest { tester.deploymentTrigger().forceTrigger(app.instanceId(), productionUsEast3, "mrTrigger", true, true, false); app.assertRunning(productionUsEast3); assertFalse(app.instance().jobPause(productionUsEast3).isPresent()); - assertEquals(app.deployment(productionUsEast3.zone(tester.controller().system())).version(), + assertEquals(app.deployment(productionUsEast3.zone()).version(), tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetPlatform()); } @@ -641,7 +633,7 @@ public class DeploymentTriggerTest { .runJob(productionUsEast3) .runJob(productionUsWest1); assertEquals(Change.empty(), app.instance().change()); - assertEquals(appVersion0, app.instance().deployments().get(productionUsEast3.zone(tester.controller().system())).revision()); + assertEquals(appVersion0, app.instance().deployments().get(productionUsEast3.zone()).revision()); assertEquals(appVersion0, latestDeployed(app.instance())); } @@ -694,7 +686,7 @@ public class DeploymentTriggerTest { tester.triggerJobs(); app1.jobAborted(systemTest).jobAborted(stagingTest); app1.runJob(systemTest).runJob(stagingTest).timeOutConvergence(productionUsCentral1); - assertEquals(version2, app1.deployment(productionUsCentral1.zone(main)).version()); + assertEquals(version2, app1.deployment(productionUsCentral1.zone()).version()); Instant triggered = app1.instanceJobs().get(productionUsCentral1).lastTriggered().get().start(); tester.clock().advance(Duration.ofHours(1)); @@ -720,14 +712,14 @@ public class DeploymentTriggerTest { assertEquals(Change.of(version1).with(revision2), app1.instance().change()); tester.triggerJobs(); app1.assertRunning(productionUsCentral1); - assertEquals(version2, app1.instance().deployments().get(productionUsCentral1.zone(main)).version()); - assertEquals(revision1, app1.deployment(productionUsCentral1.zone(main)).revision()); + assertEquals(version2, app1.instance().deployments().get(productionUsCentral1.zone()).version()); + assertEquals(revision1, app1.deployment(productionUsCentral1.zone()).revision()); assertTrue(triggered.isBefore(app1.instanceJobs().get(productionUsCentral1).lastTriggered().get().start())); // Change has a higher application version than what is deployed -- deployment should trigger. app1.timeOutUpgrade(productionUsCentral1); - assertEquals(version2, app1.deployment(productionUsCentral1.zone(main)).version()); - assertEquals(revision2, app1.deployment(productionUsCentral1.zone(main)).revision()); + assertEquals(version2, app1.deployment(productionUsCentral1.zone()).version()); + assertEquals(revision2, app1.deployment(productionUsCentral1.zone()).revision()); // Change is again strictly dominated, and us-central-1 is skipped, even though it is still failing. tester.clock().advance(Duration.ofHours(3)); // Enough time for retry @@ -760,8 +752,8 @@ public class DeploymentTriggerTest { app.runJob(systemTest).runJob(stagingTest); app.timeOutConvergence(productionEuWest1); tester.deploymentTrigger().cancelChange(app.instanceId(), PLATFORM); - assertEquals(v2, app.deployment(productionEuWest1.zone(main)).version()); - assertEquals(v1, app.deployment(productionUsEast3.zone(main)).version()); + assertEquals(v2, app.deployment(productionEuWest1.zone()).version()); + assertEquals(v1, app.deployment(productionUsEast3.zone()).version()); // New application version should run system and staging tests against both 6.1 and 6.2, in no particular order. app.submit(applicationPackage); @@ -1943,57 +1935,51 @@ public class DeploymentTriggerTest { @Test public void mixedDirectAndPipelineJobsInProduction() { - ApplicationPackage cdPackage = new ApplicationPackageBuilder().region("cd-us-east-1") - .region("cd-aws-us-east-1a") + ApplicationPackage cdPackage = new ApplicationPackageBuilder().region("us-east-3") + .region("aws-us-east-1a") .build(); - List<ZoneId> zones = List.of(ZoneId.from("test.cd-us-west-1"), - ZoneId.from("staging.cd-us-west-1"), - ZoneId.from("prod.cd-us-east-1"), - ZoneId.from("prod.cd-aws-us-east-1a")); ControllerTester wrapped = new ControllerTester(cd); - wrapped.setZones(zones) - .setRoutingMethod(zones, RoutingMethod.sharedLayer4); wrapped.upgradeSystem(Version.fromString("6.1")); wrapped.computeVersionStatus(); DeploymentTester tester = new DeploymentTester(wrapped); var app = tester.newDeploymentContext(); - app.runJob(productionCdUsEast1, cdPackage); + app.runJob(productionUsEast3, cdPackage); app.submit(cdPackage); app.runJob(systemTest); // Staging test requires unknown initial version, and is broken. - tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsEast1, "user", false, true, true); - app.runJob(productionCdUsEast1) + tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionUsEast3, "user", false, true, true); + app.runJob(productionUsEast3) .abortJob(stagingTest) // Complete failing run. .runJob(stagingTest) // Run staging-test for production zone with no prior deployment. - .runJob(productionCdAwsUsEast1a); + .runJob(productionAwsUsEast1a); // Manually deploy to east again, then upgrade the system. - app.runJob(productionCdUsEast1, cdPackage); + app.runJob(productionUsEast3, cdPackage); var version = new Version("7.1"); tester.controllerTester().upgradeSystem(version); tester.upgrader().maintain(); // System and staging tests both require unknown versions, and are broken. - tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsEast1, "user", false, true, true); - app.runJob(productionCdUsEast1) + tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionUsEast3, "user", false, true, true); + app.runJob(productionUsEast3) .triggerJobs() .jobAborted(systemTest) .jobAborted(stagingTest) .runJob(systemTest) // Run test for aws zone again. .runJob(stagingTest) // Run test for aws zone again. - .runJob(productionCdAwsUsEast1a); + .runJob(productionAwsUsEast1a); // Deploy manually again, then submit new package. - app.runJob(productionCdUsEast1, cdPackage); + app.runJob(productionUsEast3, cdPackage); app.submit(cdPackage); app.triggerJobs().runJob(systemTest); // Staging test requires unknown initial version, and is broken. - tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsEast1, "user", false, true, true); - app.runJob(productionCdUsEast1) + tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionUsEast3, "user", false, true, true); + app.runJob(productionUsEast3) .jobAborted(stagingTest) .runJob(stagingTest) - .runJob(productionCdAwsUsEast1a); + .runJob(productionAwsUsEast1a); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 15f729e7a55..0dde6bd882f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -20,7 +20,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; @@ -100,7 +99,7 @@ public class InternalStepRunnerTest { tester.triggerJobs(); tester.runner().run(); DeploymentSpec spec = tester.configServer() - .application(app.testerId().id(), JobType.stagingTest.zone(system())).get() + .application(app.testerId().id(), DeploymentContext.stagingTest.zone()).get() .applicationPackage().deploymentSpec(); assertTrue(spec.instance(app.testerId().id().instance()).isPresent()); assertEquals("domain", spec.athenzDomain().get().value()); @@ -113,24 +112,24 @@ public class InternalStepRunnerTest { "Exception to retry", "test failure"); tester.configServer().throwOnNextPrepare(exception); - tester.jobs().deploy(app.instanceId(), JobType.devUsEast1, Optional.empty(), applicationPackage()); - assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.devUsEast1).get().stepStatuses().get(Step.deployReal)); + tester.jobs().deploy(app.instanceId(), DeploymentContext.devUsEast1, Optional.empty(), applicationPackage()); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), DeploymentContext.devUsEast1).get().stepStatuses().get(Step.deployReal)); tester.configServer().throwOnNextPrepare(exception); tester.runner().run(); - assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.devUsEast1).get().stepStatuses().get(Step.deployReal)); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), DeploymentContext.devUsEast1).get().stepStatuses().get(Step.deployReal)); tester.clock().advance(Duration.ofHours(1).plusSeconds(1)); tester.configServer().throwOnNextPrepare(exception); tester.runner().run(); - assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.devUsEast1).get().stepStatuses().get(Step.deployReal)); - assertEquals(deploymentFailed, tester.jobs().last(app.instanceId(), JobType.devUsEast1).get().status()); + assertEquals(failed, tester.jobs().last(app.instanceId(), DeploymentContext.devUsEast1).get().stepStatuses().get(Step.deployReal)); + assertEquals(deploymentFailed, tester.jobs().last(app.instanceId(), DeploymentContext.devUsEast1).get().status()); } @Test public void restartsServicesAndWaitsForRestartAndReboot() { - RunId id = app.newRun(JobType.productionUsCentral1); - ZoneId zone = id.type().zone(system()); + RunId id = app.newRun(DeploymentContext.productionUsCentral1); + ZoneId zone = id.type().zone(); HostName host = tester.configServer().hostFor(instanceId, zone); tester.runner().run(); @@ -151,28 +150,28 @@ public class InternalStepRunnerTest { @Test public void waitsForEndpointsAndTimesOut() { - app.newRun(JobType.systemTest); + app.newRun(DeploymentContext.systemTest); // Tester endpoint fails to show up for staging tests, and the real deployment for system tests. - var testZone = JobType.systemTest.zone(system()); - var stagingZone = JobType.stagingTest.zone(system()); + var testZone = DeploymentContext.systemTest.zone(); + var stagingZone = DeploymentContext.stagingTest.zone(); tester.newDeploymentContext(app.testerId().id()) .deferLoadBalancerProvisioningIn(testZone.environment()); tester.newDeploymentContext(app.instanceId()) .deferLoadBalancerProvisioningIn(stagingZone.environment()); tester.runner().run(); - tester.configServer().convergeServices(app.instanceId(), JobType.stagingTest.zone(system())); + tester.configServer().convergeServices(app.instanceId(), DeploymentContext.stagingTest.zone()); tester.runner().run(); - tester.configServer().convergeServices(app.instanceId(), JobType.systemTest.zone(system())); - tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); - tester.configServer().convergeServices(app.instanceId(), JobType.stagingTest.zone(system())); - tester.configServer().convergeServices(app.testerId().id(), JobType.stagingTest.zone(system())); + tester.configServer().convergeServices(app.instanceId(), DeploymentContext.systemTest.zone()); + tester.configServer().convergeServices(app.testerId().id(), DeploymentContext.systemTest.zone()); + tester.configServer().convergeServices(app.instanceId(), DeploymentContext.stagingTest.zone()); + tester.configServer().convergeServices(app.testerId().id(), DeploymentContext.stagingTest.zone()); tester.runner().run(); tester.clock().advance(InternalStepRunner.Timeouts.of(system()).endpoint().plus(Duration.ofSeconds(1))); tester.runner().run(); - assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); + assertEquals(failed, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installReal)); } @Test @@ -180,66 +179,66 @@ public class InternalStepRunnerTest { tester.controllerTester().upgradeSystem(new Version("7.1")); tester.controllerTester().computeVersionStatus(); tester.upgrader().maintain(); - app.newRun(JobType.systemTest); + app.newRun(DeploymentContext.systemTest); // Node is down too long in system test, and no nodes go down in staging. tester.runner().run(); - tester.configServer().setVersion(tester.controller().readSystemVersion(), app.testerId().id(), JobType.systemTest.zone(system())); - tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); - tester.configServer().setVersion(tester.controller().readSystemVersion(), app.testerId().id(), JobType.stagingTest.zone(system())); - tester.configServer().convergeServices(app.testerId().id(), JobType.stagingTest.zone(system())); + tester.configServer().setVersion(tester.controller().readSystemVersion(), app.testerId().id(), DeploymentContext.systemTest.zone()); + tester.configServer().convergeServices(app.testerId().id(), DeploymentContext.systemTest.zone()); + tester.configServer().setVersion(tester.controller().readSystemVersion(), app.testerId().id(), DeploymentContext.stagingTest.zone()); + tester.configServer().convergeServices(app.testerId().id(), DeploymentContext.stagingTest.zone()); tester.runner().run(); - assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); - assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installTester)); + assertEquals(succeeded, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installTester)); + assertEquals(succeeded, tester.jobs().last(app.instanceId(), DeploymentContext.stagingTest).get().stepStatuses().get(Step.installTester)); - Node systemTestNode = tester.configServer().nodeRepository().list(JobType.systemTest.zone(system()), + Node systemTestNode = tester.configServer().nodeRepository().list(DeploymentContext.systemTest.zone(), NodeFilter.all().applications(app.instanceId())).iterator().next(); tester.clock().advance(InternalStepRunner.Timeouts.of(system()).noNodesDown().minus(Duration.ofSeconds(1))); - tester.configServer().nodeRepository().putNodes(JobType.systemTest.zone(system()), + tester.configServer().nodeRepository().putNodes(DeploymentContext.systemTest.zone(), Node.builder(systemTestNode) .serviceState(Node.ServiceState.allowedDown) .suspendedSince(tester.clock().instant()) .build()); tester.runner().run(); - assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); - assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installInitialReal)); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installReal)); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), DeploymentContext.stagingTest).get().stepStatuses().get(Step.installInitialReal)); tester.clock().advance(Duration.ofSeconds(2)); tester.runner().run(); - assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); - assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installInitialReal)); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installReal)); + assertEquals(failed, tester.jobs().last(app.instanceId(), DeploymentContext.stagingTest).get().stepStatuses().get(Step.installInitialReal)); tester.clock().advance(InternalStepRunner.Timeouts.of(system()).statelessNodesDown().minus(Duration.ofSeconds(3))); tester.runner().run(); - assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installReal)); tester.clock().advance(Duration.ofSeconds(2)); tester.runner().run(); - assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); + assertEquals(failed, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installReal)); } @Test public void startingTestsFailsIfDeploymentExpires() { - app.newRun(JobType.systemTest); + app.newRun(DeploymentContext.systemTest); tester.runner().run(); - tester.configServer().convergeServices(app.instanceId(), JobType.systemTest.zone(system())); + tester.configServer().convergeServices(app.instanceId(), DeploymentContext.systemTest.zone()); tester.runner().run(); - assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); - assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); + assertEquals(succeeded, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installReal)); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installTester)); - tester.applications().deactivate(app.instanceId(), JobType.systemTest.zone(system())); - tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); + tester.applications().deactivate(app.instanceId(), DeploymentContext.systemTest.zone()); + tester.configServer().convergeServices(app.testerId().id(), DeploymentContext.systemTest.zone()); tester.runner().run(); - assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); - assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.startTests)); - assertTrue(tester.jobs().last(app.instanceId(), JobType.systemTest).get().hasEnded()); - assertTrue(tester.jobs().last(app.instanceId(), JobType.systemTest).get().hasFailed()); + assertEquals(succeeded, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installTester)); + assertEquals(failed, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.startTests)); + assertTrue(tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().hasEnded()); + assertTrue(tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().hasFailed()); } @Test public void alternativeEndpointsAreDetected() { - var systemTestZone = JobType.systemTest.zone(system()); - var stagingZone = JobType.stagingTest.zone(system()); + var systemTestZone = DeploymentContext.systemTest.zone(); + var stagingZone = DeploymentContext.stagingTest.zone(); tester.controllerTester().zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(systemTestZone), ZoneApiMock.from(stagingZone)); var applicationPackage = new ApplicationPackageBuilder() .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service")) @@ -251,15 +250,15 @@ public class InternalStepRunnerTest { .triggerJobs(); tester.runner().run(); - assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); - assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installReal)); + assertEquals(unfinished, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installTester)); app.flushDnsUpdates(); - tester.configServer().convergeServices(app.instanceId(), JobType.systemTest.zone(system())); - tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); + tester.configServer().convergeServices(app.instanceId(), DeploymentContext.systemTest.zone()); + tester.configServer().convergeServices(app.testerId().id(), DeploymentContext.systemTest.zone()); tester.runner().run(); - assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); - assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); + assertEquals(succeeded, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installReal)); + assertEquals(succeeded, tester.jobs().last(app.instanceId(), DeploymentContext.systemTest).get().stepStatuses().get(Step.installTester)); } @Test @@ -324,7 +323,7 @@ public class InternalStepRunnerTest { RunId id = app.startSystemTestTests(); tester.runner().run(); assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.endTests)); - var testZone = JobType.systemTest.zone(system()); + var testZone = DeploymentContext.systemTest.zone(); Inspector configObject = SlimeUtils.jsonToSlime(tester.cloud().config()).get(); assertEquals(app.instanceId().serializedForm(), configObject.field("application").asString()); assertEquals(testZone.value(), configObject.field("zone").asString()); @@ -377,7 +376,7 @@ public class InternalStepRunnerTest { assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.deployTester)); tester.clock().advance(JobRunner.jobTimeout); - var testZone = JobType.systemTest.zone(tester.controller().system()); + var testZone = DeploymentContext.systemTest.zone(); tester.runner().run(); app.flushDnsUpdates(); tester.configServer().convergeServices(app.instanceId(), testZone); @@ -404,28 +403,28 @@ public class InternalStepRunnerTest { @Test public void deployToDev() { - ZoneId zone = JobType.devUsEast1.zone(system()); - tester.jobs().deploy(app.instanceId(), JobType.devUsEast1, Optional.empty(), applicationPackage()); + ZoneId zone = DeploymentContext.devUsEast1.zone(); + tester.jobs().deploy(app.instanceId(), DeploymentContext.devUsEast1, Optional.empty(), applicationPackage()); tester.runner().run(); - RunId id = tester.jobs().last(app.instanceId(), JobType.devUsEast1).get().id(); + RunId id = tester.jobs().last(app.instanceId(), DeploymentContext.devUsEast1).get().id(); assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.installReal)); Version version = new Version("7.8.9"); Future<?> concurrentDeployment = Executors.newSingleThreadExecutor().submit(() -> { - tester.jobs().deploy(app.instanceId(), JobType.devUsEast1, Optional.of(version), applicationPackage()); + tester.jobs().deploy(app.instanceId(), DeploymentContext.devUsEast1, Optional.of(version), applicationPackage()); }); while ( ! concurrentDeployment.isDone()) tester.runner().run(); - assertEquals(id.number() + 1, tester.jobs().last(app.instanceId(), JobType.devUsEast1).get().id().number()); + assertEquals(id.number() + 1, tester.jobs().last(app.instanceId(), DeploymentContext.devUsEast1).get().id().number()); ApplicationPackage otherPackage = new ApplicationPackageBuilder().region("us-central-1").build(); - tester.jobs().deploy(app.instanceId(), JobType.perfUsEast3, Optional.empty(), otherPackage); + tester.jobs().deploy(app.instanceId(), DeploymentContext.perfUsEast3, Optional.empty(), otherPackage); tester.runner().run(); // Job run order determined by JobType enum order per application. tester.configServer().convergeServices(app.instanceId(), zone); assertEquals(unfinished, tester.jobs().run(id).get().stepStatuses().get(Step.installReal)); assertEquals(applicationPackage().hash(), tester.configServer().application(app.instanceId(), zone).get().applicationPackage().hash()); - assertEquals(otherPackage.hash(), tester.configServer().application(app.instanceId(), JobType.perfUsEast3.zone(system())).get().applicationPackage().hash()); + assertEquals(otherPackage.hash(), tester.configServer().application(app.instanceId(), DeploymentContext.perfUsEast3.zone()).get().applicationPackage().hash()); tester.configServer().setVersion(version, app.instanceId(), zone); tester.runner().run(); @@ -435,7 +434,7 @@ public class InternalStepRunnerTest { @Test public void notificationIsSent() { - app.submit().failDeployment(JobType.systemTest); + app.submit().failDeployment(DeploymentContext.systemTest); MockMailer mailer = tester.controllerTester().serviceRegistry().mailer(); assertEquals(1, mailer.inbox("a@b").size()); assertEquals("Vespa application tenant.application: System test failing due to system error", @@ -445,12 +444,12 @@ public class InternalStepRunnerTest { mailer.inbox("b@a").get(0).subject()); // Re-run failing causes no additional email to be sent. - app.failDeployment(JobType.systemTest); + app.failDeployment(DeploymentContext.systemTest); assertEquals(1, mailer.inbox("a@b").size()); assertEquals(1, mailer.inbox("b@a").size()); // Failure with new package causes new email to be sent. - app.submit().failDeployment(JobType.systemTest); + app.submit().failDeployment(DeploymentContext.systemTest); assertEquals(2, mailer.inbox("a@b").size()); assertEquals(2, mailer.inbox("b@a").size()); } @@ -506,7 +505,7 @@ public class InternalStepRunnerTest { tester = new DeploymentTester(wrapped); tester.configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.values()); app = tester.newDeploymentContext(); - RunId id = app.newRun(JobType.systemTest); + RunId id = app.newRun(DeploymentContext.systemTest); tester.configServer().throwOnPrepare(instanceId -> { if (instanceId.instance().isTester()) throw new ConfigServerException(ConfigServerException.ErrorCode.PARENT_HOST_NOT_READY, "provisioning", "deploy tester"); @@ -518,7 +517,7 @@ public class InternalStepRunnerTest { List<X509Certificate> oldTrusted = new ArrayList<>(DeploymentContext.publicApplicationPackage().trustedCertificates()); X509Certificate oldCert = tester.jobs().run(id).get().testerCertificate().get(); oldTrusted.add(oldCert); - assertEquals(oldTrusted, tester.configServer().application(app.instanceId(), id.type().zone(system())).get().applicationPackage().trustedCertificates()); + assertEquals(oldTrusted, tester.configServer().application(app.instanceId(), id.type().zone()).get().applicationPackage().trustedCertificates()); tester.configServer().throwOnNextPrepare(null); tester.runner().run(); @@ -528,27 +527,19 @@ public class InternalStepRunnerTest { List<X509Certificate> newTrusted = new ArrayList<>(DeploymentContext.publicApplicationPackage().trustedCertificates()); X509Certificate newCert = tester.jobs().run(id).get().testerCertificate().get(); newTrusted.add(newCert); - assertEquals(newTrusted, tester.configServer().application(app.instanceId(), id.type().zone(system())).get().applicationPackage().trustedCertificates()); + assertEquals(newTrusted, tester.configServer().application(app.instanceId(), id.type().zone()).get().applicationPackage().trustedCertificates()); assertNotEquals(oldCert, newCert); } @Test public void certificateTimeoutAbortsJob() { - List<ZoneApiMock> zones = List.of(ZoneApiMock.fromId("test.aws-us-east-1c"), - ZoneApiMock.fromId("staging.aws-us-east-1c"), - ZoneApiMock.fromId("prod.aws-us-east-1c")); - ControllerTester wrapped = new ControllerTester(SystemName.Public); - wrapped.zoneRegistry() - .setZones(zones) - .setRoutingMethod(zones, RoutingMethod.exclusive); - tester = new DeploymentTester(wrapped); - tester.configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.values()); + tester = new DeploymentTester(new ControllerTester(SystemName.Public)); app = tester.newDeploymentContext(); RunId id = app.startSystemTestTests(); List<X509Certificate> trusted = new ArrayList<>(DeploymentContext.publicApplicationPackage().trustedCertificates()); trusted.add(tester.jobs().run(id).get().testerCertificate().get()); - assertEquals(trusted, tester.configServer().application(app.instanceId(), id.type().zone(system())).get().applicationPackage().trustedCertificates()); + assertEquals(trusted, tester.configServer().application(app.instanceId(), id.type().zone()).get().applicationPackage().trustedCertificates()); tester.clock().advance(InternalStepRunner.Timeouts.of(system()).testerCertificate().plus(Duration.ofSeconds(1))); tester.runner().run(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java index 61eca05cc67..59ee8cc6eae 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; import org.junit.Test; @@ -28,9 +27,9 @@ public class TestConfigSerializerTest { @Test public void testConfig() throws IOException { - ZoneId zone = JobType.systemTest.zone(SystemName.PublicCd); + ZoneId zone = DeploymentContext.systemTest.zone(); byte[] json = new TestConfigSerializer(SystemName.PublicCd).configJson(instanceId, - JobType.systemTest, + DeploymentContext.systemTest, true, Map.of(zone, List.of(Endpoint.of(ApplicationId.defaultId()) .target(EndpointId.of("ai"), ClusterSpec.Id.from("qrs"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 1d8dba321a2..a4b17239626 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -55,11 +55,12 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry public ZoneRegistryMock(SystemName system) { this.system = system; if (system.isPublic()) { - this.zones = List.of(ZoneApiMock.fromId("test.aws-us-east-1c"), - ZoneApiMock.fromId("staging.aws-us-east-1c"), + this.zones = List.of(ZoneApiMock.fromId("test.us-east-1"), + ZoneApiMock.fromId("staging.us-east-3"), ZoneApiMock.fromId("prod.aws-us-east-1c"), - ZoneApiMock.fromId("prod.aws-eu-west-1a")); - setRoutingMethod(this.zones, RoutingMethod.exclusive); + ZoneApiMock.fromId("prod.aws-eu-west-1a"), + ZoneApiMock.fromId("dev.aws-us-east-1c")); + setRoutingMethod(this.zones, RoutingMethod.exclusive); } else { this.zones = List.of(ZoneApiMock.fromId("test.us-east-1"), ZoneApiMock.fromId("staging.us-east-3"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java index 5f3c27ff060..c1d9c03819d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java @@ -83,6 +83,7 @@ public class ArchiveUriUpdaterTest { } private void deploy(DeploymentContext application, ZoneId zone) { - application.runJob(JobType.from(SystemName.Public, zone).orElseThrow(), new ApplicationPackage(new byte[0]), Version.fromString("7.1")); + application.runJob(JobType.deploymentTo(zone), new ApplicationPackage(new byte[0]), Version.fromString("7.1")); } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java index 7f2799b6f58..f37a5a6893d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertEquals; * @author ogronnesby */ public class CloudTrialExpirerTest { + private final ControllerTester tester = new ControllerTester(SystemName.PublicCd); private final DeploymentTester deploymentTester = new DeploymentTester(tester); private final CloudTrialExpirer expirer = new CloudTrialExpirer(tester.controller(), Duration.ofMinutes(5)); @@ -66,7 +67,7 @@ public class CloudTrialExpirerTest { @Test public void keep_inactive_trial_tenants_with_deployments() { registerTenant("with-deployments", "trial", Duration.ofDays(30)); - registerDeployment("with-deployments", "my-app", "default", "aws-us-east-1c"); + registerDeployment("with-deployments", "my-app", "default"); expirer.maintain(); assertPlan("with-deployments", "trial"); } @@ -78,17 +79,12 @@ public class CloudTrialExpirerTest { tester.controller().tenants().updateLastLogin(name, List.of(LastLoginInfo.UserLevel.user), tester.controller().clock().instant().minus(timeSinceLastLogin)); } - private void registerDeployment(String tenantName, String appName, String instanceName, String regionName) { - var zone = ZoneApiMock.newBuilder() - .withSystem(tester.zoneRegistry().system()) - .withId("prod." + regionName) - .build(); - tester.zoneRegistry().setZones(ZoneApiMock.fromId("test.aws-us-east-1c"), ZoneApiMock.fromId("staging.aws-us-east-1c"), zone); + private void registerDeployment(String tenantName, String appName, String instanceName) { var app = tester.createApplication(tenantName, appName, instanceName); var ctx = deploymentTester.newDeploymentContext(tenantName, appName, instanceName); var pkg = new ApplicationPackageBuilder() .instances("default") - .region(regionName) + .region("aws-us-east-1c") .trustDefaultCertificate() .build(); ctx.submit(pkg).deploy(); @@ -97,4 +93,5 @@ public class CloudTrialExpirerTest { private void assertPlan(String tenant, String planId) { assertEquals(planId, tester.serviceRegistry().billingController().getPlan(TenantName.from(tenant)).value()); } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java index b39b791bb57..10a8bb79c2e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirerTest.java @@ -5,10 +5,10 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; @@ -40,7 +40,7 @@ public class DeploymentExpirerTest { .build(); // Deploy dev - devApp.runJob(JobType.devUsEast1, appPackage); + devApp.runJob(DeploymentContext.devUsEast1, appPackage); // Deploy prod prodApp.submit(appPackage).deploy(); @@ -55,8 +55,8 @@ public class DeploymentExpirerTest { // Deploy dev unsuccessfully a few days before expiry tester.clock().advance(Duration.ofDays(12)); tester.configServer().throwOnNextPrepare(new RuntimeException(getClass().getSimpleName())); - tester.jobs().deploy(devApp.instanceId(), JobType.devUsEast1, Optional.empty(), appPackage); - Run lastRun = tester.jobs().last(devApp.instanceId(), JobType.devUsEast1).get(); + tester.jobs().deploy(devApp.instanceId(), DeploymentContext.devUsEast1, Optional.empty(), appPackage); + Run lastRun = tester.jobs().last(devApp.instanceId(), DeploymentContext.devUsEast1).get(); assertSame(RunStatus.error, lastRun.status()); Deployment deployment = tester.applications().requireInstance(devApp.instanceId()) .deployments().get(devZone); @@ -72,7 +72,7 @@ public class DeploymentExpirerTest { // Dev application expires when enough time has passed since most recent attempt // Redeployments done by DeploymentUpgrader do not affect this tester.clock().advance(Duration.ofDays(12).plus(Duration.ofSeconds(1))); - tester.jobs().start(devApp.instanceId(), JobType.devUsEast1, lastRun.versions(), true, Optional.of("upgrade")); + tester.jobs().start(devApp.instanceId(), DeploymentContext.devUsEast1, lastRun.versions(), true, Optional.of("upgrade")); expirer.maintain(); assertEquals(0, permanentDeployments(devApp.instance())); assertEquals(1, permanentDeployments(prodApp.instance())); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java index 812db107a26..637a8832533 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java @@ -21,9 +21,9 @@ import java.util.HashMap; import java.util.Map; import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy.canary; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; import static com.yahoo.vespa.hosted.controller.maintenance.DeploymentIssueReporter.maxFailureAge; import static com.yahoo.vespa.hosted.controller.maintenance.DeploymentIssueReporter.maxInactivity; import static com.yahoo.vespa.hosted.controller.maintenance.DeploymentIssueReporter.upgradeGracePeriod; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java index 0b47cab3ac8..112519bb717 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java @@ -8,10 +8,10 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import org.junit.Test; @@ -37,7 +37,7 @@ public class DeploymentMetricsMaintainerTest { @Test public void updates_metrics() { var application = tester.newDeploymentContext(); - application.runJob(JobType.devUsEast1, new ApplicationPackage(new byte[0]), Version.fromString("7.1")); + application.runJob(DeploymentContext.devUsEast1, new ApplicationPackage(new byte[0]), Version.fromString("7.1")); DeploymentMetricsMaintainer maintainer = maintainer(tester.controller()); Supplier<Application> app = application::application; @@ -51,7 +51,7 @@ public class DeploymentMetricsMaintainerTest { assertFalse("Never received any writes", deployment.get().activity().lastWritten().isPresent()); // Metrics are gathered and saved to application - application.runJob(JobType.devUsEast1, new ApplicationPackage(new byte[0]), Version.fromString("7.5.5")); + application.runJob(DeploymentContext.devUsEast1, new ApplicationPackage(new byte[0]), Version.fromString("7.5.5")); var metrics0 = Map.of(ClusterMetrics.QUERIES_PER_SECOND, 1D, ClusterMetrics.FEED_PER_SECOND, 2D, ClusterMetrics.DOCUMENT_COUNT, 3D, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java index 07a23decd50..653ad2bb08a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgraderTest.java @@ -13,8 +13,8 @@ import org.junit.Test; import java.time.Duration; import java.time.Instant; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.devUsEast1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.devUsEast1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1; import static com.yahoo.vespa.hosted.controller.maintenance.DeploymentUpgrader.mostLikelyWeeHour; import static java.time.temporal.ChronoUnit.MILLIS; import static org.junit.Assert.assertEquals; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java index 3c80f9df843..6bcb4284a14 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java @@ -15,9 +15,9 @@ import java.time.Duration; import java.util.List; import java.util.Optional; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 610d8d4ca9a..19971a0ee5b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -3,12 +3,10 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; @@ -43,8 +41,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; import java.util.stream.Collectors; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 658fbccd660..64e3a95605a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -10,8 +10,11 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.vespa.athenz.utils.AthenzIdentities; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock; +import com.yahoo.vespa.hosted.controller.api.integration.athenz.ZmsClientMock; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; @@ -37,9 +40,9 @@ import java.util.function.UnaryOperator; import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -50,6 +53,7 @@ import static org.junit.Assert.assertTrue; public class MetricsReporterTest { private final MetricsMock metrics = new MetricsMock(); + private final ZmsClientMock zmsClient = new ZmsClientMock(new AthenzDbMock(), AthenzIdentities.from("mock.identity")); @Test public void audit_log_metric() { @@ -553,6 +557,19 @@ public class MetricsReporterTest { assertEquals("Upgrade is overdue measure relative to window 3", Duration.ofHours(34).plusMinutes(30), metric.get()); } + @Test + public void zms_quota_metrics() { + var tester = new ControllerTester(); + var reporter = createReporter(tester.controller()); + reporter.maintain(); + + assertEquals(0.1, metrics.getMetric(d -> "subdomains".equals(d.get("resourceType")), MetricsReporter.ZMS_QUOTA_USAGE).get()); + assertEquals(0.2, metrics.getMetric(d -> "roles".equals(d.get("resourceType")), MetricsReporter.ZMS_QUOTA_USAGE).get()); + assertEquals(0.3, metrics.getMetric(d -> "policies".equals(d.get("resourceType")), MetricsReporter.ZMS_QUOTA_USAGE).get()); + assertEquals(0.4, metrics.getMetric(d -> "services".equals(d.get("resourceType")), MetricsReporter.ZMS_QUOTA_USAGE).get()); + assertEquals(0.5, metrics.getMetric(d -> "groups".equals(d.get("resourceType")), MetricsReporter.ZMS_QUOTA_USAGE).get()); + } + private void assertNodeCount(String metric, int n, Version version) { long nodeCount = metrics.getMetric((dimensions) -> version.toFullString().equals(dimensions.get("currentVersion")), metric) .stream() @@ -656,7 +673,7 @@ public class MetricsReporterTest { } private MetricsReporter createReporter(Controller controller) { - return new MetricsReporter(controller, metrics); + return new MetricsReporter(controller, metrics, zmsClient); } private static String appDimension(ApplicationId id) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainerTest.java index ccb2b6ebb74..bebecf8b52b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainerTest.java @@ -4,9 +4,9 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.RetriggerEntry; import org.junit.Test; @@ -34,11 +34,11 @@ public class RetriggerMaintainerTest { .build(); // Deploy app - devApp.runJob(JobType.devUsEast1, appPackage); + devApp.runJob(DeploymentContext.devUsEast1, appPackage); devApp.completeRollout(); // Trigger a run (to simulate a running job) - tester.deploymentTrigger().reTrigger(applicationId, JobType.devUsEast1, null); + tester.deploymentTrigger().reTrigger(applicationId, DeploymentContext.devUsEast1, null); // Add a job to the queue tester.deploymentTrigger().reTriggerOrAddToQueue(devApp.deploymentIdIn(ZoneId.from("dev", "us-east-1")), null); @@ -48,7 +48,7 @@ public class RetriggerMaintainerTest { assertEquals(1, retriggerEntries.size()); // Adding to queue triggers abort - devApp.jobAborted(JobType.devUsEast1); + devApp.jobAborted(DeploymentContext.devUsEast1); assertEquals(0, tester.jobs().active(applicationId).size()); // The maintainer runs and will actually trigger dev us-east, but keeps the entry in queue to verify it was actually run @@ -58,7 +58,7 @@ public class RetriggerMaintainerTest { assertEquals(1, tester.jobs().active(applicationId).size()); // Run outstanding jobs - devApp.runJob(JobType.devUsEast1); + devApp.runJob(DeploymentContext.devUsEast1); assertEquals(0, tester.jobs().active(applicationId).size()); // Run maintainer again, should find that the job has already run successfully and will remove the entry. diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java index 81daf0cbcfe..7026d975010 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainerTest.java @@ -5,9 +5,9 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockRoleService; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import org.junit.Test; @@ -35,11 +35,11 @@ public class TenantRoleMaintainerTest { .build(); // Deploy dev apps - devAppTenant1.runJob(JobType.devUsEast1, appPackage); - devAppTenant2.runJob(JobType.devUsEast1, appPackage); + devAppTenant1.runJob(DeploymentContext.devUsEast1, appPackage); + devAppTenant2.runJob(DeploymentContext.devUsEast1, appPackage); // Deploy perf apps - perfAppTenant1.runJob(JobType.perfUsEast3, appPackage); + perfAppTenant1.runJob(DeploymentContext.perfUsEast3, appPackage); // Deploy prod prodAppTenant2.submit(appPackage).deploy(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdaterTest.java index 08af46d8d33..c59155cb162 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/TrafficShareUpdaterTest.java @@ -6,8 +6,8 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; import org.junit.Test; @@ -33,7 +33,7 @@ public class TrafficShareUpdaterTest { ZoneId prod1 = ZoneId.from("prod", "ap-northeast-1"); ZoneId prod2 = ZoneId.from("prod", "us-east-3"); ZoneId prod3 = ZoneId.from("prod", "us-west-1"); - application.runJob(JobType.productionApNortheast1, new ApplicationPackage(new byte[0]), Version.fromString("7.1")); + application.runJob(DeploymentContext.productionApNortheast1, new ApplicationPackage(new byte[0]), Version.fromString("7.1")); // Single zone setQpsMetric(50.0, application.application().id().defaultInstance(), prod1, tester); @@ -42,7 +42,7 @@ public class TrafficShareUpdaterTest { assertTrafficFraction(1.0, 1.0, application.instanceId(), prod1, tester); // Two zones - application.runJob(JobType.productionUsEast3, new ApplicationPackage(new byte[0]), Version.fromString("7.1")); + application.runJob(DeploymentContext.productionUsEast3, new ApplicationPackage(new byte[0]), Version.fromString("7.1")); // - one cold setQpsMetric(50.0, application.application().id().defaultInstance(), prod1, tester); setQpsMetric(0.0, application.application().id().defaultInstance(), prod2, tester); @@ -59,7 +59,7 @@ public class TrafficShareUpdaterTest { assertTrafficFraction(0.47, 1.0, application.instanceId(), prod2, tester); // Three zones - application.runJob(JobType.productionUsWest1, new ApplicationPackage(new byte[0]), Version.fromString("7.1")); + application.runJob(DeploymentContext.productionUsWest1, new ApplicationPackage(new byte[0]), Version.fromString("7.1")); // - one cold setQpsMetric(53.0, application.application().id().defaultInstance(), prod1, tester); setQpsMetric(47.0, application.application().id().defaultInstance(), prod2, tester); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index ef4bfdb568e..185c1e8c891 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -28,12 +28,12 @@ import java.util.OptionalInt; import java.util.Set; import java.util.stream.Collectors; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.devUsEast1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsCentral1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsEast3; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.devUsEast1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsCentral1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsEast3; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.ALL; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.PIN; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.PLATFORM; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java index 75b9b108377..4fbe21f11fb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java @@ -11,10 +11,10 @@ import com.yahoo.path.Path; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.notify.Notifier; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; @@ -65,7 +65,7 @@ public class NotificationsDbTest { notification(1201, Type.deployment, Level.error, NotificationSource.from(ApplicationId.from(tenant.value(), "app2", "instance2")), "instance msg"), notification(1301, Type.deployment, Level.warning, NotificationSource.from(new DeploymentId(ApplicationId.from(tenant.value(), "app2", "instance2"), ZoneId.from("prod", "us-north-2"))), "deployment msg"), notification(1401, Type.feedBlock, Level.error, NotificationSource.from(new DeploymentId(ApplicationId.from(tenant.value(), "app1", "instance1"), ZoneId.from("dev", "us-south-1")), ClusterSpec.Id.from("cluster1")), "cluster msg"), - notification(1501, Type.deployment, Level.warning, NotificationSource.from(new RunId(ApplicationId.from(tenant.value(), "app1", "instance1"), JobType.devUsEast1, 4)), "run id msg")); + notification(1501, Type.deployment, Level.warning, NotificationSource.from(new RunId(ApplicationId.from(tenant.value(), "app1", "instance1"), DeploymentContext.devUsEast1, 4)), "run id msg")); private final ManualClock clock = new ManualClock(Instant.ofEpochSecond(12345)); private final MockCuratorDb curatorDb = new MockCuratorDb(SystemName.Public); @@ -78,8 +78,8 @@ public class NotificationsDbTest { assertEquals(notificationIndices(0, 1, 2, 3), notificationsDb.listNotifications(NotificationSource.from(tenant), true)); assertEquals(notificationIndices(2, 3), notificationsDb.listNotifications(NotificationSource.from(TenantAndApplicationId.from(tenant.value(), "app2")), false)); assertEquals(notificationIndices(4, 5), notificationsDb.listNotifications(NotificationSource.from(ApplicationId.from(tenant.value(), "app1", "instance1")), false)); - assertEquals(notificationIndices(5), notificationsDb.listNotifications(NotificationSource.from(new RunId(ApplicationId.from(tenant.value(), "app1", "instance1"), JobType.devUsEast1, 5)), false)); - assertEquals(List.of(), notificationsDb.listNotifications(NotificationSource.from(new RunId(ApplicationId.from(tenant.value(), "app1", "instance1"), JobType.productionUsEast3, 4)), false)); + assertEquals(notificationIndices(5), notificationsDb.listNotifications(NotificationSource.from(new RunId(ApplicationId.from(tenant.value(), "app1", "instance1"), DeploymentContext.devUsEast1, 5)), false)); + assertEquals(List.of(), notificationsDb.listNotifications(NotificationSource.from(new RunId(ApplicationId.from(tenant.value(), "app1", "instance1"), DeploymentContext.productionUsEast3, 4)), false)); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 5941812d66d..e60325a140a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -13,7 +13,6 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; @@ -25,6 +24,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentActivity; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.QuotaUsage; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.RevisionHistory; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; @@ -55,7 +55,7 @@ import static org.junit.Assert.assertEquals; public class ApplicationSerializerTest { - private static final ApplicationSerializer APPLICATION_SERIALIZER = new ApplicationSerializer(SystemName.main); + private static final ApplicationSerializer APPLICATION_SERIALIZER = new ApplicationSerializer(); private static final Path testData = Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/"); private static final ZoneId zone1 = ZoneId.from("prod", "us-west-1"); private static final ZoneId zone2 = ZoneId.from("prod", "us-east-3"); @@ -102,7 +102,7 @@ public class ApplicationSerializerTest { 3); assertEquals("https://github/org/repo/tree/commit1", applicationVersion1.sourceUrl().get()); - ApplicationVersion applicationVersion2 = ApplicationVersion.from(RevisionId.forDevelopment(31, new JobId(id1, JobType.productionUsEast3)), + ApplicationVersion applicationVersion2 = ApplicationVersion.from(RevisionId.forDevelopment(31, new JobId(id1, DeploymentContext.productionUsEast3)), new SourceRevision("repo1", "branch1", "commit1"), "a@b", Version.fromString("6.3.1"), Instant.ofEpochMilli(496)); @@ -125,10 +125,10 @@ public class ApplicationSerializerTest { Instant.ofEpochMilli(42)))); RevisionHistory revisions = RevisionHistory.ofRevisions(List.of(applicationVersion1), - Map.of(new JobId(id1, JobType.productionUsEast3), List.of(applicationVersion2))); + Map.of(new JobId(id1, DeploymentContext.productionUsEast3), List.of(applicationVersion2))); List<Instance> instances = List.of(new Instance(id1, deployments, - Map.of(JobType.systemTest, Instant.ofEpochMilli(333)), + Map.of(DeploymentContext.systemTest, Instant.ofEpochMilli(333)), List.of(AssignedRotation.fromStrings("foo", "default", "my-rotation", Set.of("us-west-1"))), rotationStatus, Change.of(new Version("6.1"))), @@ -188,10 +188,10 @@ public class ApplicationSerializerTest { assertEquals(original.require(id1.instance()).deployments().get(zone1), serialized.require(id1.instance()).deployments().get(zone1)); assertEquals(original.require(id1.instance()).deployments().get(zone2), serialized.require(id1.instance()).deployments().get(zone2)); - assertEquals(original.require(id1.instance()).jobPause(JobType.systemTest), - serialized.require(id1.instance()).jobPause(JobType.systemTest)); - assertEquals(original.require(id1.instance()).jobPause(JobType.stagingTest), - serialized.require(id1.instance()).jobPause(JobType.stagingTest)); + assertEquals(original.require(id1.instance()).jobPause(DeploymentContext.systemTest), + serialized.require(id1.instance()).jobPause(DeploymentContext.systemTest)); + assertEquals(original.require(id1.instance()).jobPause(DeploymentContext.stagingTest), + serialized.require(id1.instance()).jobPause(DeploymentContext.stagingTest)); assertEquals(original.ownershipIssueId(), serialized.ownershipIssueId()); assertEquals(original.owner(), serialized.owner()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java index e13c598f2a9..0f7f97d333a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStoreTest.java @@ -5,9 +5,9 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.RunDataStore; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockRunDataStore; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.RunLog; import com.yahoo.vespa.hosted.controller.deployment.Step; import org.junit.Test; @@ -33,7 +33,7 @@ public class BufferedLogStoreTest { RunDataStore store = new MockRunDataStore(); BufferedLogStore logs = new BufferedLogStore(chunkSize, chunkSize * maxChunks, buffer, store); RunId id = new RunId(ApplicationId.from("tenant", "application", "instance"), - JobType.productionUsWest1, + DeploymentContext.productionUsWest1, 123); byte[] manyBytes = new byte[chunkSize / 2 + 1]; // One fits, and two (over-)fills. diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializerTest.java index 0c8a773a132..1b82f622773 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializerTest.java @@ -5,9 +5,9 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import org.junit.Test; @@ -26,7 +26,7 @@ public class NotificationsSerializerTest { @Test public void serialization_test() throws IOException { - NotificationsSerializer serializer = new NotificationsSerializer(main); + NotificationsSerializer serializer = new NotificationsSerializer(); TenantName tenantName = TenantName.from("tenant1"); List<Notification> notifications = List.of( new Notification(Instant.ofEpochSecond(1234), @@ -37,7 +37,7 @@ public class NotificationsSerializerTest { new Notification(Instant.ofEpochSecond(2345), Notification.Type.deployment, Notification.Level.error, - NotificationSource.from(new RunId(ApplicationId.from(tenantName.value(), "app1", "instance1"), JobType.systemTest, 12)), + NotificationSource.from(new RunId(ApplicationId.from(tenantName.value(), "app1", "instance1"), DeploymentContext.systemTest, 12)), List.of("Failed to deploy: Node allocation failure"))); Slime serialized = serializer.toSlime(notifications); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index a3b7932197b..fd0ea50e50b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -6,12 +6,10 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.security.X509CertificateUtils; import com.yahoo.slime.SlimeUtils; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.JobProfile; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; @@ -20,16 +18,12 @@ import com.yahoo.vespa.hosted.controller.deployment.StepInfo; import org.junit.Test; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.time.Instant; -import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Optional; -import java.util.OptionalLong; import static com.yahoo.config.provision.SystemName.main; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; @@ -59,10 +53,10 @@ import static org.junit.Assert.assertTrue; public class RunSerializerTest { - private static final RunSerializer serializer = new RunSerializer(main); + private static final RunSerializer serializer = new RunSerializer(); private static final Path runFile = Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json"); private static final RunId id = new RunId(ApplicationId.from("tenant", "application", "default"), - JobType.productionUsEast3, + DeploymentContext.productionUsEast3, 112358); private static final Instant start = Instant.parse("2007-12-03T10:15:30.00Z"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json index 29f748d5408..ec36f52c23a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/complete-application.json @@ -294,7 +294,7 @@ "deploymentJobs": { "jobStatus": [ { - "jobType": "staging-test", + "jobType": "staging.zone", "pausedUntil": 321 } ] diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json index 85881fbfdbc..1216bcefab6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json @@ -1,7 +1,7 @@ [ { "id": "tenant:application:default", - "type": "production-us-east-3", + "type": "prod.us-east-3", "number": 112358, "start": 1196676930000, "sleepUntil": 1196676930100, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java index ede753f9bf2..33b7500ceac 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -402,7 +402,7 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { .build(); new ControllerTester(tester).upgradeSystem(new Version("6.1")); tester.controller().jobController().deploy(ApplicationId.from("scoober", "albums", "default"), - JobType.productionAwsUsEast1c, + JobType.prod("aws-us-east-1c"), Optional.empty(), applicationPackage); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 992ef59d9a5..6bfbb044944 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -41,7 +41,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationActio import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzDbMock; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; @@ -52,6 +51,7 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; @@ -258,7 +258,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .data(entity) .userIdentity(HOSTED_VESPA_OPERATOR), "{\"message\":\"Deployment started in run 1 of production-us-east-3 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1}"); - app1.runJob(JobType.productionUsEast3); + app1.runJob(DeploymentContext.productionUsEast3); tester.controller().applications().deactivate(app1.instanceId(), ZoneId.from("prod", "us-east-3")); // POST (deploy) an application to start a manual deployment to dev @@ -266,13 +266,13 @@ public class ApplicationApiTest extends ControllerContainerTest { .data(entity) .userIdentity(USER_ID), "{\"message\":\"Deployment started in run 1 of dev-us-east-1 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1}"); - app1.runJob(JobType.devUsEast1); + app1.runJob(DeploymentContext.devUsEast1); // POST (deploy) a job to restart a manual deployment to dev tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/dev-us-east-1", POST) .userIdentity(USER_ID), "{\"message\":\"Triggered dev-us-east-1 for tenant1.application1.instance1\"}"); - app1.runJob(JobType.devUsEast1); + app1.runJob(DeploymentContext.devUsEast1); // GET dev application package tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/dev-us-east-1/package", GET) @@ -331,7 +331,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .data(createApplicationSubmissionData(applicationPackageInstance1, 123)), "{\"message\":\"application build 1, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); - app1.runJob(JobType.systemTest).runJob(JobType.stagingTest).runJob(JobType.productionUsCentral1); + app1.runJob(DeploymentContext.systemTest).runJob(DeploymentContext.stagingTest).runJob(DeploymentContext.productionUsCentral1); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .withoutAthenzIdentity() @@ -367,7 +367,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .data("{ \"skipTests\": true, \"skipRevision\": true, \"skipUpgrade\": true }") .userIdentity(USER_ID), "{\"message\":\"Triggered production-us-west-1 for tenant2.application2.instance1, without revision and platform upgrade\"}"); - app2.runJob(JobType.productionUsWest1); + app2.runJob(DeploymentContext.productionUsWest1); // POST a re-triggering to force a production job to start with previous parameters tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2/instance/instance1/job/production-us-west-1", POST) @@ -729,11 +729,11 @@ public class ApplicationApiTest extends ControllerContainerTest { // Setup for test config tests tester.controller().jobController().deploy(ApplicationId.from("tenant1", "application1", "default"), - JobType.productionUsCentral1, + DeploymentContext.productionUsCentral1, Optional.empty(), applicationPackageDefault); tester.controller().jobController().deploy(ApplicationId.from("tenant1", "application1", "my-user"), - JobType.devUsEast1, + DeploymentContext.devUsEast1, Optional.empty(), applicationPackageDefault); @@ -1092,7 +1092,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // Create tenant and deploy var app = deploymentTester.newDeploymentContext("tenant1", "application1", "instance1"); app.submit(applicationPackage).deploy(); - tester.controller().jobController().deploy(app.instanceId(), JobType.devUsEast1, Optional.empty(), applicationPackage); + tester.controller().jobController().deploy(app.instanceId(), DeploymentContext.devUsEast1, Optional.empty(), applicationPackage); assertEquals(Set.of(ZoneId.from("prod.us-west-1"), ZoneId.from("prod.us-east-3"), ZoneId.from("prod.eu-west-1"), ZoneId.from("dev.us-east-1")), app.instance().deployments().keySet()); @@ -1616,7 +1616,7 @@ public class ApplicationApiTest extends ControllerContainerTest { assertEquals(1, activeGrants.size()); // Adding grant should trigger job - app.assertRunning(JobType.productionUsWest1); + app.assertRunning(DeploymentContext.productionUsWest1); // DELETE removes access String disallowedResponse = grantResponse @@ -1628,7 +1628,7 @@ public class ApplicationApiTest extends ControllerContainerTest { ); // Revoking access should trigger job - app.assertRunning(JobType.productionUsWest1); + app.assertRunning(DeploymentContext.productionUsWest1); // Should be no available grant activeGrants = tester.controller().supportAccess().activeGrantsFor(new DeploymentId(ApplicationId.fromSerializedForm("tenant1:application1:instance1"), zone)); @@ -1836,7 +1836,7 @@ public class ApplicationApiTest extends ControllerContainerTest { Notification.Level.warning, "Something something deprecated..."); tester.controller().notificationsDb().setNotification( - NotificationSource.from(new RunId(ApplicationId.from(tenantName.value(), "app2", "instance1"), JobType.systemTest, 12)), + NotificationSource.from(new RunId(ApplicationId.from(tenantName.value(), "app2", "instance1"), DeploymentContext.systemTest, 12)), Notification.Type.deployment, Notification.Level.error, "Failed to deploy: Node allocation failure"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index ee91787a36e..ac8bdafa2bd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -6,11 +6,11 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import org.junit.Test; @@ -24,14 +24,14 @@ import java.time.Instant; import java.util.Optional; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.INVALID_APPLICATION_PACKAGE; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.devAwsUsEast2a; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.devUsEast1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsCentral1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsEast3; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.testUsCentral1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.devAwsUsEast2a; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.devUsEast1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsCentral1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsEast3; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.testUsCentral1; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.applicationPackage; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; @@ -89,9 +89,9 @@ public class JobControllerApiHandlerHelperTest { tester.clock().advance(Duration.ofHours(4).plusSeconds(1)); tester.runner().run(); assertEquals(installationFailed, tester.jobs().last(app.instanceId(), productionUsWest1).get().status()); - assertEquals(revision2, app.deployment(productionUsCentral1.zone(tester.controller().system())).revision()); - assertEquals(revision1, app.deployment(productionUsEast3.zone(tester.controller().system())).revision()); - assertEquals(revision2, app.deployment(productionUsWest1.zone(tester.controller().system())).revision()); + assertEquals(revision2, app.deployment(productionUsCentral1.zone()).revision()); + assertEquals(revision1, app.deployment(productionUsEast3.zone()).revision()); + assertEquals(revision2, app.deployment(productionUsWest1.zone()).revision()); tester.clock().advance(Duration.ofMillis(1000)); @@ -104,7 +104,7 @@ public class JobControllerApiHandlerHelperTest { assertEquals(running, tester.jobs().last(app.instanceId(), stagingTest).get().status()); // Staging deployment expires and the job fails, and is immediately retried. - tester.controller().applications().deactivate(app.instanceId(), stagingTest.zone(tester.controller().system())); + tester.controller().applications().deactivate(app.instanceId(), stagingTest.zone()); tester.runner().run(); assertEquals(installationFailed, tester.jobs().last(app.instanceId(), stagingTest).get().status()); @@ -113,7 +113,7 @@ public class JobControllerApiHandlerHelperTest { tester.triggerJobs(); tester.runner().run(); assertEquals(running, tester.jobs().last(app.instanceId(), stagingTest).get().status()); - tester.controller().applications().deactivate(app.instanceId(), stagingTest.zone(tester.controller().system())); + tester.controller().applications().deactivate(app.instanceId(), stagingTest.zone()); tester.runner().run(); assertEquals(installationFailed, tester.jobs().last(app.instanceId(), stagingTest).get().status()); @@ -149,8 +149,8 @@ public class JobControllerApiHandlerHelperTest { var app = tester.newDeploymentContext(); tester.clock().setInstant(Instant.EPOCH); - ZoneId zone = JobType.devUsEast1.zone(tester.controller().system()); - tester.jobs().deploy(app.instanceId(), JobType.devUsEast1, Optional.empty(), applicationPackage()); + ZoneId zone = DeploymentContext.devUsEast1.zone(); + tester.jobs().deploy(app.instanceId(), DeploymentContext.devUsEast1, Optional.empty(), applicationPackage()); tester.configServer().setLogStream(() -> "1554970337.935104\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n"); assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(app.instanceId(), devUsEast1).get().id(), null), "dev-us-east-1-log-first-part.json"); @@ -159,7 +159,7 @@ public class JobControllerApiHandlerHelperTest { tester.runner().run(); assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(app.instanceId(), devUsEast1).get().id(), "8"), "dev-us-east-1-log-second-part.json"); - tester.jobs().deploy(app.instanceId(), JobType.devUsEast1, Optional.empty(), applicationPackage()); + tester.jobs().deploy(app.instanceId(), DeploymentContext.devUsEast1, Optional.empty(), applicationPackage()); assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), app.instanceId(), URI.create("https://some.url:43/root")), "dev-overview.json"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java index c195b623c11..8db6bdf9a4a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java @@ -2,9 +2,9 @@ package com.yahoo.vespa.hosted.controller.restapi.deployment; import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; @@ -36,24 +36,24 @@ public class BadgeApiTest extends ControllerContainerTest { .build(); application.submit(applicationPackage).deploy(); application.submit(applicationPackage) - .runJob(JobType.systemTest) - .runJob(JobType.stagingTest) - .runJob(JobType.productionUsWest1) - .runJob(JobType.productionAwsUsEast1a) - .runJob(JobType.testUsWest1) - .runJob(JobType.productionApSoutheast1) - .failDeployment(JobType.testApSoutheast1); + .runJob(DeploymentContext.systemTest) + .runJob(DeploymentContext.stagingTest) + .runJob(DeploymentContext.productionUsWest1) + .runJob(DeploymentContext.productionAwsUsEast1a) + .runJob(DeploymentContext.testUsWest1) + .runJob(DeploymentContext.productionApSoutheast1) + .failDeployment(DeploymentContext.testApSoutheast1); application.submit(applicationPackage) - .failTests(JobType.systemTest, true) - .runJob(JobType.stagingTest); + .failTests(DeploymentContext.systemTest, true) + .runJob(DeploymentContext.stagingTest); for (int i = 0; i < 31; i++) if ((i & 1) == 0) - application.failDeployment(JobType.productionUsWest1); + application.failDeployment(DeploymentContext.productionUsWest1); else - application.triggerJobs().abortJob(JobType.productionUsWest1); + application.triggerJobs().abortJob(DeploymentContext.productionUsWest1); application.triggerJobs(); - tester.controller().applications().deploymentTrigger().reTrigger(application.instanceId(), JobType.systemTest, "reason"); - tester.controller().applications().deploymentTrigger().reTrigger(application.instanceId(), JobType.testEuWest1, "reason"); + tester.controller().applications().deploymentTrigger().reTrigger(application.instanceId(), DeploymentContext.systemTest, "reason"); + tester.controller().applications().deploymentTrigger().reTrigger(application.instanceId(), DeploymentContext.testEuWest1, "reason"); tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default"), Files.readString(Paths.get(responseFiles + "overview.svg")), 200); @@ -64,7 +64,7 @@ public class BadgeApiTest extends ControllerContainerTest { // New change not reflected before cache entry expires. tester.serviceRegistry().clock().advance(Duration.ofSeconds(59)); - application.runJob(JobType.productionUsWest1); + application.runJob(DeploymentContext.productionUsWest1); tester.assertResponse(authenticatedRequest("http://localhost:8080/badge/v1/tenant/application/default/production-us-west-1?historyLength=32"), Files.readString(Paths.get(responseFiles + "history.svg")), 200); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index 53da8c515a7..d837b9e8264 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -6,9 +6,9 @@ import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; @@ -48,7 +48,7 @@ public class DeploymentApiTest extends ControllerContainerTest { // Deploy application without any declared jobs on the oldest version. var oldAppWithoutDeployment = deploymentTester.newDeploymentContext("tenant4", "application4", "default"); - oldAppWithoutDeployment.submit().failDeployment(JobType.systemTest); + oldAppWithoutDeployment.submit().failDeployment(DeploymentContext.systemTest); oldAppWithoutDeployment.submit(emptyPackage); // System upgrades to 5.0 for the other applications. @@ -61,8 +61,8 @@ public class DeploymentApiTest extends ControllerContainerTest { var otherProductionApp = deploymentTester.newDeploymentContext("tenant2", "application2", "i2"); var appWithoutDeployments = deploymentTester.newDeploymentContext("tenant3", "application3", "default"); failingApp.submit(applicationPackage).deploy(); - productionApp.submit(multiInstancePackage).runJob(JobType.systemTest).runJob(JobType.stagingTest).runJob(JobType.productionUsWest1); - otherProductionApp.runJob(JobType.productionUsWest1); + productionApp.submit(multiInstancePackage).runJob(DeploymentContext.systemTest).runJob(DeploymentContext.stagingTest).runJob(DeploymentContext.productionUsWest1); + otherProductionApp.runJob(DeploymentContext.productionUsWest1); // Deploy once so that job information is stored, then remove the deployment by submitting an empty deployment spec. appWithoutDeployments.submit(applicationPackage).deploy(); @@ -75,13 +75,13 @@ public class DeploymentApiTest extends ControllerContainerTest { // Applications upgrade, 1/2 succeed deploymentTester.upgrader().maintain(); deploymentTester.triggerJobs(); - productionApp.runJob(JobType.systemTest).runJob(JobType.stagingTest).runJob(JobType.productionUsWest1); - failingApp.failDeployment(JobType.systemTest).failDeployment(JobType.stagingTest); + productionApp.runJob(DeploymentContext.systemTest).runJob(DeploymentContext.stagingTest).runJob(DeploymentContext.productionUsWest1); + failingApp.failDeployment(DeploymentContext.systemTest).failDeployment(DeploymentContext.stagingTest); deploymentTester.upgrader().maintain(); deploymentTester.triggerJobs(); // Application fails application change - productionApp.submit(multiInstancePackage).failDeployment(JobType.systemTest); + productionApp.submit(multiInstancePackage).failDeployment(DeploymentContext.systemTest); tester.controller().updateVersionStatus(censorConfigServers(VersionStatus.compute(tester.controller()))); tester.assertResponse(operatorRequest("http://localhost:8080/deployment/v1/"), new File("root.json")); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json index 9c34f9410ee..13a6391d5da 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json @@ -353,21 +353,21 @@ "jobs": [ "system-test", "staging-test", - "production-aws-us-east-1a", - "test-aws-us-east-1a", "production-ap-northeast-1", "test-ap-northeast-1", "production-ap-northeast-2", "test-ap-northeast-2", "production-ap-southeast-1", "test-ap-southeast-1", + "production-aws-us-east-1a", + "test-aws-us-east-1a", + "production-eu-west-1", + "test-eu-west-1", + "production-us-central-1", + "test-us-central-1", "production-us-east-3", "test-us-east-3", "production-us-west-1", - "test-us-west-1", - "production-us-central-1", - "test-us-central-1", - "production-eu-west-1", - "test-eu-west-1" + "test-us-west-1" ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index fca3dde8f7d..303230b91ad 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -19,7 +19,6 @@ import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; @@ -410,10 +409,7 @@ public class RoutingPoliciesTest { var context = tester.newDeploymentContext("tenant1", "app1", "default"); context.submit(applicationPackage).deploy(); var zone = ZoneId.from("dev", "us-east-1"); - var zoneApi = ZoneApiMock.from(zone.environment(), zone.region()); - tester.controllerTester().serviceRegistry().zoneRegistry() - .setZones(zoneApi) - .exclusiveRoutingIn(zoneApi); + tester.controllerTester().setRoutingMethod(List.of(zone), RoutingMethod.exclusive); var prodRecords = Set.of("app1.tenant1.us-central-1.vespa.oath.cloud", "app1.tenant1.us-west-1.vespa.oath.cloud"); assertEquals(prodRecords, tester.recordNames()); @@ -624,7 +620,7 @@ public class RoutingPoliciesTest { // Application starts deployment context = context.submit(applicationPackage); - for (var testJob : List.of(JobType.systemTest, JobType.stagingTest)) { + for (var testJob : List.of(DeploymentContext.systemTest, DeploymentContext.stagingTest)) { context = context.runJob(testJob); // Since runJob implicitly tears down the deployment and immediately deletes DNS records associated with the // deployment, we consume only one DNS update at a time here @@ -879,8 +875,8 @@ public class RoutingPoliciesTest { var sharedRegion = RegionName.from("aws-us-east-1c"); return List.of(ZoneId.from(Environment.prod, sharedRegion), ZoneId.from(Environment.prod, RegionName.from("aws-eu-west-1a")), - ZoneId.from(Environment.staging, sharedRegion), - ZoneId.from(Environment.test, sharedRegion)); + ZoneId.from(Environment.staging, RegionName.from("us-east-3")), + ZoneId.from(Environment.test, RegionName.from("us-east-1"))); } private static class RoutingPoliciesTester { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java index 02154470de2..277c8e1ef85 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; @@ -115,7 +114,7 @@ public class RotationRepositoryTest { // We're now out of rotations and next deployment fails var application3 = tester.newDeploymentContext("tenant3", "app3", "default"); application3.submit(applicationPackage) - .runJobExpectingFailure(JobType.systemTest, Optional.of("out of rotations")); + .runJobExpectingFailure(DeploymentContext.systemTest, Optional.of("out of rotations")); } @Test @@ -124,7 +123,7 @@ public class RotationRepositoryTest { .globalServiceId("foo") .region("us-east-3") .build(); - application.submit(applicationPackage).runJobExpectingFailure(JobType.systemTest, Optional.of("less than 2 prod zones are defined")); + application.submit(applicationPackage).runJobExpectingFailure(DeploymentContext.systemTest, Optional.of("less than 2 prod zones are defined")); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index 297410112d5..7a137d4e410 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.versions; import com.yahoo.component.Version; -import com.yahoo.component.Vtag; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; @@ -31,10 +30,10 @@ import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsEast3; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsEast3; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; import static java.util.stream.Collectors.toSet; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; diff --git a/controller-server/src/test/resources/testConfig.json b/controller-server/src/test/resources/testConfig.json index 7b91b4930a1..5c3d5942001 100644 --- a/controller-server/src/test/resources/testConfig.json +++ b/controller-server/src/test/resources/testConfig.json @@ -1,20 +1,20 @@ { "application": "tenant:application:default", - "zone": "test.aws-us-east-1c", + "zone": "test.us-east-1", "system": "publiccd", "isCI": true, "endpoints": { - "test.aws-us-east-1c": [ + "test.us-east-1": [ "https://ai.default.default.global.vespa.oath.cloud/" ] }, "zoneEndpoints": { - "test.aws-us-east-1c": { + "test.us-east-1": { "ai": "https://ai.default.default.global.vespa.oath.cloud/" } }, "clusters": { - "test.aws-us-east-1c": [ + "test.us-east-1": [ "facts" ] } diff --git a/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java b/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java index cf7e3851d55..d6166f8fcbe 100644 --- a/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java +++ b/document/src/main/java/com/yahoo/document/select/rule/ComparisonNode.java @@ -282,6 +282,8 @@ public class ComparisonNode implements ExpressionNode { return getAsNumber(((NumericFieldValue)value).getNumber()); } else if (value instanceof BoolFieldValue) { return ((BoolFieldValue)value).getBoolean() ? 1 : 0; + } else if (value instanceof Boolean) { + return (Boolean)value ? 1 : 0; } else { return Double.NaN; //new IllegalStateException("Term '" + value + "' (" + value.getClass() + ") does not evaluate to a number."); } diff --git a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java index bf8cf07a097..38fdadb18e4 100644 --- a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java +++ b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java @@ -507,6 +507,16 @@ public class DocumentSelectorTestCase { assertEquals(Result.FALSE, evaluate("14.3 == null", documents.get(0))); assertEquals(Result.FALSE, evaluate("null = 0", documents.get(0))); + // Boolean literals in comparisons + assertEquals(Result.TRUE, evaluate("true = true", documents.get(0))); + assertEquals(Result.TRUE, evaluate("true == true", documents.get(0))); + assertEquals(Result.FALSE, evaluate("true == false", documents.get(0))); + assertEquals(Result.TRUE, evaluate("false == false", documents.get(0))); + assertEquals(Result.TRUE, evaluate("true == 1", documents.get(0))); + assertEquals(Result.FALSE, evaluate("true == 0", documents.get(0))); + assertEquals(Result.FALSE, evaluate("false == 1", documents.get(0))); + assertEquals(Result.TRUE, evaluate("false == 0", documents.get(0))); + // Field values assertEquals(Result.TRUE, evaluate("test.hint = 24", documents.get(0))); assertEquals(Result.FALSE, evaluate("test.hint = 24", documents.get(1))); @@ -748,12 +758,19 @@ public class DocumentSelectorTestCase { @Test public void boolean_fields_can_be_used_for_equality_comparisons() throws ParseException { var documents = createDocs(); - assertEquals(Result.TRUE, evaluate("test.truth == 1", documents.get(8))); // has explicit field set to true - assertEquals(Result.FALSE, evaluate("test.truth == 1", documents.get(9))); // has explicit field set to false + // Doc 8 has bool field set explicitly to true, doc 9 has field explicitly set to false + assertEquals(Result.TRUE, evaluate("test.truth == 1", documents.get(8))); + assertEquals(Result.TRUE, evaluate("test.truth == true", documents.get(8))); + assertEquals(Result.FALSE, evaluate("test.truth == 1", documents.get(9))); + assertEquals(Result.FALSE, evaluate("test.truth == true", documents.get(9))); assertEquals(Result.TRUE, evaluate("test.truth == 0", documents.get(9))); + assertEquals(Result.TRUE, evaluate("test.truth == false", documents.get(9))); // FIXME very un-intuitive behavior when nulls are implicitly returned: - assertEquals(Result.FALSE, evaluate("test.truth == 1", documents.get(0))); // Does not have field set in document - assertEquals(Result.FALSE, evaluate("test.truth == 0", documents.get(0))); // Does not have field set in document + // Doc 1 does not have the bool field set, but the implicit null value is neither true nor false + assertEquals(Result.FALSE, evaluate("test.truth == 1", documents.get(0))); + assertEquals(Result.FALSE, evaluate("test.truth == true", documents.get(0))); + assertEquals(Result.FALSE, evaluate("test.truth == 0", documents.get(0))); + assertEquals(Result.FALSE, evaluate("test.truth == false", documents.get(0))); } @Test diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp index 6644fec2da0..1606cbc634d 100644 --- a/document/src/tests/documentselectparsertest.cpp +++ b/document/src/tests/documentselectparsertest.cpp @@ -637,6 +637,16 @@ TEST_F(DocumentSelectParserTest, operators_1) PARSE("14.3 == null", *_doc[0], False); PARSE("null = 0", *_doc[0], False); + // Boolean literals in comparisons + PARSE("true = true", *_doc[0], True); + PARSE("true == true", *_doc[0], True); + PARSE("true == false", *_doc[0], False); + PARSE("false == false", *_doc[0], True); + PARSE("true == 1", *_doc[0], True); + PARSE("true == 0", *_doc[0], False); + PARSE("false == 1", *_doc[0], False); + PARSE("false == 0", *_doc[0], True); + // Field values PARSE("testdoctype1.headerval = 24", *_doc[0], True); PARSE("testdoctype1.headerval = 24", *_doc[1], False); @@ -925,12 +935,19 @@ TEST_F(DocumentSelectParserTest, operators_9) TEST_F(DocumentSelectParserTest, can_use_boolean_fields_in_expressions) { createDocs(); - PARSE("testdoctype1.boolfield == 1", *_doc[11], True); // has explicit field set to true - PARSE("testdoctype1.boolfield == 1", *_doc[12], False); // has explicit field set to false + // Doc 11 has bool field set explicitly to true, doc 12 has field explicitly set to false + PARSE("testdoctype1.boolfield == 1", *_doc[11], True); + PARSE("testdoctype1.boolfield == true", *_doc[11], True); + PARSE("testdoctype1.boolfield == 1", *_doc[12], False); + PARSE("testdoctype1.boolfield == true", *_doc[12], False); PARSE("testdoctype1.boolfield == 0", *_doc[12], True); + PARSE("testdoctype1.boolfield == false", *_doc[12], True); // FIXME very un-intuitive behavior when nulls are implicitly returned: - PARSE("testdoctype1.boolfield == 1", *_doc[1], False); // Does not have field set in document - PARSE("testdoctype1.boolfield == 0", *_doc[1], False); // Does not have field set in document + // Doc 1 does not have the bool field set, but the implicit null value is neither true nor false + PARSE("testdoctype1.boolfield == 1", *_doc[1], False); + PARSE("testdoctype1.boolfield == true", *_doc[1], False); + PARSE("testdoctype1.boolfield == 0", *_doc[1], False); + PARSE("testdoctype1.boolfield == false", *_doc[1], False); } namespace { @@ -989,6 +1006,7 @@ namespace { void visitFloatValueNode(const select::FloatValueNode &) override {} void visitVariableValueNode(const select::VariableValueNode &) override {} void visitIntegerValueNode(const select::IntegerValueNode &) override {} + void visitBoolValueNode(const select::BoolValueNode &) override {} void visitCurrentTimeValueNode(const select::CurrentTimeValueNode &) override {} void visitStringValueNode(const select::StringValueNode &) override {} void visitNullValueNode(const select::NullValueNode &) override {} @@ -1392,6 +1410,9 @@ public: void visitIntegerValueNode(const select::IntegerValueNode& node) override { data << node.getValue(); } + void visitBoolValueNode(const select::BoolValueNode& node) override { + data << node.bool_value_str(); + } void visitCurrentTimeValueNode(const select::CurrentTimeValueNode&) override {} void visitStringValueNode(const select::StringValueNode& str) override { data << '"' << str.getValue() << '"'; @@ -1491,6 +1512,17 @@ TEST_F(DocumentSelectParserTest, test_ambiguous_field_spec_expression_is_handled parse_to_tree("(testdoctype1.foo) AND (testdoctype1.bar)")); } +TEST_F(DocumentSelectParserTest, test_ambiguous_bool_expression_is_handled_correctly) +{ + createDocs(); + using namespace std::string_literals; + // Bools both as high level Nodes and low level ValueNodes + EXPECT_EQ("(OR (AND true false) (== (FIELD testdoctype1 myfield) true))"s, + parse_to_tree("true and false or testdoctype1.myfield == true")); + EXPECT_EQ("(!= true false)"s, parse_to_tree("true != false")); + EXPECT_EQ("(!= true false)"s, parse_to_tree("(true) != (false)")); +} + TEST_F(DocumentSelectParserTest, special_tokens_are_allowed_as_freestanding_identifier_names) { createDocs(); EXPECT_EQ("(NOT (DOCTYPE user))", parse_to_tree("not user")); diff --git a/document/src/vespa/document/bucket/bucketselector.cpp b/document/src/vespa/document/bucket/bucketselector.cpp index 34f613bef84..b9dc6a9cfa9 100644 --- a/document/src/vespa/document/bucket/bucketselector.cpp +++ b/document/src/vespa/document/bucket/bucketselector.cpp @@ -152,6 +152,7 @@ using namespace document::select; void visitFloatValueNode(const FloatValueNode &) override {} void visitVariableValueNode(const VariableValueNode &) override {} void visitIntegerValueNode(const IntegerValueNode &) override {} + void visitBoolValueNode(const BoolValueNode &) override {} void visitCurrentTimeValueNode(const CurrentTimeValueNode &) override {} void visitStringValueNode(const StringValueNode &) override {} void visitNullValueNode(const NullValueNode &) override {} diff --git a/document/src/vespa/document/select/cloningvisitor.cpp b/document/src/vespa/document/select/cloningvisitor.cpp index 55e8c1effc1..81cb31145ec 100644 --- a/document/src/vespa/document/select/cloningvisitor.cpp +++ b/document/src/vespa/document/select/cloningvisitor.cpp @@ -10,30 +10,6 @@ namespace document::select { -const int CloningVisitor::OrPriority; -const int CloningVisitor::AndPriority; -const int CloningVisitor::NotPriority; -const int CloningVisitor::ComparePriority; -const int CloningVisitor::AddPriority; -const int CloningVisitor::SubPriority; -const int CloningVisitor::MulPriority; -const int CloningVisitor::DivPriority; -const int CloningVisitor::ModPriority; -const int CloningVisitor::DocumentTypePriority; -const int CloningVisitor::FieldValuePriority; -const int CloningVisitor::InvalidConstPriority; -const int CloningVisitor::InvalidValPriority; -const int CloningVisitor::ConstPriority; -const int CloningVisitor::FuncPriority; -const int CloningVisitor::VariablePriority; -const int CloningVisitor::FloatPriority; -const int CloningVisitor::IntegerPriority; -const int CloningVisitor::CurrentTimePriority; -const int CloningVisitor::StringPriority; -const int CloningVisitor::NullValPriority; -const int CloningVisitor::IdPriority; -const int CloningVisitor::SearchColPriority; - CloningVisitor::CloningVisitor() : _node(), _valueNode(), @@ -63,7 +39,7 @@ CloningVisitor::visitAndBranch(const And &expr) setNodeParentheses(priority); std::unique_ptr<Node> rhs(std::move(_node)); _priority = priority; - _node.reset(new And(std::move(lhs), std::move(rhs), "and")); + _node = std::make_unique<And>(std::move(lhs), std::move(rhs), "and"); }; @@ -83,7 +59,7 @@ CloningVisitor::visitOrBranch(const Or &expr) setNodeParentheses(priority); std::unique_ptr<Node> rhs(std::move(_node)); _priority = priority; - _node.reset(new Or(std::move(lhs), std::move(rhs), "or")); + _node = std::make_unique<Or>(std::move(lhs), std::move(rhs), "or"); }; @@ -96,7 +72,7 @@ CloningVisitor::visitNotBranch(const Not &expr) _resultSet.calcNot(); std::unique_ptr<Node> child(std::move(_node)); _priority = priority; - _node.reset(new Not(std::move(child), "not")); + _node = std::make_unique<Not>(std::move(child), "not"); }; @@ -116,10 +92,7 @@ CloningVisitor::visitComparison(const Compare &expr) const Operator &op(expr.getOperator()); _priority = priority; _resultSet.fill(); // should be less if const - _node.reset(new Compare(std::move(lhs), - op, - std::move(rhs), - expr.getBucketIdFactory())); + _node = std::make_unique<Compare>(std::move(lhs), op, std::move(rhs), expr.getBucketIdFactory()); }; @@ -149,7 +122,7 @@ CloningVisitor::visitFunctionValueNode(const FunctionValueNode &expr) setValueNodeParentheses(priority); std::unique_ptr<ValueNode> child(std::move(_valueNode)); _priority = priority; - _valueNode.reset(new FunctionValueNode(expr.getFunctionName(), std::move(child))); + _valueNode = std::make_unique<FunctionValueNode>(expr.getFunctionName(), std::move(child)); }; @@ -160,7 +133,7 @@ CloningVisitor::visitConstant(const Constant &expr) _priority = ConstPriority; bool val = expr.getConstantValue(); _resultSet.add(val ? Result::True : Result::False); - _node.reset(new Constant(val)); + _node = std::make_unique<Constant>(val); } @@ -171,7 +144,7 @@ CloningVisitor::visitInvalidConstant(const InvalidConstant &expr) _constVal = true; _priority = InvalidConstPriority; _resultSet.add(Result::Invalid); - _node.reset(new InvalidConstant("invalid")); + _node = std::make_unique<InvalidConstant>("invalid"); } @@ -218,7 +191,7 @@ CloningVisitor::visitFloatValueNode(const FloatValueNode &expr) void CloningVisitor::visitVariableValueNode(const VariableValueNode &expr) { - _valueNode.reset(new VariableValueNode(expr.getVariableName())); + _valueNode = std::make_unique<VariableValueNode>(expr.getVariableName()); _priority = VariablePriority; } @@ -233,6 +206,15 @@ CloningVisitor::visitIntegerValueNode(const IntegerValueNode &expr) void +CloningVisitor::visitBoolValueNode(const BoolValueNode &expr) +{ + _constVal = true; + _valueNode = expr.clone(); + _priority = BoolPriority; +} + + +void CloningVisitor::visitCurrentTimeValueNode(const CurrentTimeValueNode &expr) { _constVal = false; diff --git a/document/src/vespa/document/select/cloningvisitor.h b/document/src/vespa/document/select/cloningvisitor.h index a49f44190ef..cce171f81cb 100644 --- a/document/src/vespa/document/select/cloningvisitor.h +++ b/document/src/vespa/document/select/cloningvisitor.h @@ -20,33 +20,33 @@ protected: uint32_t _fieldNodes; ResultSet _resultSet; - static const int OrPriority = 100; - static const int AndPriority = 200; - static const int NotPriority = 300; - static const int ComparePriority = 400; - static const int AddPriority = 500; - static const int SubPriority = 500; - static const int MulPriority = 600; - static const int DivPriority = 600; - static const int ModPriority = 700; - static const int DocumentTypePriority = 1000; - static const int FieldValuePriority = 1000; - static const int InvalidConstPriority = 1000; - static const int InvalidValPriority = 1000; - static const int ConstPriority = 1000; - static const int FuncPriority = 1000; - static const int VariablePriority = 1000; - static const int FloatPriority = 1000; - static const int IntegerPriority = 1000; - static const int CurrentTimePriority = 1000; - static const int StringPriority = 1000; - static const int NullValPriority = 1000; - static const int IdPriority = 1000; - static const int SearchColPriority = 1000; + static constexpr int OrPriority = 100; + static constexpr int AndPriority = 200; + static constexpr int NotPriority = 300; + static constexpr int ComparePriority = 400; + static constexpr int AddPriority = 500; + static constexpr int SubPriority = 500; + static constexpr int MulPriority = 600; + static constexpr int DivPriority = 600; + static constexpr int ModPriority = 700; + static constexpr int DocumentTypePriority = 1000; + static constexpr int FieldValuePriority = 1000; + static constexpr int InvalidConstPriority = 1000; + static constexpr int InvalidValPriority = 1000; + static constexpr int ConstPriority = 1000; + static constexpr int FuncPriority = 1000; + static constexpr int VariablePriority = 1000; + static constexpr int FloatPriority = 1000; + static constexpr int IntegerPriority = 1000; + static constexpr int BoolPriority = 1000; + static constexpr int CurrentTimePriority = 1000; + static constexpr int StringPriority = 1000; + static constexpr int NullValPriority = 1000; + static constexpr int IdPriority = 1000; public: CloningVisitor(); - ~CloningVisitor(); + ~CloningVisitor() override; void visitAndBranch(const And &expr) override; void visitOrBranch(const Or &expr) override; @@ -62,6 +62,7 @@ public: void visitFloatValueNode(const FloatValueNode &expr) override; void visitVariableValueNode(const VariableValueNode &expr) override; void visitIntegerValueNode(const IntegerValueNode &expr) override; + void visitBoolValueNode(const BoolValueNode &expr) override; void visitCurrentTimeValueNode(const CurrentTimeValueNode &expr) override; void visitStringValueNode(const StringValueNode &expr) override; void visitNullValueNode(const NullValueNode &expr) override; @@ -77,7 +78,7 @@ public: int rhsPriority, bool rhsConstVal); void swap(CloningVisitor &rhs); - void revisit(void); + void revisit(); }; } diff --git a/document/src/vespa/document/select/gid_filter.cpp b/document/src/vespa/document/select/gid_filter.cpp index 3a3e215db68..a1954eb6bc2 100644 --- a/document/src/vespa/document/select/gid_filter.cpp +++ b/document/src/vespa/document/select/gid_filter.cpp @@ -26,6 +26,7 @@ struct NoOpVisitor : Visitor { void visitFloatValueNode(const FloatValueNode&) override {} void visitVariableValueNode(const VariableValueNode&) override {} void visitIntegerValueNode(const IntegerValueNode&) override {} + void visitBoolValueNode(const BoolValueNode&) override {} void visitCurrentTimeValueNode(const CurrentTimeValueNode&) override {} void visitStringValueNode(const StringValueNode&) override {} void visitNullValueNode(const NullValueNode&) override {} diff --git a/document/src/vespa/document/select/grammar/parser.yy b/document/src/vespa/document/select/grammar/parser.yy index 8d64b2382b5..409f8e3b29e 100644 --- a/document/src/vespa/document/select/grammar/parser.yy +++ b/document/src/vespa/document/select/grammar/parser.yy @@ -75,7 +75,7 @@ %token <string_val> ID USER GROUP SCHEME NAMESPACE SPECIFIC BUCKET GID TYPE %type <string_val> ident mangled_ident -%type <abstract_node> bool_ +%type <value_node> bool_ /* TODO 'leaf' is a bad name for something that isn't a leaf... */ %type <abstract_node> expression comparison logical_expr leaf doc_type %type <string_val> id_arg @@ -190,8 +190,8 @@ null_ ; bool_ - : TRUE { $$ = new Constant(true); } - | FALSE { $$ = new Constant(false); } + : TRUE { $$ = new BoolValueNode(true); } + | FALSE { $$ = new BoolValueNode(false); } ; number @@ -287,6 +287,7 @@ field_spec value : null_ { $$ = $1; } + | bool_ { $$ = $1; } | string { $$ = $1; } | id_spec { $$ = $1; } | variable { $$ = $1; } @@ -328,10 +329,9 @@ comparison ; leaf - : bool_ { $$ = $1; } - | comparison { $$ = $1; } + : comparison { $$ = $1; } | doc_type { $$ = $1; } - | arith_expr { /* Actually field_spec, see comment below..! */ + | arith_expr { /* Actually bool_ or field_spec, see comment below..! */ // Grammar-wise, we _do not_ accept arbitrary arith_exprs at this level. But the // selection grammar as it stands is otherwise ambiguous with LR(1) parsing. // More specifically, if we used field_spec instead of arith_expr here, the parser @@ -343,15 +343,21 @@ leaf // conflict goes away. We can then do a sneaky "run-time" type check to ensure we only // get the expected node from the rule. // It's not pretty, but it avoids an undefined grammar (which is much less pretty!). + // The same goes for boolean constants, which may be used both as higher-level (non-value) + // nodes or as value nodes to compare against. auto node = steal<ValueNode>($1); - if (dynamic_cast<FieldValueNode*>(node.get()) == nullptr) { - throw syntax_error(@$, "expected field spec, doctype, bool or comparison"); + if (auto* as_bool = dynamic_cast<BoolValueNode*>(node.get())) { + $$ = new Constant(as_bool->bool_value()); // Replace single bool node subtree with constant. + } else { + if (dynamic_cast<FieldValueNode*>(node.get()) == nullptr) { + throw syntax_error(@$, "expected field spec, doctype, bool or comparison"); + } + // Implicit rewrite to non-null comparison node + $$ = new Compare(std::move(node), + FunctionOperator::NE, + std::make_unique<NullValueNode>(), + bucket_id_factory); } - // Implicit rewrite to non-null comparison node - $$ = new Compare(std::move(node), - FunctionOperator::NE, - std::make_unique<NullValueNode>(), - bucket_id_factory); } ; diff --git a/document/src/vespa/document/select/traversingvisitor.cpp b/document/src/vespa/document/select/traversingvisitor.cpp index a0b7441903e..f7e76e6fe32 100644 --- a/document/src/vespa/document/select/traversingvisitor.cpp +++ b/document/src/vespa/document/select/traversingvisitor.cpp @@ -97,6 +97,12 @@ TraversingVisitor::visitIntegerValueNode(const IntegerValueNode &) void +TraversingVisitor::visitBoolValueNode(const BoolValueNode &) +{ +} + + +void TraversingVisitor::visitCurrentTimeValueNode(const CurrentTimeValueNode &) { } diff --git a/document/src/vespa/document/select/traversingvisitor.h b/document/src/vespa/document/select/traversingvisitor.h index be240c42537..2a7531c06f0 100644 --- a/document/src/vespa/document/select/traversingvisitor.h +++ b/document/src/vespa/document/select/traversingvisitor.h @@ -24,6 +24,7 @@ public: void visitFloatValueNode(const FloatValueNode &) override; void visitVariableValueNode(const VariableValueNode &) override; void visitIntegerValueNode(const IntegerValueNode &) override; + void visitBoolValueNode(const BoolValueNode &) override; void visitCurrentTimeValueNode(const CurrentTimeValueNode &) override; void visitStringValueNode(const StringValueNode &) override; void visitNullValueNode(const NullValueNode &) override; diff --git a/document/src/vespa/document/select/valuenode.h b/document/src/vespa/document/select/valuenode.h index 71efa73b1d8..fb86128aa2c 100644 --- a/document/src/vespa/document/select/valuenode.h +++ b/document/src/vespa/document/select/valuenode.h @@ -57,12 +57,11 @@ protected: } } - ValueNode::UP wrapParens(ValueNode* node) const { - ValueNode::UP ret(node); + std::unique_ptr<ValueNode> wrapParens(std::unique_ptr<ValueNode> node) const { if (_parentheses) { - ret->setParentheses(); + node->setParentheses(); } - return ret; + return node; } std::unique_ptr<Value> defaultTrace(std::unique_ptr<Value> val, std::ostream& out) const; diff --git a/document/src/vespa/document/select/valuenodes.cpp b/document/src/vespa/document/select/valuenodes.cpp index 1c7d47d0591..452779ca5ba 100644 --- a/document/src/vespa/document/select/valuenodes.cpp +++ b/document/src/vespa/document/select/valuenodes.cpp @@ -115,11 +115,28 @@ IntegerValueNode::print(std::ostream& out, bool verbose, if (hadParentheses()) out << ')'; } +void +BoolValueNode::visit(Visitor& visitor) const +{ + visitor.visitBoolValueNode(*this); +} + +void +BoolValueNode::print(std::ostream& out, + [[maybe_unused]] bool verbose, + [[maybe_unused]] const std::string& indent) const +{ + if (hadParentheses()) out << '('; + out << bool_value_str(); + if (hadParentheses()) out << ')'; +} + + int64_t CurrentTimeValueNode::getValue() const { struct timeval mytime; - gettimeofday(&mytime, 0); + gettimeofday(&mytime, nullptr); return mytime.tv_sec; } diff --git a/document/src/vespa/document/select/valuenodes.h b/document/src/vespa/document/select/valuenodes.h index f6fa0400d7c..0c4584e7eee 100644 --- a/document/src/vespa/document/select/valuenodes.h +++ b/document/src/vespa/document/select/valuenodes.h @@ -28,7 +28,7 @@ public: void visit(Visitor& visitor) const override; ValueNode::UP clone() const override { - return wrapParens(new InvalidValueNode(_name)); + return wrapParens(std::make_unique<InvalidValueNode>(_name)); } }; @@ -46,7 +46,7 @@ public: void visit(Visitor& visitor) const override; ValueNode::UP clone() const override { - return wrapParens(new NullValueNode()); + return wrapParens(std::make_unique<NullValueNode>()); } }; @@ -66,7 +66,7 @@ public: void visit(Visitor& visitor) const override; ValueNode::UP clone() const override { - return wrapParens(new StringValueNode(_value)); + return wrapParens(std::make_unique<StringValueNode>(_value)); } }; @@ -88,7 +88,28 @@ public: void visit(Visitor& visitor) const override; ValueNode::UP clone() const override { - return wrapParens(new IntegerValueNode(_value, _isBucketValue)); + return wrapParens(std::make_unique<IntegerValueNode>(_value, _isBucketValue)); + } +}; + +// Inherit from IntegerValueNode to be implicitly treated as an integer value by +// all code that does not explicitly know (or care) about bool values. +class BoolValueNode : public IntegerValueNode { +public: + explicit BoolValueNode(bool value) : IntegerValueNode(value, false) {} + + [[nodiscard]] bool bool_value() const noexcept { + return bool(getValue()); + } + [[nodiscard]] const char* bool_value_str() const noexcept { + return bool_value() ? "true" : "false"; + } + + void print(std::ostream& out, bool verbose, const std::string& indent) const override; + void visit(Visitor& visitor) const override; + + std::unique_ptr<ValueNode> clone() const override { + return wrapParens(std::make_unique<BoolValueNode>(bool_value())); } }; @@ -105,7 +126,7 @@ public: void visit(Visitor& visitor) const override; ValueNode::UP clone() const override { - return wrapParens(new CurrentTimeValueNode); + return wrapParens(std::make_unique<CurrentTimeValueNode>()); } }; @@ -123,7 +144,7 @@ public: void visit(Visitor& visitor) const override; ValueNode::UP clone() const override { - return wrapParens(new VariableValueNode(_value)); + return wrapParens(std::make_unique<VariableValueNode>(_value)); } }; @@ -131,7 +152,7 @@ class FloatValueNode : public ValueNode { double _value; public: - FloatValueNode(double val) : _value(val) {} + explicit FloatValueNode(double val) : _value(val) {} double getValue() const { return _value; } @@ -143,7 +164,7 @@ public: void visit(Visitor& visitor) const override; ValueNode::UP clone() const override { - return wrapParens(new FloatValueNode(_value)); + return wrapParens(std::make_unique<FloatValueNode>(_value)); } }; @@ -172,7 +193,7 @@ public: void visit(Visitor& visitor) const override; ValueNode::UP clone() const override { - return wrapParens(new FieldValueNode(_doctype, _fieldExpression)); + return wrapParens(std::make_unique<FieldValueNode>(_doctype, _fieldExpression)); } static vespalib::string extractFieldName(const vespalib::string & fieldExpression); @@ -229,10 +250,10 @@ private: ValueNode::UP clone() const override { if (_left_expr) { - return wrapParens(new FieldExprNode(std::unique_ptr<FieldExprNode>( + return wrapParens(std::make_unique<FieldExprNode>(std::unique_ptr<FieldExprNode>( static_cast<FieldExprNode*>(_left_expr->clone().release())), _right_expr)); } else { - return wrapParens(new FieldExprNode(_right_expr)); + return wrapParens(std::make_unique<FieldExprNode>(_right_expr)); } } }; @@ -261,7 +282,7 @@ public: void visit(Visitor& visitor) const override; ValueNode::UP clone() const override { - return wrapParens(new IdValueNode(_bucketIdFactory, _id, _typestring, _widthBits, _divisionBits)); + return wrapParens(std::make_unique<IdValueNode>(_bucketIdFactory, _id, _typestring, _widthBits, _divisionBits)); } int getWidthBits() const { return _widthBits; } @@ -298,7 +319,7 @@ public: void visit(Visitor& visitor) const override; ValueNode::UP clone() const override { - return wrapParens(new FunctionValueNode(_funcname, _source->clone())); + return wrapParens(std::make_unique<FunctionValueNode>(_funcname, _source->clone())); } const ValueNode& getChild() const { return *_source; } @@ -339,9 +360,7 @@ public: void visit(Visitor& visitor) const override; ValueNode::UP clone() const override { - return wrapParens(new ArithmeticValueNode(_left->clone(), - getOperatorName(), - _right->clone())); + return wrapParens(std::make_unique<ArithmeticValueNode>(_left->clone(), getOperatorName(), _right->clone())); } const ValueNode& getLeft() const { return *_left; } diff --git a/document/src/vespa/document/select/visitor.h b/document/src/vespa/document/select/visitor.h index a9b134f80fc..4664b831238 100644 --- a/document/src/vespa/document/select/visitor.h +++ b/document/src/vespa/document/select/visitor.h @@ -27,6 +27,7 @@ class InvalidConstant; class FieldValueNode; class FloatValueNode; class IntegerValueNode; +class BoolValueNode; class CurrentTimeValueNode; class StringValueNode; class NullValueNode; @@ -35,7 +36,7 @@ class VariableValueNode; class Visitor { public: - virtual ~Visitor() {} + virtual ~Visitor() = default; virtual void visitAndBranch(const And &) = 0; virtual void visitComparison(const Compare &) = 0; @@ -51,6 +52,7 @@ public: virtual void visitFloatValueNode(const FloatValueNode &) = 0; virtual void visitVariableValueNode(const VariableValueNode &) = 0; virtual void visitIntegerValueNode(const IntegerValueNode &) = 0; + virtual void visitBoolValueNode(const BoolValueNode&) = 0; virtual void visitCurrentTimeValueNode(const CurrentTimeValueNode &) = 0; virtual void visitStringValueNode(const StringValueNode &) = 0; virtual void visitNullValueNode(const NullValueNode &) = 0; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 61e777a9576..75977da369c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -27,6 +27,9 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.identity.CredentialsMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.VespaServiceDumper; import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; +import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; +import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; import java.time.Clock; import java.time.Duration; @@ -137,6 +140,31 @@ public class NodeAgentImpl implements NodeAgent { if (loopThread != null) throw new IllegalStateException("Can not re-start a node agent."); + // TODO: Remove after this has rolled out everywhere + int[] stats = new int[]{0, 0, 0}; + ContainerPath vespaHome = initialContext.paths().underVespaHome(""); + FileFinder.files(initialContext.paths().of("/")).forEachPath(path -> { + UnixPath unixPath = new UnixPath(path); + + String permissions = unixPath.getPermissions(); + if (!permissions.endsWith("---")) { + unixPath.setPermissions(permissions.substring(0, 6) + "---"); + stats[0]++; + } + + if (path.startsWith(vespaHome) && unixPath.getOwnerId() != initialContext.users().vespa().uid()) { + unixPath.setOwnerId(initialContext.users().vespa().uid()); + stats[1]++; + } + + if (path.startsWith(vespaHome) && unixPath.getGroupId() != initialContext.users().vespa().gid()) { + unixPath.setGroupId(initialContext.users().vespa().gid()); + stats[2]++; + } + }); + if (stats[0] + stats[1] + stats[2] > 0) + initialContext.log(logger, "chmod %d, chown UID %d, chown GID %d files", stats[0], stats[1], stats[2]); + loopThread = new Thread(() -> { while (!terminated.get()) { try { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java index 964ed5e0e4d..2a2e3d611c9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemProvider.java @@ -26,6 +26,8 @@ import java.nio.file.attribute.FileAttributeView; import java.nio.file.attribute.GroupPrincipal; import java.nio.file.attribute.PosixFileAttributeView; import java.nio.file.attribute.PosixFileAttributes; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; import java.nio.file.attribute.UserPrincipal; import java.nio.file.spi.FileSystemProvider; import java.util.HashMap; @@ -44,6 +46,12 @@ import static com.yahoo.yolean.Exceptions.uncheck; * @author freva */ class ContainerFileSystemProvider extends FileSystemProvider { + + private static final FileAttribute<?> DEFAULT_FILE_PERMISSIONS = PosixFilePermissions.asFileAttribute(Set.of( // 0640 + PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.GROUP_READ)); + private static final FileAttribute<?> DEFAULT_DIRECTORY_PERMISSIONS = PosixFilePermissions.asFileAttribute(Set.of( // 0750 + PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.GROUP_READ, PosixFilePermission.GROUP_EXECUTE)); + private final ContainerFileSystem containerFs; private final ContainerUserPrincipalLookupService userPrincipalLookupService; @@ -82,7 +90,8 @@ class ContainerFileSystemProvider extends FileSystemProvider { Path pathOnHost = pathOnHost(path); try (SecureDirectoryStream<Path> sds = leafDirectoryStream(pathOnHost)) { boolean existedBefore = Files.exists(pathOnHost); - SeekableByteChannel seekableByteChannel = sds.newByteChannel(pathOnHost.getFileName(), addNoFollow(options), attrs); + SeekableByteChannel seekableByteChannel = sds.newByteChannel( + pathOnHost.getFileName(), addNoFollow(options), addPermissions(DEFAULT_FILE_PERMISSIONS, attrs)); if (!existedBefore) fixOwnerToContainerRoot(toContainerPath(path)); return seekableByteChannel; } @@ -99,7 +108,7 @@ class ContainerFileSystemProvider extends FileSystemProvider { public void createDirectory(Path dir, FileAttribute<?>... attrs) throws IOException { Path pathOnHost = pathOnHost(dir); boolean existedBefore = Files.exists(pathOnHost); - provider(pathOnHost).createDirectory(pathOnHost); + provider(pathOnHost).createDirectory(pathOnHost, addPermissions(DEFAULT_DIRECTORY_PERMISSIONS, attrs)); if (!existedBefore) fixOwnerToContainerRoot(toContainerPath(dir)); } @@ -324,4 +333,16 @@ class ContainerFileSystemProvider extends FileSystemProvider { copy[options.length] = LinkOption.NOFOLLOW_LINKS; return copy; } + + private static FileAttribute<?>[] addPermissions(FileAttribute<?> defaultPermissions, FileAttribute<?>... attrs) { + for (FileAttribute<?> attr : attrs) { + if (attr.name().equals("posix:permissions") || attr.name().equals("unix:permissions")) + return attrs; + } + + FileAttribute<?>[] copy = new FileAttribute<?>[attrs.length + 1]; + System.arraycopy(attrs, 0, copy, 0, attrs.length); + copy[attrs.length] = defaultPermissions; + return copy; + } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java index c3affccc32b..b26f0fe5bf8 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/fs/ContainerFileSystemTest.java @@ -16,6 +16,8 @@ import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermissions; import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -174,6 +176,19 @@ class ContainerFileSystemTest { Files.writeString(file.pathOnHost(), "hello"); // Writing through host FS works } + @Test + public void permissions() throws IOException { + assertPermissions(Files.createDirectory(containerFs.getPath("/dir1")), "rwxr-x---"); + assertPermissions(Files.createDirectory(containerFs.getPath("/dir2"), permissionsFromString("r-x-w-rw-")), "r-x-w-rw-"); + + assertPermissions(Files.createDirectories(containerFs.getPath("/sub/dir/leaf"), permissionsFromString("r-x-w-rw-")), "r-x-w-rw-"); + assertPermissions(containerFs.getPath("/sub/dir"), "r-x-w-rw-"); // Non-leafs get the same permission as the leaf + + // TODO: Uncomment when JimFS forwards attributes for SecureDirectoryStream::newByteChannel +// assertPermissions(Files.createFile(containerFs.getPath("/file1")), "rw-r-----"); +// assertPermissions(Files.createFile(containerFs.getPath("/file2"), permissionsFromString("r-x-w-rw-")), "r-x-w-rw-"); + } + private static void assertOwnership(ContainerPath path, int contUid, int contGid, int hostUid, int hostGid) throws IOException { assertOwnership(path, contUid, contGid); assertOwnership(path.pathOnHost(), hostUid, hostGid); @@ -184,4 +199,13 @@ class ContainerFileSystemTest { assertEquals(uid, attrs.get("uid")); assertEquals(gid, attrs.get("gid")); } + + private static void assertPermissions(Path path, String expected) throws IOException { + String actual = PosixFilePermissions.toString(Files.getPosixFilePermissions(path)); + assertEquals(expected, actual); + } + + private static FileAttribute<?> permissionsFromString(String permissions) { + return PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString(permissions)); + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java index 3274f12dbc6..1b0c7602f82 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/FailedExpirer.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.History; @@ -66,45 +67,49 @@ public class FailedExpirer extends NodeRepositoryMaintainer { @Override protected double maintain() { - List<Node> remainingNodes = new ArrayList<>(nodeRepository.nodes().list(Node.State.failed) - .nodeType(NodeType.tenant, NodeType.host) - .asList()); + NodeList allNodes = nodeRepository.nodes().list(); + List<Node> remainingNodes = new ArrayList<>(allNodes.state(Node.State.failed) + .nodeType(NodeType.tenant, NodeType.host) + .asList()); - recycleIf(remainingNodes, node -> node.allocation().isEmpty()); - recycleIf(remainingNodes, node -> - !node.allocation().get().membership().cluster().isStateful() && - node.history().hasEventBefore(History.Event.Type.failed, clock().instant().minus(statelessExpiry))); - recycleIf(remainingNodes, node -> - node.allocation().get().membership().cluster().isStateful() && - node.history().hasEventBefore(History.Event.Type.failed, clock().instant().minus(statefulExpiry))); + recycleIf(remainingNodes, + node -> node.allocation().isEmpty(), + allNodes); + recycleIf(remainingNodes, + node -> !node.allocation().get().membership().cluster().isStateful() && + node.history().hasEventBefore(History.Event.Type.failed, clock().instant().minus(statelessExpiry)), + allNodes); + recycleIf(remainingNodes, + node -> node.allocation().get().membership().cluster().isStateful() && + node.history().hasEventBefore(History.Event.Type.failed, clock().instant().minus(statefulExpiry)), + allNodes); return 1.0; } /** Recycle the nodes matching condition, and remove those nodes from the nodes list. */ - private void recycleIf(List<Node> nodes, Predicate<Node> recycleCondition) { - List<Node> nodesToRecycle = nodes.stream().filter(recycleCondition).collect(Collectors.toList()); - nodes.removeAll(nodesToRecycle); - recycle(nodesToRecycle); + private void recycleIf(List<Node> failedNodes, Predicate<Node> recycleCondition, NodeList allNodes) { + List<Node> nodesToRecycle = failedNodes.stream().filter(recycleCondition).collect(Collectors.toList()); + failedNodes.removeAll(nodesToRecycle); + recycle(nodesToRecycle, allNodes); } /** Move eligible nodes to dirty or parked. This may be a subset of the given nodes */ - private void recycle(List<Node> nodes) { + private void recycle(List<Node> nodes, NodeList allNodes) { List<Node> nodesToRecycle = new ArrayList<>(); for (Node candidate : nodes) { - if (broken(candidate)) { + if (broken(candidate, allNodes)) { List<String> unparkedChildren = !candidate.type().isHost() ? List.of() : - nodeRepository.nodes().list() - .childrenOf(candidate) - .not().state(Node.State.parked) - .mapToList(Node::hostname); + allNodes.childrenOf(candidate) + .not().state(Node.State.parked) + .mapToList(Node::hostname); if (unparkedChildren.isEmpty()) { nodeRepository.nodes().park(candidate.hostname(), false, Agent.FailedExpirer, "Parked by FailedExpirer due to hardware issue or high fail count"); } else { log.info(String.format("Expired failed node %s with hardware issue was not parked because of " + - "unparked children: %s", candidate.hostname(), - String.join(", ", unparkedChildren))); + "unparked children: %s", + candidate.hostname(), String.join(", ", unparkedChildren))); } } else { nodesToRecycle.add(candidate); @@ -114,8 +119,8 @@ public class FailedExpirer extends NodeRepositoryMaintainer { } /** Returns whether node is broken and cannot be recycled */ - private boolean broken(Node node) { - return NodeFailer.hasHardwareIssue(node, nodeRepository) || + private boolean broken(Node node, NodeList allNodes) { + return NodeFailer.hasHardwareIssue(node, allNodes) || (node.type().isHost() && node.status().failCount() >= maxAllowedFailures); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index 237cbaedf46..3c5b20da4d0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -164,7 +164,7 @@ public class NodeFailer extends NodeRepositoryMaintainer { for (Node node : activeNodes) { if (allSuspended(node, activeNodes)) { - Node host = node.parentHostname().flatMap(parent -> nodeRepository().nodes().node(parent)).orElse(node); + Node host = node.parentHostname().flatMap(parent -> activeNodes.node(parent)).orElse(node); if (host.type().isHost()) { List<String> failureReports = reasonsToFailHost(host); if ( ! failureReports.isEmpty()) { @@ -188,8 +188,8 @@ public class NodeFailer extends NodeRepositoryMaintainer { } /** Returns whether node has any kind of hardware issue */ - static boolean hasHardwareIssue(Node node, NodeRepository nodeRepository) { - Node host = node.parentHostname().flatMap(parent -> nodeRepository.nodes().node(parent)).orElse(node); + static boolean hasHardwareIssue(Node node, NodeList allNodes) { + Node host = node.parentHostname().flatMap(parent -> allNodes.node(parent)).orElse(node); return reasonsToFailHost(host).size() > 0; } @@ -207,10 +207,7 @@ public class NodeFailer extends NodeRepositoryMaintainer { private boolean allSuspended(Node node, NodeList activeNodes) { if (!nodeRepository().nodes().suspended(node)) return false; if (node.parentHostname().isPresent()) return true; // optimization - return activeNodes.stream() - .filter(childNode -> childNode.parentHostname().isPresent() && - childNode.parentHostname().get().equals(node.hostname())) - .allMatch(nodeRepository().nodes()::suspended); + return activeNodes.childrenOf(node.hostname()).stream().allMatch(nodeRepository().nodes()::suspended); } /** diff --git a/searchcommon/src/vespa/searchcommon/attribute/i_multi_value_attribute.h b/searchcommon/src/vespa/searchcommon/attribute/i_multi_value_attribute.h index 641e602e522..ea1fbe0b2b4 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/i_multi_value_attribute.h +++ b/searchcommon/src/vespa/searchcommon/attribute/i_multi_value_attribute.h @@ -18,15 +18,15 @@ namespace search::attribute { class IMultiValueAttribute { public: template<typename MultiValueType> - class Tag {}; + class MultiValueTag {}; template<typename T> - using ArrayTag = Tag<T>; + using ArrayTag = MultiValueTag<T>; using ArrayEnumTag = ArrayTag<vespalib::datastore::AtomicEntryRef>; template<typename T> - using WeightedSetTag = Tag<search::multivalue::WeightedValue<T>>; + using WeightedSetTag = MultiValueTag<search::multivalue::WeightedValue<T>>; using WeightedSetEnumTag = WeightedSetTag<vespalib::datastore::AtomicEntryRef>; diff --git a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp index 87e9faa4fe2..8555af7c685 100644 --- a/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/attributefieldvaluenode.cpp @@ -103,7 +103,7 @@ AttributeFieldValueNode::traceValue(const Context &context, std::ostream& out) c document::select::ValueNode::UP AttributeFieldValueNode::clone() const { - return wrapParens(new AttributeFieldValueNode(getDocType(), getFieldName(), _attr_guard_index)); + return wrapParens(std::make_unique<AttributeFieldValueNode>(getDocType(), getFieldName(), _attr_guard_index)); } } // namespace proton diff --git a/searchlib/src/tests/attribute/multi_value_read_view/multi_value_read_view_test.cpp b/searchlib/src/tests/attribute/multi_value_read_view/multi_value_read_view_test.cpp index c81fea79590..edebaa84ddb 100644 --- a/searchlib/src/tests/attribute/multi_value_read_view/multi_value_read_view_test.cpp +++ b/searchlib/src/tests/attribute/multi_value_read_view/multi_value_read_view_test.cpp @@ -214,7 +214,7 @@ MultiValueReadViewTest::check_values_helper(const IAttributeVector &attr, const vespalib::Stash stash; auto mv_attr = attr.as_multi_value_attribute(); EXPECT_NE(nullptr, mv_attr); - auto read_view = mv_attr->make_read_view(IMultiValueAttribute::Tag<MultiValueType>(), stash); + auto read_view = mv_attr->make_read_view(IMultiValueAttribute::MultiValueTag<MultiValueType>(), stash); EXPECT_NE(nullptr, read_view); bool is_imported = attr.isImported(); auto values = read_view->get_values(is_imported ? 4 : 1); diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 1d3305d2c1a..ec75a0d6d06 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -1041,7 +1041,6 @@ public: 100100.25, global_filter_lower_limit, 1.0); EXPECT_EQUAL(11u, bp->getState().estimate().estHits); - EXPECT_EQUAL(approximate, bp->may_approximate()); EXPECT_EQUAL(100100.25 * 100100.25, bp->get_distance_threshold()); return bp; } @@ -1068,7 +1067,6 @@ TEST_F("NN blueprint handles empty filter", NearestNeighborBlueprintFixture) auto empty_filter = GlobalFilter::create(); bp->set_global_filter(*empty_filter); EXPECT_EQUAL(3u, bp->getState().estimate().estHits); - EXPECT_TRUE(bp->may_approximate()); EXPECT_EQUAL(NNBA::INDEX_TOP_K, bp->get_algorithm()); } @@ -1081,7 +1079,6 @@ TEST_F("NN blueprint handles strong filter", NearestNeighborBlueprintFixture) auto strong_filter = GlobalFilter::create(std::move(filter)); bp->set_global_filter(*strong_filter); EXPECT_EQUAL(1u, bp->getState().estimate().estHits); - EXPECT_TRUE(bp->may_approximate()); EXPECT_EQUAL(NNBA::INDEX_TOP_K_WITH_FILTER, bp->get_algorithm()); } @@ -1099,7 +1096,6 @@ TEST_F("NN blueprint handles weak filter", NearestNeighborBlueprintFixture) auto weak_filter = GlobalFilter::create(std::move(filter)); bp->set_global_filter(*weak_filter); EXPECT_EQUAL(3u, bp->getState().estimate().estHits); - EXPECT_TRUE(bp->may_approximate()); EXPECT_EQUAL(NNBA::INDEX_TOP_K_WITH_FILTER, bp->get_algorithm()); } @@ -1112,7 +1108,6 @@ TEST_F("NN blueprint handles strong filter triggering brute force search", Neare auto strong_filter = GlobalFilter::create(std::move(filter)); bp->set_global_filter(*strong_filter); EXPECT_EQUAL(11u, bp->getState().estimate().estHits); - EXPECT_FALSE(bp->may_approximate()); EXPECT_EQUAL(NNBA::BRUTE_FORCE_FALLBACK, bp->get_algorithm()); } diff --git a/searchlib/src/tests/sortspec/multilevelsort.cpp b/searchlib/src/tests/sortspec/multilevelsort.cpp index 0154e95c155..9c1caaff662 100644 --- a/searchlib/src/tests/sortspec/multilevelsort.cpp +++ b/searchlib/src/tests/sortspec/multilevelsort.cpp @@ -9,6 +9,7 @@ #include <vespa/searchlib/uca/ucaconverter.h> #include <vespa/vespalib/util/testclock.h> #include <vespa/vespalib/testkit/testapp.h> +#include <type_traits> #include <vespa/log/log.h> LOG_SETUP("multilevelsort_test"); @@ -61,8 +62,8 @@ private: template<typename T> void fill(FloatingPointAttribute *attr, uint32_t size, uint32_t unique = 0); void fill(StringAttribute *attr, uint32_t size, const std::vector<std::string> &values); - template<typename T, typename V> - int compareTemplate(T *vector, uint32_t a, uint32_t b); + template <typename V> + int compareTemplate(AttributeVector *vector, uint32_t a, uint32_t b); int compare(AttributeVector *vector, AttrType type, uint32_t a, uint32_t b); void sortAndCheck(const std::vector<Spec> &spec, uint32_t num, uint32_t unique, const std::vector<std::string> &strValues); @@ -138,14 +139,23 @@ MultilevelSortTest::fill(StringAttribute *attr, uint32_t size, const std::vector } } -template<typename T, typename V> +template <typename V> +V get_helper(AttributeVector *vector, uint32_t doc_id) { + if constexpr (std::is_floating_point_v<V>) { + return vector->getFloat(doc_id); + } else { + return vector->getInt(doc_id); + } +} + +template <typename V> int -MultilevelSortTest::compareTemplate(T *vector, uint32_t a, uint32_t b) +MultilevelSortTest::compareTemplate(AttributeVector *vector, uint32_t a, uint32_t b) { V va; V vb; - vector->getAll(a, &va, 1); - vector->getAll(b, &vb, 1); + va = get_helper<V>(vector, a); + vb = get_helper<V>(vector, b); if (va == vb) { return 0; } else if (va < vb) { @@ -158,17 +168,17 @@ int MultilevelSortTest::compare(AttributeVector *vector, AttrType type, uint32_t a, uint32_t b) { if (type == INT8) { - return compareTemplate<Int8, int8_t>(static_cast<Int8*>(vector), a, b); + return compareTemplate<int8_t>(vector, a, b); } else if (type == INT16) { - return compareTemplate<Int16, int16_t>(static_cast<Int16*>(vector), a, b); + return compareTemplate<int16_t>(vector, a, b); } else if (type == INT32) { - return compareTemplate<Int32, int32_t>(static_cast<Int32*>(vector), a, b); + return compareTemplate<int32_t>(vector, a, b); } else if (type == INT64) { - return compareTemplate<Int64, int64_t>(static_cast<Int64*>(vector), a, b); + return compareTemplate<int64_t>(vector, a, b); } else if (type == FLOAT) { - return compareTemplate<Float, float>(static_cast<Float*>(vector), a, b); + return compareTemplate<float>(vector, a, b); } else if (type == DOUBLE) { - return compareTemplate<Double, double>(static_cast<Double*>(vector), a, b); + return compareTemplate<double>(vector, a, b); } else if (type == STRING) { StringAttribute *vString = static_cast<StringAttribute*>(vector); const char *va = vString->get(a); diff --git a/searchlib/src/vespa/searchlib/attribute/attrvector.h b/searchlib/src/vespa/searchlib/attribute/attrvector.h index 7713c210033..d1d2a1e8f3c 100644 --- a/searchlib/src/vespa/searchlib/attribute/attrvector.h +++ b/searchlib/src/vespa/searchlib/attribute/attrvector.h @@ -79,7 +79,6 @@ private: typedef typename B::WeightedFloat WeightedFloat; BaseType get(DocId doc) const override { return getHelper(doc, 0); } EnumHandle getEnum(DocId doc) const override { return getEnumHelper(doc, 0); } - uint32_t getAll(DocId doc, BaseType * v, uint32_t sz) const override { return getAllHelper<BaseType, BaseType>(doc, v, sz); } uint32_t get(DocId doc, EnumHandle * e, uint32_t sz) const override { return getAllEnumHelper(doc, e, sz); } uint32_t getValueCount(DocId doc) const override { return getValueCountHelper(doc); } @@ -125,7 +124,6 @@ private: } uint32_t get(DocId doc, WeightedEnum * v, uint32_t sz) const override { return getAllEnumHelper(doc, v, sz); } - uint32_t getAll(DocId doc, Weighted * v, uint32_t sz) const override { return getAllHelper<Weighted, BaseType>(doc, v, sz); } uint32_t get(DocId doc, WeightedInt * v, uint32_t sz) const override { return getAllHelper<WeightedInt, largeint_t>(doc, v, sz); } uint32_t get(DocId doc, WeightedFloat * v, uint32_t sz) const override { return getAllHelper<WeightedFloat, double>(doc, v, sz); } }; diff --git a/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp b/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp index 9c944a70b94..941450fccb5 100644 --- a/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp +++ b/searchlib/src/vespa/searchlib/attribute/extendableattributes.cpp @@ -54,15 +54,15 @@ MultiExtAttribute<T>::as_multi_value_attribute() const } template <typename T> -const attribute::IMultiValueReadView<T>* -MultiExtAttribute<T>::make_read_view(attribute::IMultiValueAttribute::Tag<T>, vespalib::Stash& stash) const +const attribute::IArrayReadView<T>* +MultiExtAttribute<T>::make_read_view(attribute::IMultiValueAttribute::ArrayTag<T>, vespalib::Stash& stash) const { return &stash.create<attribute::ExtendableNumericArrayMultiValueReadView<T, T>>(this->_data, this->_idx); } template <typename T> -const attribute::IMultiValueReadView<multivalue::WeightedValue<T>>* -MultiExtAttribute<T>::make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<T>>, vespalib::Stash& stash) const +const attribute::IWeightedSetReadView<T>* +MultiExtAttribute<T>::make_read_view(attribute::IMultiValueAttribute::WeightedSetTag<T>, vespalib::Stash& stash) const { return &stash.create<attribute::ExtendableNumericArrayMultiValueReadView<multivalue::WeightedValue<T>, T>>(this->_data, this->_idx); } @@ -110,14 +110,14 @@ MultiStringExtAttribute::as_multi_value_attribute() const return this; } -const attribute::IMultiValueReadView<const char*>* -MultiStringExtAttribute::make_read_view(attribute::IMultiValueAttribute::Tag<const char*>, vespalib::Stash& stash) const +const attribute::IArrayReadView<const char*>* +MultiStringExtAttribute::make_read_view(attribute::IMultiValueAttribute::ArrayTag<const char*>, vespalib::Stash& stash) const { return &stash.create<attribute::ExtendableStringArrayMultiValueReadView<const char*>>(this->_buffer, this->_offsets, this->_idx); } -const attribute::IMultiValueReadView<multivalue::WeightedValue<const char*>>* -MultiStringExtAttribute::make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<const char*>>, vespalib::Stash& stash) const +const attribute::IWeightedSetReadView<const char*>* +MultiStringExtAttribute::make_read_view(attribute::IMultiValueAttribute::WeightedSetTag<const char*>, vespalib::Stash& stash) const { return &stash.create<attribute::ExtendableStringArrayMultiValueReadView<multivalue::WeightedValue<const char*>>>(this->_buffer, this->_offsets, this->_idx); } @@ -150,8 +150,8 @@ WeightedSetIntegerExtAttribute::get(DocId doc, AttributeVector::WeightedInt * v, return valueCount; } -const attribute::IMultiValueReadView<multivalue::WeightedValue<int64_t>>* -WeightedSetIntegerExtAttribute::make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<int64_t>>, vespalib::Stash& stash) const +const attribute::IWeightedSetReadView<int64_t>* +WeightedSetIntegerExtAttribute::make_read_view(attribute::IMultiValueAttribute::WeightedSetTag<int64_t>, vespalib::Stash& stash) const { return &stash.create<attribute::ExtendableNumericWeightedSetMultiValueReadView<multivalue::WeightedValue<int64_t>, int64_t>>(this->_data, this->_idx, this->get_weights()); } @@ -182,8 +182,8 @@ WeightedSetFloatExtAttribute::get(DocId doc, AttributeVector::WeightedFloat * v, return valueCount; } -const attribute::IMultiValueReadView<multivalue::WeightedValue<double>>* -WeightedSetFloatExtAttribute::make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<double>>, vespalib::Stash& stash) const +const attribute::IWeightedSetReadView<double>* +WeightedSetFloatExtAttribute::make_read_view(attribute::IMultiValueAttribute::WeightedSetTag<double>, vespalib::Stash& stash) const { return &stash.create<attribute::ExtendableNumericWeightedSetMultiValueReadView<multivalue::WeightedValue<double>, double>>(this->_data, this->_idx, this->get_weights()); } @@ -216,8 +216,8 @@ WeightedSetStringExtAttribute::get(DocId doc, AttributeVector::WeightedConstChar return getAllHelper(doc, v, sz); } -const attribute::IMultiValueReadView<multivalue::WeightedValue<const char*>>* -WeightedSetStringExtAttribute::make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<const char*>>, vespalib::Stash& stash) const +const attribute::IWeightedSetReadView<const char*>* +WeightedSetStringExtAttribute::make_read_view(attribute::IMultiValueAttribute::WeightedSetTag<const char*>, vespalib::Stash& stash) const { return &stash.create<attribute::ExtendableStringWeightedSetMultiValueReadView<multivalue::WeightedValue<const char*>>>(this->_buffer, this->_offsets, this->_idx, this->get_weights()); } diff --git a/searchlib/src/vespa/searchlib/attribute/extendableattributes.h b/searchlib/src/vespa/searchlib/attribute/extendableattributes.h index ebd1d1d764c..86e6be8eff7 100644 --- a/searchlib/src/vespa/searchlib/attribute/extendableattributes.h +++ b/searchlib/src/vespa/searchlib/attribute/extendableattributes.h @@ -151,8 +151,8 @@ public: const attribute::IMultiValueAttribute* as_multi_value_attribute() const override; // Implements attribute::IMultiValueAttribute - const attribute::IMultiValueReadView<T>* make_read_view(attribute::IMultiValueAttribute::Tag<T>, vespalib::Stash& stash) const override; - const attribute::IMultiValueReadView<multivalue::WeightedValue<T>>* make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<T>>, vespalib::Stash& stash) const override; + const attribute::IArrayReadView<T>* make_read_view(attribute::IMultiValueAttribute::ArrayTag<T>, vespalib::Stash& stash) const override; + const attribute::IWeightedSetReadView<T>* make_read_view(attribute::IMultiValueAttribute::WeightedSetTag<T>, vespalib::Stash& stash) const override; }; template <typename T> @@ -185,8 +185,8 @@ public: void onAddDocs(DocId ) override { } const attribute::IMultiValueAttribute* as_multi_value_attribute() const override; // Implements attribute::IMultiValueAttribute - const attribute::IMultiValueReadView<const char*>* make_read_view(attribute::IMultiValueAttribute::Tag<const char*>, vespalib::Stash& stash) const override; - const attribute::IMultiValueReadView<multivalue::WeightedValue<const char*>>* make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<const char*>>, vespalib::Stash& stash) const override; + const attribute::IArrayReadView<const char*>* make_read_view(attribute::IMultiValueAttribute::ArrayTag<const char*>, vespalib::Stash& stash) const override; + const attribute::IWeightedSetReadView<const char*>* make_read_view(attribute::IMultiValueAttribute::WeightedSetTag<const char*>, vespalib::Stash& stash) const override; }; @@ -229,7 +229,7 @@ public: bool add(int64_t v, int32_t w = 1) override; uint32_t get(DocId doc, AttributeVector::WeightedInt * v, uint32_t sz) const override; // Implements attribute::IMultiValueAttribute - const attribute::IMultiValueReadView<multivalue::WeightedValue<int64_t>>* make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<int64_t>>, vespalib::Stash& stash) const override; + const attribute::IWeightedSetReadView<int64_t>* make_read_view(attribute::IMultiValueAttribute::WeightedSetTag<int64_t>, vespalib::Stash& stash) const override; }; class WeightedSetFloatExtAttribute @@ -248,7 +248,7 @@ public: bool add(double v, int32_t w = 1) override; uint32_t get(DocId doc, AttributeVector::WeightedFloat * v, uint32_t sz) const override; // Implements attribute::IMultiValueAttribute - const attribute::IMultiValueReadView<multivalue::WeightedValue<double>>* make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<double>>, vespalib::Stash& stash) const override; + const attribute::IWeightedSetReadView<double>* make_read_view(attribute::IMultiValueAttribute::WeightedSetTag<double>, vespalib::Stash& stash) const override; }; class WeightedSetStringExtAttribute @@ -276,7 +276,7 @@ public: uint32_t get(DocId doc, AttributeVector::WeightedString * v, uint32_t sz) const override; uint32_t get(DocId doc, AttributeVector::WeightedConstChar * v, uint32_t sz) const override; // Implements attribute::IMultiValueAttribute - const attribute::IMultiValueReadView<multivalue::WeightedValue<const char*>>* make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<const char*>>, vespalib::Stash& stash) const override; + const attribute::IWeightedSetReadView<const char*>* make_read_view(attribute::IMultiValueAttribute::WeightedSetTag<const char*>, vespalib::Stash& stash) const override; }; } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/floatbase.h b/searchlib/src/vespa/searchlib/attribute/floatbase.h index f10e8189c29..8ca6eda6421 100644 --- a/searchlib/src/vespa/searchlib/attribute/floatbase.h +++ b/searchlib/src/vespa/searchlib/attribute/floatbase.h @@ -50,8 +50,6 @@ class FloatingPointAttributeTemplate : public FloatingPointAttribute { public: using Weighted = WeightedType<T>; - virtual uint32_t getAll(DocId doc, T * v, uint32_t sz) const = 0; - virtual uint32_t getAll(DocId doc, Weighted * v, uint32_t sz) const = 0; protected: using EnumEntryType = T; using LoadedNumericValueT = attribute::LoadedNumericValue<T>; diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp index c8a6309c282..a6a0dac9097 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.cpp @@ -160,7 +160,7 @@ bool ImportedAttributeVectorReadGuard::isImported() const template <typename MultiValueType> const IMultiValueReadView<MultiValueType>* -ImportedAttributeVectorReadGuard::make_read_view_helper(Tag<MultiValueType> tag, vespalib::Stash &stash) const +ImportedAttributeVectorReadGuard::make_read_view_helper(MultiValueTag<MultiValueType> tag, vespalib::Stash &stash) const { auto target_mv_attr = _target_attribute.as_multi_value_attribute(); if (target_mv_attr == nullptr) { diff --git a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h index 9e596078678..233ce5d06df 100644 --- a/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h +++ b/searchlib/src/vespa/searchlib/attribute/imported_attribute_vector_read_guard.h @@ -89,7 +89,7 @@ public: bool isImported() const override; bool isUndefined(DocId doc) const override; template <typename MultiValueType> - const IMultiValueReadView<MultiValueType>* make_read_view_helper(Tag<MultiValueType> tag, vespalib::Stash& stash) const; + const IMultiValueReadView<MultiValueType>* make_read_view_helper(MultiValueTag<MultiValueType> tag, vespalib::Stash& stash) const; const IArrayReadView<int8_t>* make_read_view(ArrayTag<int8_t> tag, vespalib::Stash& stash) const override; const IArrayReadView<int16_t>* make_read_view(ArrayTag<int16_t> tag, vespalib::Stash& stash) const override; const IArrayReadView<int32_t>* make_read_view(ArrayTag<int32_t> tag, vespalib::Stash& stash) const override; diff --git a/searchlib/src/vespa/searchlib/attribute/integerbase.h b/searchlib/src/vespa/searchlib/attribute/integerbase.h index 926ced7c7e0..65d16ce934a 100644 --- a/searchlib/src/vespa/searchlib/attribute/integerbase.h +++ b/searchlib/src/vespa/searchlib/attribute/integerbase.h @@ -49,8 +49,6 @@ class IntegerAttributeTemplate : public IntegerAttribute { public: using Weighted = WeightedType<T>; - virtual uint32_t getAll(DocId doc, T * v, uint32_t sz) const = 0; - virtual uint32_t getAll(DocId doc, Weighted * v, uint32_t sz) const = 0; protected: using EnumEntryType = T; using LoadedNumericValueT = attribute::LoadedNumericValue<T>; diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.h b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.h index e07498d9ca4..cc128b0eef1 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multinumericattribute.h @@ -91,9 +91,6 @@ public: (void) doc; return std::numeric_limits<uint32_t>::max(); // does not have enum } - uint32_t getAll(DocId doc, T * v, uint32_t sz) const override { - return getHelper(doc, v, sz); - } uint32_t get(DocId doc, largeint_t * v, uint32_t sz) const override { return getHelper(doc, v, sz); } @@ -125,9 +122,6 @@ public: } return available; } - uint32_t getAll(DocId doc, Weighted * v, uint32_t sz) const override{ - return getWeightedHelper<Weighted, T>(doc, v, sz); - } uint32_t get(DocId doc, WeightedInt * v, uint32_t sz) const override { return getWeightedHelper<WeightedInt, largeint_t>(doc, v, sz); } diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h index 9f8506d3cb4..82bb98d5cf2 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.h @@ -77,9 +77,6 @@ public: } return valueCount; } - uint32_t getAll(DocId doc, T * v, uint32_t sz) const override { - return getHelper(doc, v, sz); - } uint32_t get(DocId doc, largeint_t * v, uint32_t sz) const override { return getHelper(doc, v, sz); } @@ -96,9 +93,6 @@ public: } return valueCount; } - uint32_t getAll(DocId doc, Weighted * v, uint32_t sz) const override { - return getWeightedHelper<Weighted, T>(doc, v, sz); - } uint32_t get(DocId doc, WeightedInt * v, uint32_t sz) const override { return getWeightedHelper<WeightedInt, largeint_t>(doc, v, sz); } @@ -107,8 +101,8 @@ public: } // Implements attribute::IMultiValueAttribute - const attribute::IMultiValueReadView<T>* make_read_view(attribute::IMultiValueAttribute::Tag<T>, vespalib::Stash& stash) const override; - const attribute::IMultiValueReadView<multivalue::WeightedValue<T>>* make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<T>>, vespalib::Stash& stash) const override; + const attribute::IArrayReadView<T>* make_read_view(attribute::IMultiValueAttribute::ArrayTag<T>, vespalib::Stash& stash) const override; + const attribute::IWeightedSetReadView<T>* make_read_view(attribute::IMultiValueAttribute::WeightedSetTag<T>, vespalib::Stash& stash) const override; private: using AttributeReader = PrimitiveReader<typename B::LoadedValueType>; diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp index 81f8c1c910e..36c91a12498 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp @@ -116,15 +116,15 @@ MultiValueNumericEnumAttribute<B, M>::onLoad(vespalib::Executor *) } template <typename B, typename M> -const attribute::IMultiValueReadView<typename B::BaseClass::BaseType>* -MultiValueNumericEnumAttribute<B, M>::make_read_view(attribute::IMultiValueAttribute::Tag<typename B::BaseClass::BaseType>, vespalib::Stash& stash) const +const attribute::IArrayReadView<typename B::BaseClass::BaseType>* +MultiValueNumericEnumAttribute<B, M>::make_read_view(attribute::IMultiValueAttribute::ArrayTag<typename B::BaseClass::BaseType>, vespalib::Stash& stash) const { return &stash.create<attribute::EnumeratedMultiValueReadView<T, M>>(this->_mvMapping.make_read_view(this->getCommittedDocIdLimit()), this->_enumStore); } template <typename B, typename M> -const attribute::IMultiValueReadView<multivalue::WeightedValue<typename B::BaseClass::BaseType>>* -MultiValueNumericEnumAttribute<B, M>::make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<typename B::BaseClass::BaseType>>, vespalib::Stash& stash) const +const attribute::IWeightedSetReadView<typename B::BaseClass::BaseType>* +MultiValueNumericEnumAttribute<B, M>::make_read_view(attribute::IMultiValueAttribute::WeightedSetTag<typename B::BaseClass::BaseType>, vespalib::Stash& stash) const { return &stash.create<attribute::EnumeratedMultiValueReadView<multivalue::WeightedValue<T>, M>>(this->_mvMapping.make_read_view(this->getCommittedDocIdLimit()), this->_enumStore); } diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.h b/searchlib/src/vespa/searchlib/attribute/multistringattribute.h index 532af930220..4c3eeeed232 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.h @@ -107,8 +107,8 @@ public: getSearch(QueryTermSimpleUP term, const attribute::SearchContextParams & params) const override; // Implements attribute::IMultiValueAttribute - const attribute::IMultiValueReadView<const char*>* make_read_view(attribute::IMultiValueAttribute::Tag<const char*>, vespalib::Stash& stash) const override; - const attribute::IMultiValueReadView<multivalue::WeightedValue<const char*>>* make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<const char*>>, vespalib::Stash& stash) const override; + const attribute::IArrayReadView<const char*>* make_read_view(attribute::IMultiValueAttribute::ArrayTag<const char*>, vespalib::Stash& stash) const override; + const attribute::IWeightedSetReadView<const char*>* make_read_view(attribute::IMultiValueAttribute::WeightedSetTag<const char*>, vespalib::Stash& stash) const override; }; diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp index 0edd459efc7..c3f8f99b4ab 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp @@ -50,15 +50,15 @@ MultiValueStringAttributeT<B, M>::getSearch(QueryTermSimpleUP qTerm, } template <typename B, typename M> -const attribute::IMultiValueReadView<const char*>* -MultiValueStringAttributeT<B, M>::make_read_view(attribute::IMultiValueAttribute::Tag<const char*>, vespalib::Stash& stash) const +const attribute::IArrayReadView<const char*>* +MultiValueStringAttributeT<B, M>::make_read_view(attribute::IMultiValueAttribute::ArrayTag<const char*>, vespalib::Stash& stash) const { return &stash.create<attribute::EnumeratedMultiValueReadView<const char*, M>>(this->_mvMapping.make_read_view(this->getCommittedDocIdLimit()), this->_enumStore); } template <typename B, typename M> -const attribute::IMultiValueReadView<multivalue::WeightedValue<const char*>>* -MultiValueStringAttributeT<B, M>::make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<const char*>>, vespalib::Stash& stash) const +const attribute::IWeightedSetReadView<const char*>* +MultiValueStringAttributeT<B, M>::make_read_view(attribute::IMultiValueAttribute::WeightedSetTag<const char*>, vespalib::Stash& stash) const { return &stash.create<attribute::EnumeratedMultiValueReadView<multivalue::WeightedValue<const char*>, M>>(this->_mvMapping.make_read_view(this->getCommittedDocIdLimit()), this->_enumStore); } diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h index 5a2ee5c80d9..1d0e8d04ca8 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.h @@ -82,8 +82,8 @@ public: const IMultiValueAttribute* as_multi_value_attribute() const override; // Implements attribute::IMultiValueAttribute - const attribute::IMultiValueReadView<ValueType>* make_read_view(attribute::IMultiValueAttribute::Tag<ValueType>, vespalib::Stash& stash) const override; - const attribute::IMultiValueReadView<multivalue::WeightedValue<ValueType>>* make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<ValueType>>, vespalib::Stash& stash) const override; + const attribute::IArrayReadView<ValueType>* make_read_view(attribute::IMultiValueAttribute::ArrayTag<ValueType>, vespalib::Stash& stash) const override; + const attribute::IWeightedSetReadView<ValueType>* make_read_view(attribute::IMultiValueAttribute::WeightedSetTag<ValueType>, vespalib::Stash& stash) const override; }; } // namespace search diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp index 4e0e460da9c..622058550a2 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattribute.hpp @@ -300,8 +300,8 @@ MultiValueAttribute<B, M>::as_multi_value_attribute() const } template <typename B, typename M> -const attribute::IMultiValueReadView<multivalue::ValueType_t<M>>* -MultiValueAttribute<B, M>::make_read_view(attribute::IMultiValueAttribute::Tag<ValueType>, vespalib::Stash& stash) const +const attribute::IArrayReadView<multivalue::ValueType_t<M>>* +MultiValueAttribute<B, M>::make_read_view(attribute::IMultiValueAttribute::ArrayTag<ValueType>, vespalib::Stash& stash) const { if constexpr (std::is_same_v<MultiValueType, ValueType>) { return &stash.create<attribute::RawMultiValueReadView<MultiValueType>>(this->_mvMapping.make_read_view(this->getCommittedDocIdLimit())); @@ -311,8 +311,8 @@ MultiValueAttribute<B, M>::make_read_view(attribute::IMultiValueAttribute::Tag<V } template <typename B, typename M> -const attribute::IMultiValueReadView<multivalue::WeightedValue<multivalue::ValueType_t<M>>>* -MultiValueAttribute<B, M>::make_read_view(attribute::IMultiValueAttribute::Tag<multivalue::WeightedValue<ValueType>>, vespalib::Stash& stash) const +const attribute::IWeightedSetReadView<multivalue::ValueType_t<M>>* +MultiValueAttribute<B, M>::make_read_view(attribute::IMultiValueAttribute::WeightedSetTag<ValueType>, vespalib::Stash& stash) const { if constexpr (std::is_same_v<MultiValueType, multivalue::WeightedValue<ValueType>>) { return &stash.create<attribute::RawMultiValueReadView<MultiValueType>>(this->_mvMapping.make_read_view(this->getCommittedDocIdLimit())); diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h index f7efe27277e..77c8b7e318f 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h @@ -45,12 +45,6 @@ public: uint32_t getEnum(DocId) const override { return std::numeric_limits<uint32_t>::max(); // does not have enum } - uint32_t getAll(DocId doc, int8_t * v, uint32_t sz) const override { - if (sz > 0) { - v[0] = getFast(doc); - } - return 1; - } uint32_t get(DocId doc, largeint_t * v, uint32_t sz) const override { if (sz > 0) { v[0] = static_cast<largeint_t>(getFast(doc)); @@ -69,7 +63,6 @@ public: } return 1; } - uint32_t getAll(DocId, Weighted *, uint32_t) const override { return 0; } uint32_t get(DocId doc, WeightedInt * v, uint32_t sz) const override { if (sz > 0) { v[0] = WeightedInt(static_cast<largeint_t>(getFast(doc))); diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.h b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.h index 71a5f4f738e..cf3fa85a060 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.h @@ -93,11 +93,6 @@ public: (void) doc; return std::numeric_limits<uint32_t>::max(); // does not have enum } - uint32_t getAll(DocId doc, T * v, uint32_t sz) const override { - (void) sz; - v[0] = getFast(doc); - return 1; - } uint32_t get(DocId doc, largeint_t * v, uint32_t sz) const override { (void) sz; v[0] = static_cast<largeint_t>(getFast(doc)); @@ -113,10 +108,6 @@ public: e[0] = getEnum(doc); return 1; } - uint32_t getAll(DocId doc, Weighted * v, uint32_t sz) const override { - (void) doc; (void) v; (void) sz; - return 0; - } uint32_t get(DocId doc, WeightedInt * v, uint32_t sz) const override { (void) sz; v[0] = WeightedInt(static_cast<largeint_t>(getFast(doc))); diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.h b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.h index a269aec5c6b..5b0e1c6131e 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.h @@ -74,12 +74,6 @@ public: double getFloat(DocId doc) const override { return static_cast<double>(get(doc)); } - uint32_t getAll(DocId doc, T * v, uint32_t sz) const override { - if (sz > 0) { - v[0] = get(doc); - } - return 1; - } uint32_t get(DocId doc, largeint_t * v, uint32_t sz) const override { if (sz > 0) { v[0] = getInt(doc); @@ -92,12 +86,6 @@ public: } return 1; } - uint32_t getAll(DocId doc, Weighted * v, uint32_t sz) const override { - if (sz > 0) { - v[0] = Weighted(get(doc)); - } - return 1; - } uint32_t get(DocId doc, WeightedInt * v, uint32_t sz) const override { if (sz > 0) { v[0] = WeightedInt(getInt(doc)); diff --git a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.h b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.h index f6059d3d510..646edc786a3 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singlesmallnumericattribute.h @@ -100,12 +100,6 @@ public: uint32_t getEnum(DocId) const override { return std::numeric_limits<uint32_t>::max(); // does not have enum } - uint32_t getAll(DocId doc, T * v, uint32_t sz) const override { - if (sz > 0) { - v[0] = getFast(doc); - } - return 1; - } uint32_t get(DocId doc, largeint_t * v, uint32_t sz) const override { if (sz > 0) { v[0] = static_cast<largeint_t>(getFast(doc)); @@ -124,7 +118,6 @@ public: } return 1; } - uint32_t getAll(DocId, Weighted *, uint32_t) const override { return 0; } uint32_t get(DocId doc, WeightedInt * v, uint32_t sz) const override { if (sz > 0) { v[0] = WeightedInt(static_cast<largeint_t>(getFast(doc))); diff --git a/searchlib/src/vespa/searchlib/features/attributefeature.cpp b/searchlib/src/vespa/searchlib/features/attributefeature.cpp index 3b768633dd3..3e39ea830ca 100644 --- a/searchlib/src/vespa/searchlib/features/attributefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/attributefeature.cpp @@ -344,7 +344,7 @@ struct MultiValueExecutorCreator { bool handle(vespalib::Stash &stash, const IAttributeVector *attribute) { auto multi_value_attribute = attribute->as_multi_value_attribute(); if (multi_value_attribute != nullptr) { - _array_read_view = multi_value_attribute->make_read_view(attribute::IMultiValueAttribute::Tag<typename T::BaseType>(), stash); + _array_read_view = multi_value_attribute->make_read_view(attribute::IMultiValueAttribute::ArrayTag<typename T::BaseType>(), stash); } return _array_read_view != nullptr; } diff --git a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp index 98f276d1fed..ce7df5c9fb7 100644 --- a/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/dotproductfeature.cpp @@ -390,7 +390,7 @@ make_multi_value_read_view(const IAttributeVector& attribute, vespalib::Stash& s { auto multi_value_attribute = attribute.as_multi_value_attribute(); if (multi_value_attribute != nullptr) { - return multi_value_attribute->make_read_view(attribute::IMultiValueAttribute::Tag<AT>(), stash); + return multi_value_attribute->make_read_view(attribute::IMultiValueAttribute::MultiValueTag<AT>(), stash); } return nullptr; } diff --git a/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp index 5e3dc727279..49d4149953a 100644 --- a/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/internal_max_reduce_prod_join_feature.cpp @@ -144,10 +144,9 @@ template<typename BaseType, typename V> FeatureExecutor & selectTypedExecutor(const IAttributeVector *attribute, V && vector, vespalib::Stash &stash) { if (!attribute->isImported()) { - using VT = BaseType; auto multi_value_attribute = attribute->as_multi_value_attribute(); if (multi_value_attribute != nullptr) { - auto array_read_view = multi_value_attribute->make_read_view(attribute::IMultiValueAttribute::Tag<VT>(), stash); + auto array_read_view = multi_value_attribute->make_read_view(attribute::IMultiValueAttribute::ArrayTag<BaseType>(), stash); if (array_read_view != nullptr) { return stash.create<RawExecutor<BaseType>>(array_read_view, std::forward<V>(vector)); } diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp index bdcbb3db633..73eaa773c53 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.cpp @@ -89,6 +89,7 @@ NearestNeighborBlueprint::NearestNeighborBlueprint(const queryeval::FieldSpec& f _found_hits(), _algorithm(Algorithm::BRUTE_FORCE), _global_filter(GlobalFilter::create()), + _global_filter_set(false), _global_filter_hits(), _global_filter_hit_ratio() { @@ -122,51 +123,41 @@ void NearestNeighborBlueprint::set_global_filter(const GlobalFilter &global_filter) { _global_filter = global_filter.shared_from_this(); + _global_filter_set = true; auto nns_index = _attr_tensor.nearest_neighbor_index(); - LOG(debug, "set_global_filter with: %s / %s / %s", - (_approximate ? "approximate" : "exact"), - (nns_index ? "nns_index" : "no_index"), - (_global_filter->has_filter() ? "has_filter" : "no_filter")); if (_approximate && nns_index) { uint32_t est_hits = _attr_tensor.get_num_docs(); if (_global_filter->has_filter()) { uint32_t max_hits = _global_filter->filter()->countTrueBits(); - LOG(debug, "set_global_filter getNumDocs: %u / max_hits %u", est_hits, max_hits); double max_hit_ratio = static_cast<double>(max_hits) / est_hits; if (max_hit_ratio < _global_filter_lower_limit) { - _approximate = false; _algorithm = Algorithm::BRUTE_FORCE_FALLBACK; - LOG(debug, "too many hits filtered out, using brute force implementation"); } else { est_hits = std::min(est_hits, max_hits); } _global_filter_hits = max_hits; _global_filter_hit_ratio = max_hit_ratio; } - if (_approximate) { + if (_algorithm != Algorithm::BRUTE_FORCE_FALLBACK) { est_hits = std::min(est_hits, _target_num_hits); setEstimate(HitEstimate(est_hits, false)); - perform_top_k(); - LOG(debug, "perform_top_k found %zu hits", _found_hits.size()); + perform_top_k(nns_index); } } } void -NearestNeighborBlueprint::perform_top_k() +NearestNeighborBlueprint::perform_top_k(const search::tensor::NearestNeighborIndex* nns_index) { - auto nns_index = _attr_tensor.nearest_neighbor_index(); - if (_approximate && nns_index) { - auto lhs = _query_tensor->cells(); - uint32_t k = _target_num_hits; - if (_global_filter->has_filter()) { - auto filter = _global_filter->filter(); - _found_hits = nns_index->find_top_k_with_filter(k, lhs, *filter, k + _explore_additional_hits, _distance_threshold); - _algorithm = Algorithm::INDEX_TOP_K_WITH_FILTER; - } else { - _found_hits = nns_index->find_top_k(k, lhs, k + _explore_additional_hits, _distance_threshold); - _algorithm = Algorithm::INDEX_TOP_K; - } + auto lhs = _query_tensor->cells(); + uint32_t k = _target_num_hits; + if (_global_filter->has_filter()) { + auto filter = _global_filter->filter(); + _found_hits = nns_index->find_top_k_with_filter(k, lhs, *filter, k + _explore_additional_hits, _distance_threshold); + _algorithm = Algorithm::INDEX_TOP_K_WITH_FILTER; + } else { + _found_hits = nns_index->find_top_k(k, lhs, k + _explore_additional_hits, _distance_threshold); + _algorithm = Algorithm::INDEX_TOP_K; } } @@ -191,14 +182,15 @@ NearestNeighborBlueprint::visitMembers(vespalib::ObjectVisitor& visitor) const visitor.visitString("query_tensor", _query_tensor->type().to_spec()); visitor.visitInt("target_num_hits", _target_num_hits); visitor.visitInt("explore_additional_hits", _explore_additional_hits); - visitor.visitBool("approximate", _approximate); + visitor.visitBool("wanted_approximate", _approximate); visitor.visitBool("has_index", _attr_tensor.nearest_neighbor_index()); visitor.visitString("algorithm", to_string(_algorithm)); visitor.visitInt("top_k_hits", _found_hits.size()); visitor.openStruct("global_filter", "GlobalFilter"); - visitor.visitBool("is_set", (_global_filter != nullptr)); - visitor.visitBool("has_filter", (_global_filter && _global_filter->has_filter())); + visitor.visitBool("wanted", getState().want_global_filter()); + visitor.visitBool("set", _global_filter_set); + visitor.visitBool("calculated", _global_filter->has_filter()); visitor.visitFloat("lower_limit", _global_filter_lower_limit); visitor.visitFloat("upper_limit", _global_filter_upper_limit); if (_global_filter_hits.has_value()) { diff --git a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h index 7922036dc42..7637c4dd6b7 100644 --- a/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/nearest_neighbor_blueprint.h @@ -41,10 +41,11 @@ private: std::vector<search::tensor::NearestNeighborIndex::Neighbor> _found_hits; Algorithm _algorithm; std::shared_ptr<const GlobalFilter> _global_filter; + bool _global_filter_set; std::optional<uint32_t> _global_filter_hits; std::optional<double> _global_filter_hit_ratio; - void perform_top_k(); + void perform_top_k(const search::tensor::NearestNeighborIndex* nns_index); public: NearestNeighborBlueprint(const queryeval::FieldSpec& field, const tensor::ITensorAttribute& attr_tensor, @@ -60,7 +61,6 @@ public: const vespalib::eval::Value& get_query_tensor() const { return *_query_tensor; } uint32_t get_target_num_hits() const { return _target_num_hits; } void set_global_filter(const GlobalFilter &global_filter) override; - bool may_approximate() const { return _approximate; } Algorithm get_algorithm() const { return _algorithm; } double get_distance_threshold() const { return _distance_threshold; } diff --git a/storage/src/vespa/storage/persistence/fieldvisitor.h b/storage/src/vespa/storage/persistence/fieldvisitor.h index 3216a360c82..93782d3fbe2 100644 --- a/storage/src/vespa/storage/persistence/fieldvisitor.h +++ b/storage/src/vespa/storage/persistence/fieldvisitor.h @@ -45,6 +45,7 @@ public: void visitFloatValueNode(const document::select::FloatValueNode &) override {} void visitVariableValueNode(const document::select::VariableValueNode &) override {} void visitIntegerValueNode(const document::select::IntegerValueNode &) override {} + void visitBoolValueNode(const document::select::BoolValueNode &) override {} void visitCurrentTimeValueNode(const document::select::CurrentTimeValueNode &) override {} void visitStringValueNode(const document::select::StringValueNode &) override {} void visitNullValueNode(const document::select::NullValueNode &) override {} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java index 136ae1df8ae..8ffb9331ddb 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java @@ -23,6 +23,7 @@ import com.yahoo.vespa.athenz.client.zms.bindings.ResponseListEntity; import com.yahoo.vespa.athenz.client.zms.bindings.RoleEntity; import com.yahoo.vespa.athenz.client.zms.bindings.ServiceEntity; import com.yahoo.vespa.athenz.client.zms.bindings.ServiceListResponseEntity; +import com.yahoo.vespa.athenz.client.zms.bindings.StatisticsEntity; import com.yahoo.vespa.athenz.client.zms.bindings.TenancyRequestEntity; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.utils.AthenzIdentities; @@ -408,6 +409,17 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { execute(request, response -> readEntity(response, Void.class)); } + @Override + public QuotaUsage getQuotaUsage() { + var uri = zmsUrl.resolve(String.format("domain/%s/quota", identity.getDomainName())); + var quotaEntity = execute(RequestBuilder.get(uri).build(), response -> readEntity(response, StatisticsEntity.class)); + + uri = zmsUrl.resolve(String.format("domain/%s/stats", identity.getDomainName())); + var usageEntity = execute(RequestBuilder.get(uri).build(), response -> readEntity(response, StatisticsEntity.class)); + + return QuotaUsage.calculateUsage(usageEntity, quotaEntity); + } + public AthenzRoleInformation getFullRoleInformation(AthenzRole role) { var uri = zmsUrl.resolve(String.format("domain/%s/role/%s?pending=true&auditLog=true", role.domain().getName(), role.roleName())); var request = RequestBuilder.get(uri).build(); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/QuotaUsage.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/QuotaUsage.java new file mode 100644 index 00000000000..8e9c7de9272 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/QuotaUsage.java @@ -0,0 +1,55 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zms; + +import com.yahoo.vespa.athenz.client.zms.bindings.StatisticsEntity; + +/** + * @author olaa + */ +public class QuotaUsage { + + private double subdomainUsage; + private double roleUsage; + private double policyUsage; + private double serviceUsage; + private double groupUsage; + + + public QuotaUsage(double subdomainUsage, double roleUsage, double policyUsage, double serviceUsage, double groupUsage) { + this.subdomainUsage = subdomainUsage; + this.roleUsage = roleUsage; + this.policyUsage = policyUsage; + this.serviceUsage = serviceUsage; + this.groupUsage = groupUsage; + } + + public static QuotaUsage calculateUsage(StatisticsEntity used, StatisticsEntity quota) { + return new QuotaUsage( + (double) used.getSubdomains() / quota.getSubdomains(), + (double) used.getRoles() / quota.getRoles(), + (double) used.getPolicies() / quota.getPolicies(), + (double) used.getServices() / quota.getServices(), + (double) used.getGroups() / quota.getGroups() + ); + } + + public double getSubdomainUsage() { + return subdomainUsage; + } + + public double getRoleUsage() { + return roleUsage; + } + + public double getPolicyUsage() { + return policyUsage; + } + + public double getServiceUsage() { + return serviceUsage; + } + + public double getGroupUsage() { + return groupUsage; + } +}
\ No newline at end of file diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java index 3ff2ff843a0..b07f6da1a01 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java @@ -83,5 +83,7 @@ public interface ZmsClient extends AutoCloseable { AthenzRoleInformation getFullRoleInformation(AthenzRole role); + QuotaUsage getQuotaUsage(); + void close(); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/StatisticsEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/StatisticsEntity.java new file mode 100644 index 00000000000..bba6195363c --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/StatisticsEntity.java @@ -0,0 +1,51 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zms.bindings; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author olaa + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class StatisticsEntity { + + private int subdomains; + private int roles; + private int policies; + private int services; + private int groups; + + public StatisticsEntity(@JsonProperty("subdomain") int subdomains, + @JsonProperty("role") int roles, + @JsonProperty("policy") int policies, + @JsonProperty("service") int services, + @JsonProperty("group") int groups) { + this.subdomains = subdomains; + this.roles = roles; + this.policies = policies; + this.services = services; + this.groups = groups; + } + + public int getSubdomains() { + return subdomains; + } + + public int getRoles() { + return roles; + } + + public int getPolicies() { + return policies; + } + + public int getServices() { + return services; + } + + public int getGroups() { + return groups; + } + +}
\ No newline at end of file |