diff options
author | Håkon Hallingstad <hakon@oath.com> | 2018-11-05 09:01:08 +0100 |
---|---|---|
committer | Håkon Hallingstad <hakon@oath.com> | 2018-11-05 09:01:08 +0100 |
commit | fb37b7df39bd3e8a422f5087264e4a1accb02ab7 (patch) | |
tree | ad0cbec8e5961dd1bb2bd57fa4a9f8b14b831312 /orchestrator/src/main | |
parent | d4572d3347aafa7061ed6104eb56ec5d63e7e9f8 (diff) |
Send probe when suspending many nodes
When suspending all nodes on a host, first do a suspend-all probe that will try
to suspend the nodes as normal in Orchestrator and cluster controller, but
actually not commit anything. A probe failure will result in the same failure
as a non-probe failure: A 409 response with description is sent back to the
client.
Diffstat (limited to 'orchestrator/src/main')
11 files changed, 85 insertions, 77 deletions
diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java index f1f572621ce..d3bdaa6dc64 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java @@ -22,16 +22,16 @@ public class OrchestratorContext { private final Clock clock; private final TimeBudget timeBudget; - private boolean probe; + private final boolean probe; /** Create an OrchestratorContext for operations on multiple applications. */ public static OrchestratorContext createContextForMultiAppOp(Clock clock) { - return new OrchestratorContext(clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_BATCH_OP), true); + return new OrchestratorContext(clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_BATCH_OP), false); } /** Create an OrchestratorContext for an operation on a single application. */ public static OrchestratorContext createContextForSingleAppOp(Clock clock) { - return new OrchestratorContext(clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_SINGLE_OP), true); + return new OrchestratorContext(clock, TimeBudget.fromNow(clock, DEFAULT_TIMEOUT_FOR_SINGLE_OP), false); } private OrchestratorContext(Clock clock, TimeBudget timeBudget, boolean probe) { @@ -48,13 +48,6 @@ public class OrchestratorContext { return new ClusterControllerClientTimeouts(timeBudget.timeLeftAsTimeBudget()); } - - /** Mark this operation as a non-committal probe. */ - public OrchestratorContext markAsProbe() { - this.probe = true; - return this; - } - /** Whether the operation is a no-op probe to test whether it would have succeeded, if it had been committal. */ public boolean isProbe() { return probe; @@ -69,7 +62,7 @@ public class OrchestratorContext { } /** Create an OrchestratorContext for an operation on a single application, but limited to current timeout. */ - public OrchestratorContext createSubcontextForSingleAppOp() { + public OrchestratorContext createSubcontextForSingleAppOp(boolean probe) { Instant now = clock.instant(); Instant deadline = timeBudget.deadline().get(); Instant maxDeadline = now.plus(DEFAULT_TIMEOUT_FOR_SINGLE_OP); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java index 6811788ffb7..77bb1e99e19 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorImpl.java @@ -107,7 +107,7 @@ public class OrchestratorImpl implements Orchestrator { ApplicationInstanceReference reference = getApplicationInstance(hostName).reference(); OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock); try (MutableStatusRegistry statusRegistry = statusService - .lockApplicationInstance_forCurrentThreadOnly(reference, context.getTimeLeft())) { + .lockApplicationInstance_forCurrentThreadOnly(context, reference)) { statusRegistry.setHostState(hostName, status); } } @@ -130,9 +130,8 @@ public class OrchestratorImpl implements Orchestrator { ApplicationInstance appInstance = getApplicationInstance(hostName); OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock); - try (MutableStatusRegistry statusRegistry = statusService.lockApplicationInstance_forCurrentThreadOnly( - appInstance.reference(), - context.getTimeLeft())) { + try (MutableStatusRegistry statusRegistry = statusService + .lockApplicationInstance_forCurrentThreadOnly(context, appInstance.reference())) { final HostStatus currentHostState = statusRegistry.getHostStatus(hostName); if (HostStatus.NO_REMARKS == currentHostState) { @@ -159,9 +158,8 @@ public class OrchestratorImpl implements Orchestrator { NodeGroup nodeGroup = new NodeGroup(appInstance, hostName); OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock); - try (MutableStatusRegistry statusRegistry = statusService.lockApplicationInstance_forCurrentThreadOnly( - appInstance.reference(), - context.getTimeLeft())) { + try (MutableStatusRegistry statusRegistry = statusService + .lockApplicationInstance_forCurrentThreadOnly(context, appInstance.reference())) { ApplicationApi applicationApi = new ApplicationApiImpl( nodeGroup, statusRegistry, @@ -181,9 +179,7 @@ public class OrchestratorImpl implements Orchestrator { ApplicationInstanceReference applicationReference = nodeGroup.getApplicationReference(); try (MutableStatusRegistry hostStatusRegistry = - statusService.lockApplicationInstance_forCurrentThreadOnly( - applicationReference, - context.getTimeLeft())) { + statusService.lockApplicationInstance_forCurrentThreadOnly(context, applicationReference)) { ApplicationInstanceStatus appStatus = statusService.forApplicationInstance(applicationReference).getApplicationInstanceStatus(); if (appStatus == ApplicationInstanceStatus.ALLOWED_TO_BE_DOWN) { return; @@ -222,6 +218,8 @@ public class OrchestratorImpl implements Orchestrator { @Override public void suspendAll(HostName parentHostname, List<HostName> hostNames) throws BatchHostStateChangeDeniedException, BatchHostNameNotFoundException, BatchInternalErrorException { + OrchestratorContext context = OrchestratorContext.createContextForMultiAppOp(clock); + List<NodeGroup> nodeGroupsOrderedByApplication; try { nodeGroupsOrderedByApplication = nodeGroupsOrderedForSuspend(hostNames); @@ -229,10 +227,18 @@ public class OrchestratorImpl implements Orchestrator { throw new BatchHostNameNotFoundException(parentHostname, hostNames, e); } - OrchestratorContext context = OrchestratorContext.createContextForMultiAppOp(clock); + suspendAllNodeGroups(context, parentHostname, nodeGroupsOrderedByApplication, true); + suspendAllNodeGroups(context, parentHostname, nodeGroupsOrderedByApplication, false); + } + + private void suspendAllNodeGroups(OrchestratorContext context, + HostName parentHostname, + List<NodeGroup> nodeGroupsOrderedByApplication, + boolean probe) + throws BatchHostStateChangeDeniedException, BatchInternalErrorException { for (NodeGroup nodeGroup : nodeGroupsOrderedByApplication) { try { - suspendGroup(context.createSubcontextForSingleAppOp(), nodeGroup); + suspendGroup(context.createSubcontextForSingleAppOp(probe), nodeGroup); } catch (HostStateChangeDeniedException e) { throw new BatchHostStateChangeDeniedException(parentHostname, nodeGroup, e); } catch (RuntimeException e) { @@ -307,9 +313,7 @@ public class OrchestratorImpl implements Orchestrator { OrchestratorContext context = OrchestratorContext.createContextForSingleAppOp(clock); ApplicationInstanceReference appRef = OrchestratorUtil.toApplicationInstanceReference(appId, instanceLookupService); try (MutableStatusRegistry statusRegistry = - statusService.lockApplicationInstance_forCurrentThreadOnly( - appRef, - context.getTimeLeft())) { + statusService.lockApplicationInstance_forCurrentThreadOnly(context, appRef)) { // Short-circuit if already in wanted state if (status == statusRegistry.getApplicationInstanceStatus()) return; diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java index 50ffe13b437..04725c330e0 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerClientImpl.java @@ -35,7 +35,10 @@ public class ClusterControllerClientImpl implements ClusterControllerClient{ int storageNodeIndex, ClusterControllerNodeState wantedState) throws IOException { ClusterControllerStateRequest.State state = new ClusterControllerStateRequest.State(wantedState, REQUEST_REASON); - ClusterControllerStateRequest stateRequest = new ClusterControllerStateRequest(state, ClusterControllerStateRequest.Condition.SAFE); + ClusterControllerStateRequest stateRequest = new ClusterControllerStateRequest( + state, + ClusterControllerStateRequest.Condition.SAFE, + context.isProbe() ? true : null); ClusterControllerClientTimeouts timeouts = context.getClusterControllerTimeouts(); try { @@ -67,7 +70,8 @@ public class ClusterControllerClientImpl implements ClusterControllerClient{ OrchestratorContext context, ClusterControllerNodeState wantedState) throws IOException { ClusterControllerStateRequest.State state = new ClusterControllerStateRequest.State(wantedState, REQUEST_REASON); - ClusterControllerStateRequest stateRequest = new ClusterControllerStateRequest(state, ClusterControllerStateRequest.Condition.FORCE); + ClusterControllerStateRequest stateRequest = new ClusterControllerStateRequest( + state, ClusterControllerStateRequest.Condition.FORCE, null); ClusterControllerClientTimeouts timeouts = context.getClusterControllerTimeouts(); try { diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateRequest.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateRequest.java index 9a29ad04d78..b07a9e2bfa1 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateRequest.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/controller/ClusterControllerStateRequest.java @@ -1,8 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.controller; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import javax.annotation.concurrent.Immutable; import java.util.Collections; import java.util.Map; import java.util.Objects; @@ -10,6 +12,8 @@ import java.util.Objects; /** * @author hakonhall */ +@Immutable +@JsonInclude(JsonInclude.Include.NON_NULL) public class ClusterControllerStateRequest { @JsonProperty("state") @@ -18,26 +22,29 @@ public class ClusterControllerStateRequest { @JsonProperty("condition") public final Condition condition; - public ClusterControllerStateRequest(State currentState, Condition condition) { + @JsonProperty("probe") + public final Boolean probe; + + public ClusterControllerStateRequest(State currentState, Condition condition, Boolean probe) { Map<String, State> state = Collections.singletonMap("user", currentState); this.state = Collections.unmodifiableMap(state); this.condition = condition; + this.probe = probe; } @Override - public boolean equals(Object object) { - if (!(object instanceof ClusterControllerStateRequest)) { - return false; - } - - final ClusterControllerStateRequest that = (ClusterControllerStateRequest) object; - return Objects.equals(this.state, that.state) - && Objects.equals(this.condition, that.condition); + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ClusterControllerStateRequest that = (ClusterControllerStateRequest) o; + return Objects.equals(state, that.state) && + condition == that.condition && + Objects.equals(probe, that.probe); } @Override public int hashCode() { - return Objects.hash(state, condition); + return Objects.hash(state, condition, probe); } @Override diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApi.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApi.java index 0ca509d13f1..e2f371a5ce1 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApi.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApi.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.orchestrator.model; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.orchestrator.OrchestratorContext; import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; import com.yahoo.vespa.orchestrator.status.HostStatus; @@ -26,7 +27,7 @@ public interface ApplicationApi { ApplicationInstanceStatus getApplicationStatus(); - void setHostState(HostName hostName, HostStatus status); + void setHostState(OrchestratorContext context, HostName hostName, HostStatus status); List<HostName> getNodesInGroupWithStatus(HostStatus status); List<StorageNode> getStorageNodesInGroupInClusterOrder(); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java index c5bcaf4de82..9ec1697a45f 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/ApplicationApiImpl.java @@ -6,12 +6,12 @@ import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceCluster; import com.yahoo.vespa.applicationmodel.ServiceInstance; +import com.yahoo.vespa.orchestrator.OrchestratorContext; import com.yahoo.vespa.orchestrator.OrchestratorUtil; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientFactory; import com.yahoo.vespa.orchestrator.status.ApplicationInstanceStatus; import com.yahoo.vespa.orchestrator.status.HostStatus; import com.yahoo.vespa.orchestrator.status.MutableStatusRegistry; -import com.yahoo.vespa.orchestrator.status.ReadOnlyStatusRegistry; import java.util.Collection; import java.util.Comparator; @@ -20,6 +20,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import static com.yahoo.vespa.orchestrator.OrchestratorUtil.getHostsUsedByApplicationInstance; @@ -29,7 +30,6 @@ public class ApplicationApiImpl implements ApplicationApi { private final NodeGroup nodeGroup; private final MutableStatusRegistry hostStatusService; private final List<ClusterApi> clusterInOrder; - private final ClusterControllerClientFactory clusterControllerClientFactory; private final Map<HostName, HostStatus> hostStatusMap; public ApplicationApiImpl(NodeGroup nodeGroup, @@ -38,11 +38,9 @@ public class ApplicationApiImpl implements ApplicationApi { this.applicationInstance = nodeGroup.getApplication(); this.nodeGroup = nodeGroup; this.hostStatusService = hostStatusService; - this.hostStatusMap = createHostStatusMap( - getHostsUsedByApplicationInstance(applicationInstance), - hostStatusService); + Collection<HostName> hosts = getHostsUsedByApplicationInstance(applicationInstance); + this.hostStatusMap = hosts.stream().collect(Collectors.toMap(Function.identity(), hostStatusService::getHostStatus)); this.clusterInOrder = makeClustersInOrder(nodeGroup, hostStatusMap, clusterControllerClientFactory); - this.clusterControllerClientFactory = clusterControllerClientFactory; } @Override @@ -50,14 +48,6 @@ public class ApplicationApiImpl implements ApplicationApi { return OrchestratorUtil.toApplicationId(applicationInstance.reference()); } - private static Map<HostName, HostStatus> createHostStatusMap(Collection<HostName> hosts, - ReadOnlyStatusRegistry hostStatusService) { - return hosts.stream() - .collect(Collectors.toMap( - hostName -> hostName, - hostName -> hostStatusService.getHostStatus(hostName))); - } - private HostStatus getHostStatus(HostName hostName) { return hostStatusMap.getOrDefault(hostName, HostStatus.NO_REMARKS); } @@ -104,7 +94,7 @@ public class ApplicationApiImpl implements ApplicationApi { } @Override - public void setHostState(HostName hostName, HostStatus status) { + public void setHostState(OrchestratorContext context, HostName hostName, HostStatus status) { hostStatusService.setHostState(hostName, status); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java index 9900c8de752..ade45c6c515 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java @@ -104,6 +104,12 @@ public class StorageNodeImpl implements StorageNode { HostedVespaPolicy.SET_NODE_STATE_CONSTRAINT, "Failed to set state to " + wantedNodeState + " in cluster controller: " + response.reason); } + + String logSuffix = context.isProbe() ? + " has been set to " + wantedNodeState: + " would have been set to " + wantedNodeState + " (this is a probe)"; + logger.log(LogLevel.INFO, "Storage node " + nodeIndex + " in cluster " + clusterId + + " application " + applicationInstance.reference().asString() + " on host " + hostName() + logSuffix); } @Override diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java index a781fd2358a..4aa0f3452e3 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/policy/HostedVespaPolicy.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.policy; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.orchestrator.OrchestratorContext; @@ -52,13 +51,11 @@ public class HostedVespaPolicy implements Policy { // These storage nodes are guaranteed to be NO_REMARKS for (StorageNode storageNode : application.getUpStorageNodesInGroupInClusterOrder()) { storageNode.setNodeState(context, ClusterControllerNodeState.MAINTENANCE); - log.log(LogLevel.INFO, "The storage node on " + storageNode.hostName() + " has been set to MAINTENANCE"); } // Ensure all nodes in the group are marked as allowed to be down for (HostName hostName : application.getNodesInGroupWithStatus(HostStatus.NO_REMARKS)) { - application.setHostState(hostName, HostStatus.ALLOWED_TO_BE_DOWN); - log.log(LogLevel.INFO, hostName + " is now allowed to be down (suspended)"); + application.setHostState(context, hostName, HostStatus.ALLOWED_TO_BE_DOWN); } } @@ -68,12 +65,10 @@ public class HostedVespaPolicy implements Policy { // Always defer to Cluster Controller whether it's OK to resume storage node for (StorageNode storageNode : application.getStorageNodesAllowedToBeDownInGroupInReverseClusterOrder()) { storageNode.setNodeState(context, ClusterControllerNodeState.UP); - log.log(LogLevel.INFO, "The storage node on " + storageNode.hostName() + " has been set to UP"); } for (HostName hostName : application.getNodesInGroupWithStatus(HostStatus.ALLOWED_TO_BE_DOWN)) { - application.setHostState(hostName, HostStatus.NO_REMARKS); - log.log(LogLevel.INFO, hostName + " is no longer allowed to be down (resumed)"); + application.setHostState(context, hostName, HostStatus.NO_REMARKS); } } @@ -98,13 +93,11 @@ public class HostedVespaPolicy implements Policy { // These storage nodes are guaranteed to be NO_REMARKS for (StorageNode storageNode : applicationApi.getStorageNodesInGroupInClusterOrder()) { storageNode.setNodeState(context, ClusterControllerNodeState.DOWN); - log.log(LogLevel.INFO, "The storage node on " + storageNode.hostName() + " has been set DOWN"); } // Ensure all nodes in the group are marked as allowed to be down for (HostName hostName : applicationApi.getNodesInGroupWithStatus(HostStatus.NO_REMARKS)) { - applicationApi.setHostState(hostName, HostStatus.ALLOWED_TO_BE_DOWN); - log.log(LogLevel.INFO, hostName + " is now allowed to be down (suspended)"); + applicationApi.setHostState(context, hostName, HostStatus.ALLOWED_TO_BE_DOWN); } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/InMemoryStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/InMemoryStatusService.java index bd5eb6f3e29..70128ae12eb 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/InMemoryStatusService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/InMemoryStatusService.java @@ -3,8 +3,8 @@ package com.yahoo.vespa.orchestrator.status; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.orchestrator.OrchestratorContext; -import java.time.Duration; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -46,8 +46,8 @@ public class InMemoryStatusService implements StatusService { @Override public MutableStatusRegistry lockApplicationInstance_forCurrentThreadOnly( - ApplicationInstanceReference applicationInstanceReference, - Duration timeout) { + OrchestratorContext context, + ApplicationInstanceReference applicationInstanceReference) { Lock lock = instanceLockService.get(applicationInstanceReference); return new InMemoryMutableStatusRegistry(lock, applicationInstanceReference); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java index 76adef72b2b..99f6c113193 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/StatusService.java @@ -2,8 +2,8 @@ package com.yahoo.vespa.orchestrator.status; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; +import com.yahoo.vespa.orchestrator.OrchestratorContext; -import java.time.Duration; import java.util.Set; /** @@ -25,7 +25,7 @@ public interface StatusService { * possibly inconsistent snapshot values. It is not recommended that this method is used for anything other * than monitoring, logging, debugging, etc. It should never be used for multi-step operations (e.g. * read-then-write) where consistency is required. For those cases, use - * {@link #lockApplicationInstance_forCurrentThreadOnly(ApplicationInstanceReference, Duration)}. + * {@link #lockApplicationInstance_forCurrentThreadOnly(OrchestratorContext, ApplicationInstanceReference)}. */ ReadOnlyStatusRegistry forApplicationInstance(ApplicationInstanceReference applicationInstanceReference); @@ -54,8 +54,8 @@ public interface StatusService { * This may leave the registry in an inconsistent state (as judged by the client code). */ MutableStatusRegistry lockApplicationInstance_forCurrentThreadOnly( - ApplicationInstanceReference applicationInstanceReference, - Duration timeout); + OrchestratorContext context, + ApplicationInstanceReference applicationInstanceReference); /** * Returns all application instances that are allowed to be down. The intention is to use this diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java index 7df29e038c1..3360a12c32e 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZookeeperStatusService.java @@ -7,6 +7,7 @@ import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.orchestrator.OrchestratorContext; import com.yahoo.vespa.orchestrator.OrchestratorUtil; import org.apache.curator.framework.recipes.locks.InterProcessSemaphoreMutex; import org.apache.zookeeper.KeeperException.NoNodeException; @@ -90,14 +91,15 @@ public class ZookeeperStatusService implements StatusService { */ @Override public MutableStatusRegistry lockApplicationInstance_forCurrentThreadOnly( - ApplicationInstanceReference applicationInstanceReference, - Duration timeout) { + OrchestratorContext context, + ApplicationInstanceReference applicationInstanceReference) { + Duration duration = context.getTimeLeft(); String lockPath = applicationInstanceLock2Path(applicationInstanceReference); Lock lock = new Lock(lockPath, curator); - lock.acquire(timeout); + lock.acquire(duration); try { - return new ZkMutableStatusRegistry(lock, applicationInstanceReference); + return new ZkMutableStatusRegistry(lock, applicationInstanceReference, context.isProbe()); } catch (Throwable t) { // In case the constructor throws an exception. lock.close(); @@ -215,23 +217,31 @@ public class ZookeeperStatusService implements StatusService { private class ZkMutableStatusRegistry implements MutableStatusRegistry { private final Lock lock; private final ApplicationInstanceReference applicationInstanceReference; + private final boolean probe; public ZkMutableStatusRegistry( Lock lock, - ApplicationInstanceReference applicationInstanceReference) { + ApplicationInstanceReference applicationInstanceReference, + boolean probe) { this.lock = lock; this.applicationInstanceReference = applicationInstanceReference; + this.probe = probe; } @Override public void setHostState(final HostName hostName, final HostStatus status) { + if (probe) return; + log.log(LogLevel.INFO, "Setting host " + hostName + " to status " + status); setHostStatus(applicationInstanceReference, hostName, status); } @Override public void setApplicationInstanceStatus(ApplicationInstanceStatus applicationInstanceStatus) { - String path = applicationInstanceSuspendedPath(applicationInstanceReference); + if (probe) return; + + log.log(LogLevel.INFO, "Setting app " + applicationInstanceReference.asString() + " to status " + applicationInstanceStatus); + String path = applicationInstanceSuspendedPath(applicationInstanceReference); try { switch (applicationInstanceStatus) { case NO_REMARKS: |