blob: 894b9ebd8ce967d39d2cfd3f4281128f7c92d4f9 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
|
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.prelude.cluster;
import static com.yahoo.container.protect.Error.BACKEND_COMMUNICATION_ERROR;
import static com.yahoo.container.protect.Error.NO_ANSWER_WHEN_PINGING_NODE;
import java.util.logging.Logger;
import com.yahoo.prelude.fastsearch.VespaBackEndSearcher;
import com.yahoo.search.result.ErrorMessage;
/**
* A node monitor is responsible for maintaining the state of a monitored node.
* It has the following properties:
* <ul>
* <li>A node is taken out of operation if it gives no response in 10 s</li>
* <li>A node is put back in operation when it responds correctly again</li>
* <li>A node is initially considered not in operation until we have some data from it</li>
* </ul>
*
* @author bratseth
* @author Steinar Knutsen
*/
public class NodeMonitor {
protected static Logger log = Logger.getLogger(NodeMonitor.class.getName());
/** The object representing the monitored node */
private final VespaBackEndSearcher node;
private boolean isWorking = true;
/** The last time this node responded successfully */
private long succeededAt = 0;
/** Whether it is assumed the node has documents available to serve */
private boolean searchNodesOnline = false;
/**
* Creates a new node monitor for a node
*/
public NodeMonitor(final VespaBackEndSearcher node) {
this.node = node;
}
/**
* Returns whether this node is currently in a state suitable for receiving
* traffic. As far as we know, that is
*/
public boolean isWorking() {
return isWorking;
}
// Whether or not dispatch has ever responded successfully
private boolean statusIsKnown = false;
public VespaBackEndSearcher getNode() {
return node;
}
/**
* Called when this node fails.
*
* @param error a container which should contain a short description
*/
public void failed(ErrorMessage error) {
long respondedAt = System.currentTimeMillis();
statusIsKnown = true;
if (error.getCode() == NO_ANSWER_WHEN_PINGING_NODE.code) {
// Only count not being able to talk to backend at all
// as errors we care about
if ((respondedAt - succeededAt) > 10000) {
this.searchNodesOnline = false;
setWorking(false, "Not working for 10 s: " + error.toString());
}
} else if (error.getCode() == BACKEND_COMMUNICATION_ERROR.code) {
this.searchNodesOnline = false;
setWorking(false, "Backend communication error: " + error.toString());
} else {
succeededAt = respondedAt;
}
}
/**
* Called when a response is received from this node.
*/
public void responded(boolean searchNodesOnline) {
succeededAt = System.currentTimeMillis();
statusIsKnown = true;
this.searchNodesOnline = searchNodesOnline;
if (! isWorking)
setWorking(true, "Responds correctly");
}
/** Changes the state of this node if required */
private void setWorking(boolean working, String explanation) {
if (isWorking == working) return; // Old news
if (statusIsKnown) {
if (working)
log.info("Putting " + node + " in service: " + explanation);
else
log.info("Taking " + node + " out of service: " + explanation);
}
isWorking = working;
}
boolean searchNodesOnline() { return searchNodesOnline; }
/** Returns true if we have had enough time to determine the status of this node since creating the monitor */
boolean statusIsKnown() { return statusIsKnown; }
}
|
