diff options
163 files changed, 1514 insertions, 1486 deletions
diff --git a/client/go/internal/cli/cmd/feed_test.go b/client/go/internal/cli/cmd/feed_test.go index d1904ad815e..fc2c5ec7520 100644 --- a/client/go/internal/cli/cmd/feed_test.go +++ b/client/go/internal/cli/cmd/feed_test.go @@ -80,13 +80,17 @@ func TestFeed(t *testing.T) { require.Nil(t, cli.Run("feed", "-")) assert.Equal(t, want, stdout.String()) - httpClient.NextResponseString(500, `{"message":"it's broken yo"}`) + for i := 0; i < 10; i++ { + httpClient.NextResponseString(500, `{"message":"it's broken yo"}`) + } require.Nil(t, cli.Run("feed", jsonFile1)) - assert.Equal(t, "feed: got status 500 ({\"message\":\"it's broken yo\"}) for put id:ns:type::doc1: retrying\n", stderr.String()) + assert.Equal(t, "feed: got status 500 ({\"message\":\"it's broken yo\"}) for put id:ns:type::doc1: giving up after 10 attempts\n", stderr.String()) stderr.Reset() - httpClient.NextResponseError(fmt.Errorf("something else is broken")) + for i := 0; i < 10; i++ { + httpClient.NextResponseError(fmt.Errorf("something else is broken")) + } require.Nil(t, cli.Run("feed", jsonFile1)) - assert.Equal(t, "feed: got error \"something else is broken\" (no body) for put id:ns:type::doc1: retrying\n", stderr.String()) + assert.Equal(t, "feed: got error \"something else is broken\" (no body) for put id:ns:type::doc1: giving up after 10 attempts\n", stderr.String()) } func TestFeedInvalid(t *testing.T) { diff --git a/client/go/internal/mock/http.go b/client/go/internal/mock/http.go index 8bab716ea60..3d4ead596b0 100644 --- a/client/go/internal/mock/http.go +++ b/client/go/internal/mock/http.go @@ -14,8 +14,8 @@ type HTTPClient struct { // The responses to return for future requests. Once a response is consumed, it's removed from this slice. nextResponses []HTTPResponse - // The error to return for the next request. If non-nil, this error is returned before any responses in nextResponses. - nextError error + // The errors to return for future requests. If non-nil, these errors are returned before any responses in nextResponses. + nextErrors []error // LastRequest is the last HTTP request made through this. LastRequest *http.Request @@ -52,13 +52,14 @@ func (c *HTTPClient) NextResponse(response HTTPResponse) { } func (c *HTTPClient) NextResponseError(err error) { - c.nextError = err + c.nextErrors = append(c.nextErrors, err) } func (c *HTTPClient) Do(request *http.Request, timeout time.Duration) (*http.Response, error) { - if c.nextError != nil { - err := c.nextError - c.nextError = nil + c.LastRequest = request + if len(c.nextErrors) > 0 { + err := c.nextErrors[0] + c.nextErrors = c.nextErrors[1:] return nil, err } response := HTTPResponse{Status: 200} @@ -66,7 +67,6 @@ func (c *HTTPClient) Do(request *http.Request, timeout time.Duration) (*http.Res response = c.nextResponses[0] c.nextResponses = c.nextResponses[1:] } - c.LastRequest = request if c.ReadBody && request.Body != nil { body, err := io.ReadAll(request.Body) if err != nil { diff --git a/client/go/internal/vespa/document/dispatcher.go b/client/go/internal/vespa/document/dispatcher.go index fa4424809cf..fb7a532e332 100644 --- a/client/go/internal/vespa/document/dispatcher.go +++ b/client/go/internal/vespa/document/dispatcher.go @@ -67,7 +67,7 @@ func (d *Dispatcher) logResult(doc Document, result Result, retry bool) { if result.Trace != "" { d.msgs <- fmt.Sprintf("feed: trace for %s %s:\n%s", doc.Operation, doc.Id, result.Trace) } - if !d.verbose && result.Success() { + if !d.verbose && (retry || result.Success()) { return } var msg strings.Builder diff --git a/client/go/internal/vespa/document/http.go b/client/go/internal/vespa/document/http.go index d6e7745e6b1..a2e399549c4 100644 --- a/client/go/internal/vespa/document/http.go +++ b/client/go/internal/vespa/document/http.go @@ -265,18 +265,18 @@ func (c *Client) Send(document Document) Result { req, buf, err := c.prepare(document) defer c.buffers.Put(buf) if err != nil { - return resultWithErr(result, err) + return resultWithErr(result, err, 0) } bodySize := len(document.Body) if buf.Len() > 0 { bodySize = buf.Len() } resp, err := c.leastBusyClient().Do(req, c.clientTimeout()) + elapsed := c.now().Sub(start) if err != nil { - return resultWithErr(result, err) + return resultWithErr(result, err, elapsed) } defer resp.Body.Close() - elapsed := c.now().Sub(start) return c.resultWithResponse(resp, bodySize, result, elapsed, buf, false) } @@ -290,20 +290,21 @@ func (c *Client) Get(id Id) Result { result := Result{Id: id} req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { - return resultWithErr(result, err) + return resultWithErr(result, err, 0) } resp, err := c.leastBusyClient().Do(req, c.clientTimeout()) + elapsed := c.now().Sub(start) if err != nil { - return resultWithErr(result, err) + return resultWithErr(result, err, elapsed) } defer resp.Body.Close() - elapsed := c.now().Sub(start) return c.resultWithResponse(resp, 0, result, elapsed, buf, true) } -func resultWithErr(result Result, err error) Result { +func resultWithErr(result Result, err error, elapsed time.Duration) Result { result.Status = StatusTransportFailure result.Err = err + result.Latency = elapsed return result } @@ -322,14 +323,14 @@ func (c *Client) resultWithResponse(resp *http.Response, sentBytes int, result R buf.Reset() written, err := io.Copy(buf, resp.Body) if err != nil { - result = resultWithErr(result, err) + result = resultWithErr(result, err, elapsed) } else { if result.Success() && c.options.TraceLevel > 0 { var jsonResponse struct { Trace json.RawValue `json:"trace"` } if err := json.Unmarshal(buf.Bytes(), &jsonResponse); err != nil { - result = resultWithErr(result, fmt.Errorf("failed to decode json response: %w", err)) + result = resultWithErr(result, fmt.Errorf("failed to decode json response: %w", err), elapsed) } else { result.Trace = string(jsonResponse.Trace) } diff --git a/client/go/internal/vespa/document/http_test.go b/client/go/internal/vespa/document/http_test.go index c797ba5607f..6faa14705f0 100644 --- a/client/go/internal/vespa/document/http_test.go +++ b/client/go/internal/vespa/document/http_test.go @@ -74,6 +74,9 @@ func TestClientSend(t *testing.T) { {Document{Condition: "foo", Id: mustParseId("id:ns:type::doc4"), Operation: OperationUpdate, Body: []byte(`{"fields":{"baz": "789"}}`)}, "PUT", "https://example.com:1337/document/v1/ns/type/docid/doc4?timeout=5000ms&condition=foo"}, + {Document{Id: mustParseId("id:ns:type::doc5"), Operation: OperationPut, Body: []byte(`{"fields":{"baz": "789"}}`)}, + "POST", + "https://example.com:1337/document/v1/ns/type/docid/doc5?timeout=5000ms"}, } httpClient := mock.HTTPClient{ReadBody: true} client, _ := NewClient(ClientOptions{ @@ -89,53 +92,61 @@ func TestClientSend(t *testing.T) { Id: doc.Id, Latency: time.Second, } - if i < 3 { + switch i { + case 0, 1, 2: msg := `{"message":"All good!"}` httpClient.NextResponseString(200, msg) wantRes.Status = StatusSuccess wantRes.HTTPStatus = 200 wantRes.BytesRecv = 23 - } else { + case 3: errMsg := `something went wront` httpClient.NextResponseString(502, errMsg) wantRes.Status = StatusVespaFailure wantRes.HTTPStatus = 502 wantRes.Body = []byte(errMsg) wantRes.BytesRecv = 20 + case 4: + transportErr := fmt.Errorf("transport error") + httpClient.NextResponseError(transportErr) + wantRes.Err = transportErr + wantRes.Status = StatusTransportFailure } res := client.Send(doc) - wantRes.BytesSent = int64(len(httpClient.LastBody)) + if res.Err == nil { + wantRes.BytesSent = int64(len(httpClient.LastBody)) + } if !reflect.DeepEqual(res, wantRes) { - t.Fatalf("got result %+v, want %+v", res, wantRes) + t.Fatalf("#%d: got result %+v, want %+v", i, res, wantRes) } stats.Add(res) r := httpClient.LastRequest if r.Method != tt.method { - t.Errorf("got r.Method = %q, want %q", r.Method, tt.method) + t.Errorf("#%d: got r.Method = %q, want %q", i, r.Method, tt.method) } var headers http.Header = map[string][]string{ "Content-Type": {"application/json; charset=utf-8"}, } if !reflect.DeepEqual(r.Header, headers) { - t.Errorf("got r.Header = %v, want %v", r.Header, headers) + t.Errorf("#%d: got r.Header = %v, want %v", i, r.Header, headers) } if r.URL.String() != tt.url { - t.Errorf("got r.URL = %q, want %q", r.URL, tt.url) + t.Errorf("#%d: got r.URL = %q, want %q", i, r.URL, tt.url) } if !bytes.Equal(httpClient.LastBody, doc.Body) { - t.Errorf("got r.Body = %q, want %q", string(httpClient.LastBody), doc.Body) + t.Errorf("#%d: got r.Body = %q, want %q", i, string(httpClient.LastBody), doc.Body) } } want := Stats{ - Requests: 4, + Requests: 5, Responses: 4, ResponsesByCode: map[int]int64{ 200: 3, 502: 1, }, - Errors: 0, + Errors: 1, Inflight: 0, - TotalLatency: 4 * time.Second, + TotalLatency: 5 * time.Second, MinLatency: time.Second, MaxLatency: time.Second, BytesSent: 75, diff --git a/client/go/internal/vespa/document/stats.go b/client/go/internal/vespa/document/stats.go index 3e647d0f893..e53d787cd01 100644 --- a/client/go/internal/vespa/document/stats.go +++ b/client/go/internal/vespa/document/stats.go @@ -86,9 +86,9 @@ func (s *Stats) Add(result Result) { if s.ResponsesByCode == nil { s.ResponsesByCode = make(map[int]int64) } - responsesByCode := s.ResponsesByCode[result.HTTPStatus] - s.ResponsesByCode[result.HTTPStatus] = responsesByCode + 1 if result.Err == nil { + responsesByCode := s.ResponsesByCode[result.HTTPStatus] + s.ResponsesByCode[result.HTTPStatus] = responsesByCode + 1 s.Responses++ } else { s.Errors++ diff --git a/client/src/main/java/ai/vespa/client/dsl/Field.java b/client/src/main/java/ai/vespa/client/dsl/Field.java index 6d199ead2b8..59459899189 100644 --- a/client/src/main/java/ai/vespa/client/dsl/Field.java +++ b/client/src/main/java/ai/vespa/client/dsl/Field.java @@ -571,6 +571,29 @@ public class Field extends QueryChain { return common("nearestNeighbor", annotation, (Object) rankFeature); } + /** + * Fuzzy query. + * https://docs.vespa.ai/en/reference/query-language-reference.html#fuzzy + * + * @param text the text to match fuzzily + * @return the query + */ + public Query fuzzy(String text) { + return common("fuzzy", annotation, text); + } + + /** + * Fuzzy query. + * https://docs.vespa.ai/en/reference/query-language-reference.html#fuzzy + * + * @param annotation the annotation + * @param text the text to match fuzzily + * @return the query + */ + public Query fuzzy(Annotation annotation, String text) { + return common("fuzzy", annotation, text); + } + private Query common(String relation, Annotation annotation, Object value) { return common(relation, annotation, value, values.toArray()); } @@ -629,6 +652,8 @@ public class Field extends QueryChain { return hasAnnotation ? Text.format("([%s]nearestNeighbor(%s, %s))", annotation, fieldName, valuesStr) : Text.format("nearestNeighbor(%s, %s)", fieldName, valuesStr); + case "fuzzy": + return Text.format("%s contains (%sfuzzy(%s))", fieldName, annotation, values.get(0)); default: Object value = values.get(0); valuesStr = value instanceof Long ? value + "L" : value.toString(); diff --git a/client/src/main/java/ai/vespa/client/dsl/Q.java b/client/src/main/java/ai/vespa/client/dsl/Q.java index 2bb998cd3e5..e4cfd4aa1ef 100644 --- a/client/src/main/java/ai/vespa/client/dsl/Q.java +++ b/client/src/main/java/ai/vespa/client/dsl/Q.java @@ -22,7 +22,7 @@ public final class Q { throw new RuntimeException(e); } } - private static Sources SELECT_ALL_FROM_SOURCES_ALL = new Sources(new Select("*"), "*"); + private static final Sources SELECT_ALL_FROM_SOURCES_ALL = new Sources(new Select("*"), "*"); public static Select select(String fieldName) { return new Select(fieldName); } diff --git a/client/src/test/java/ai/vespa/client/dsl/QTest.java b/client/src/test/java/ai/vespa/client/dsl/QTest.java index aae8b2c8923..c242349873c 100644 --- a/client/src/test/java/ai/vespa/client/dsl/QTest.java +++ b/client/src/test/java/ai/vespa/client/dsl/QTest.java @@ -485,6 +485,18 @@ class QTest { } @Test + void fuzzy() { + String q = Q.p("f1").fuzzy("text to match").build(); + assertEquals("yql=select * from sources * where f1 contains (fuzzy(\"text to match\"))", q); + } + + @Test + void fuzzy_with_annotation() { + String q = Q.p("f1").fuzzy(A.a("maxEditDistance", 3).append(A.a("prefixLength", 10)), "text to match").build(); + assertEquals("yql=select * from sources * where f1 contains ({\"prefixLength\":10,\"maxEditDistance\":3}fuzzy(\"text to match\"))", q); + } + + @Test void use_contains_instead_of_contains_equiv_when_input_size_is_1() { String q = Q.p("f1").containsEquiv(Collections.singletonList("p1")) .build(); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java index 2e8e2707166..19c4e4b1e89 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java @@ -17,6 +17,7 @@ import java.util.Objects; import java.util.TreeMap; import static com.yahoo.vdslib.state.NodeState.ORCHESTRATOR_RESERVED_DESCRIPTION; +import static com.yahoo.vespa.clustercontroller.core.NodeStateChangeChecker.Result; public class ContentCluster { @@ -129,7 +130,7 @@ public class ContentCluster { * @param newState state wanted to be set * @param inMoratorium whether the CC is in moratorium */ - public NodeStateChangeChecker.Result calculateEffectOfNewState( + public Result calculateEffectOfNewState( Node node, ClusterState clusterState, SetUnitStateRequest.Condition condition, NodeState oldState, NodeState newState, boolean inMoratorium) { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java index 069139b8c9e..04ad5899cd2 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java @@ -216,10 +216,6 @@ abstract public class NodeInfo implements Comparable<NodeInfo> { return RPCCommunicator.SET_DISTRIBUTION_STATES_RPC_VERSION; } - public String getSlobrokAddress() { - return "storage/cluster." + cluster.getName() + "/" + node.getType() + "/" + node.getIndex(); - } - public void markRpcAddressOutdated(Timer timer) { lastSeenInSlobrok = timer.getCurrentTimeInMillis(); } @@ -237,6 +233,10 @@ abstract public class NodeInfo implements Comparable<NodeInfo> { return node.getType().equals(NodeType.STORAGE); } + public String type() { + return isDistributor() ? "distributor" : "storage node"; + } + public int getNodeIndex() { return node.getIndex(); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeChecker.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeChecker.java index 54240330de3..c9f5cfeb9c8 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeChecker.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeChecker.java @@ -11,7 +11,6 @@ import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo; -import com.yahoo.vespa.clustercontroller.core.hostinfo.Metrics; import com.yahoo.vespa.clustercontroller.core.hostinfo.StorageNode; import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest; import java.util.ArrayList; @@ -30,9 +29,9 @@ import static com.yahoo.vdslib.state.State.DOWN; import static com.yahoo.vdslib.state.State.MAINTENANCE; import static com.yahoo.vdslib.state.State.RETIRED; import static com.yahoo.vdslib.state.State.UP; -import static com.yahoo.vespa.clustercontroller.core.NodeStateChangeChecker.Result.allowSettingOfWantedState; -import static com.yahoo.vespa.clustercontroller.core.NodeStateChangeChecker.Result.createAlreadySet; -import static com.yahoo.vespa.clustercontroller.core.NodeStateChangeChecker.Result.createDisallowed; +import static com.yahoo.vespa.clustercontroller.core.NodeStateChangeChecker.Result.allow; +import static com.yahoo.vespa.clustercontroller.core.NodeStateChangeChecker.Result.alreadySet; +import static com.yahoo.vespa.clustercontroller.core.NodeStateChangeChecker.Result.disallow; import static com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest.Condition.FORCE; import static com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest.Condition.SAFE; import static java.util.logging.Level.FINE; @@ -64,214 +63,150 @@ public class NodeStateChangeChecker { throw new IllegalArgumentException("Cannot have both 1 group and maxNumberOfGroupsAllowedToBeDown > 1"); } - public static class Result { - - public enum Action { - MUST_SET_WANTED_STATE, - ALREADY_SET, - DISALLOWED - } - - private final Action action; - private final String reason; - - private Result(Action action, String reason) { - this.action = action; - this.reason = reason; - } - - public static Result createDisallowed(String reason) { - return new Result(Action.DISALLOWED, reason); - } - - public static Result allowSettingOfWantedState() { - return new Result(Action.MUST_SET_WANTED_STATE, "Preconditions fulfilled and new state different"); - } - - public static Result createAlreadySet() { - return new Result(Action.ALREADY_SET, "Basic preconditions fulfilled and new state is already effective"); - } - - public boolean settingWantedStateIsAllowed() { return action == Action.MUST_SET_WANTED_STATE; } - - public boolean settingWantedStateIsNotAllowed() { return ! settingWantedStateIsAllowed(); } - - public boolean wantedStateAlreadySet() { - return action == Action.ALREADY_SET; - } - - public String getReason() { - return reason; - } - - public String toString() { - return "action " + action + ": " + reason; - } - } - public Result evaluateTransition(Node node, ClusterState clusterState, SetUnitStateRequest.Condition condition, NodeState oldWantedState, NodeState newWantedState) { - if (condition == FORCE) { - return allowSettingOfWantedState(); - } + if (condition == FORCE) + return allow(); - if (inMoratorium) { - return createDisallowed("Master cluster controller is bootstrapping and in moratorium"); - } + if (inMoratorium) + return disallow("Master cluster controller is bootstrapping and in moratorium"); - if (condition != SAFE) { - return createDisallowed("Condition not implemented: " + condition.name()); - } + if (condition != SAFE) + return disallow("Condition not implemented: " + condition.name()); - if (node.getType() != STORAGE) { - return createDisallowed("Safe-set of node state is only supported for storage nodes! " + - "Requested node type: " + node.getType().toString()); - } + if (node.getType() != STORAGE) + return disallow("Safe-set of node state is only supported for storage nodes! " + + "Requested node type: " + node.getType().toString()); StorageNodeInfo nodeInfo = clusterInfo.getStorageNodeInfo(node.getIndex()); - if (nodeInfo == null) { - return createDisallowed("Unknown node " + node); - } + if (nodeInfo == null) + return disallow("Unknown node " + node); - // If the new state and description equals the existing, we're done. This is done for 2 cases: - // - We can short-circuit setting of a new wanted state, which e.g. hits ZooKeeper. - // - We ensure that clients that have previously set the wanted state, continue - // to see the same conclusion, even though they possibly would have been denied - // MUST_SET_WANTED_STATE if re-evaluated. This is important for implementing idempotent clients. - if (newWantedState.getState().equals(oldWantedState.getState()) && - Objects.equals(newWantedState.getDescription(), oldWantedState.getDescription())) { - return createAlreadySet(); - } + if (noChanges(oldWantedState, newWantedState)) + return alreadySet(); return switch (newWantedState.getState()) { case UP -> canSetStateUp(nodeInfo, oldWantedState); case MAINTENANCE -> canSetStateMaintenanceTemporarily(nodeInfo, clusterState, newWantedState.getDescription()); case DOWN -> canSetStateDownPermanently(nodeInfo, clusterState, newWantedState.getDescription()); - default -> createDisallowed("Destination node state unsupported in safe mode: " + newWantedState); + default -> disallow("Destination node state unsupported in safe mode: " + newWantedState); }; } + private static boolean noChanges(NodeState oldWantedState, NodeState newWantedState) { + // If the new state and description equals the existing, we're done. This is done for 2 cases: + // - We can short-circuit setting of a new wanted state, which e.g. hits ZooKeeper. + // - We ensure that clients that have previously set the wanted state, continue + // to see the same conclusion, even though they possibly would have been + // DISALLOWED if re-evaluated. This is important for implementing idempotent clients. + return newWantedState.getState().equals(oldWantedState.getState()) + && Objects.equals(newWantedState.getDescription(), oldWantedState.getDescription()); + } + private Result canSetStateDownPermanently(NodeInfo nodeInfo, ClusterState clusterState, String newDescription) { - NodeState oldWantedState = nodeInfo.getUserWantedState(); - if (oldWantedState.getState() != UP && !oldWantedState.getDescription().equals(newDescription)) { - // Refuse to override whatever an operator or unknown entity is doing. - // - // Note: The new state&description is NOT equal to the old state&description: - // that would have been short-circuited prior to this. - return createDisallowed("A conflicting wanted state is already set: " + - oldWantedState.getState() + ": " + oldWantedState.getDescription()); - } + var result = checkIfStateSetWithDifferentDescription(nodeInfo, newDescription); + if (result.notAllowed()) + return result; State reportedState = nodeInfo.getReportedState().getState(); - if (reportedState != UP) { - return createDisallowed("Reported state (" + reportedState - + ") is not UP, so no bucket data is available"); - } + if (reportedState != UP) + return disallow("Reported state (" + reportedState + ") is not UP, so no bucket data is available"); State currentState = clusterState.getNodeState(nodeInfo.getNode()).getState(); - if (currentState != RETIRED) { - return createDisallowed("Only retired nodes are allowed to be set to DOWN in safe mode - is " - + currentState); - } + if (currentState != RETIRED) + return disallow("Only retired nodes are allowed to be set to DOWN in safe mode - is " + currentState); HostInfo hostInfo = nodeInfo.getHostInfo(); Integer hostInfoNodeVersion = hostInfo.getClusterStateVersionOrNull(); int clusterControllerVersion = clusterState.getVersion(); - if (hostInfoNodeVersion == null || hostInfoNodeVersion != clusterControllerVersion) { - return createDisallowed("Cluster controller at version " + clusterControllerVersion - + " got info for storage node " + nodeInfo.getNodeIndex() + " at a different version " - + hostInfoNodeVersion); - } + int nodeIndex = nodeInfo.getNodeIndex(); + if (hostInfoNodeVersion == null || hostInfoNodeVersion != clusterControllerVersion) + return disallow("Cluster controller at version " + clusterControllerVersion + + " got info for storage node " + nodeIndex + " at a different version " + + hostInfoNodeVersion); - Optional<Metrics.Value> bucketsMetric; - bucketsMetric = hostInfo.getMetrics().getValueAt(BUCKETS_METRIC_NAME, BUCKETS_METRIC_DIMENSIONS); - if (bucketsMetric.isEmpty() || bucketsMetric.get().getLast() == null) { - return createDisallowed("Missing last value of the " + BUCKETS_METRIC_NAME + - " metric for storage node " + nodeInfo.getNodeIndex()); - } + var bucketsMetric = hostInfo.getMetrics().getValueAt(BUCKETS_METRIC_NAME, BUCKETS_METRIC_DIMENSIONS); + if (bucketsMetric.isEmpty() || bucketsMetric.get().getLast() == null) + return disallow("Missing last value of the " + BUCKETS_METRIC_NAME + " metric for storage node " + nodeIndex); long lastBuckets = bucketsMetric.get().getLast(); - if (lastBuckets > 0) { - return createDisallowed("The storage node manages " + lastBuckets + " buckets"); - } + if (lastBuckets > 0) + return disallow("The storage node manages " + lastBuckets + " buckets"); - return allowSettingOfWantedState(); + return allow(); } private Result canSetStateUp(NodeInfo nodeInfo, NodeState oldWantedState) { - if (oldWantedState.getState() == UP) { - // The description is not significant when wanting to set the state to UP - return createAlreadySet(); - } + if (oldWantedState.getState() == UP) + return alreadySet(); // The description is not significant when wanting to set the state to UP - if (nodeInfo.getReportedState().getState() != UP) { - return createDisallowed("Refuse to set wanted state to UP, " + - "since the reported state is not UP (" + - nodeInfo.getReportedState().getState() + ")"); - } + State reportedState = nodeInfo.getReportedState().getState(); + if (reportedState != UP) + return disallow("Refuse to set wanted state to UP, since the reported state is not UP (" + reportedState + ")"); - return allowSettingOfWantedState(); + return allow(); } private Result canSetStateMaintenanceTemporarily(StorageNodeInfo nodeInfo, ClusterState clusterState, String newDescription) { - NodeState oldWantedState = nodeInfo.getUserWantedState(); - if (oldWantedState.getState() != UP && !oldWantedState.getDescription().equals(newDescription)) { - // Refuse to override whatever an operator or unknown entity is doing. If the description is - // identical, we assume it is the same operator. - // - // Note: The new state&description is NOT equal to the old state&description: - // that would have been short-circuited prior to this. - return createDisallowed("A conflicting wanted state is already set: " + - oldWantedState.getState() + ": " + oldWantedState.getDescription()); - } + var result = checkIfStateSetWithDifferentDescription(nodeInfo, newDescription); + if (result.notAllowed()) + return result; if (maxNumberOfGroupsAllowedToBeDown == -1) { - var otherGroupCheck = anotherNodeInAnotherGroupHasWantedState(nodeInfo); - if (otherGroupCheck.settingWantedStateIsNotAllowed()) { - return otherGroupCheck; - } - if (anotherNodeInGroupAlreadyAllowed(nodeInfo, newDescription)) { - return allowSettingOfWantedState(); - } + result = checkIfAnotherNodeInAnotherGroupHasWantedState(nodeInfo); + if (result.notAllowed()) + return result; + if (anotherNodeInGroupAlreadyAllowed(nodeInfo, newDescription)) + return allow(); } else { - var result = otherNodesHaveWantedState(nodeInfo, newDescription, clusterState); - if (result.isPresent()) - return result.get(); + var optionalResult = checkIfOtherNodesHaveWantedState(nodeInfo, newDescription, clusterState); + if (optionalResult.isPresent()) + return optionalResult.get(); } - if (clusterState.getNodeState(nodeInfo.getNode()).getState() == DOWN) { + if (nodeIsDown(clusterState, nodeInfo)) { log.log(FINE, "node is DOWN, allow"); - return allowSettingOfWantedState(); + return allow(); } - Result allNodesAreUpCheck = checkAllNodesAreUp(clusterState); - if (allNodesAreUpCheck.settingWantedStateIsNotAllowed()) { - log.log(FINE, "allNodesAreUpCheck: " + allNodesAreUpCheck); - return allNodesAreUpCheck; + result = checkIfNodesAreUpOrRetired(clusterState); + if (result.notAllowed()) { + log.log(FINE, "nodesAreUpOrRetired: " + result); + return result; } - Result checkDistributorsResult = checkDistributors(nodeInfo.getNode(), clusterState.getVersion()); - if (checkDistributorsResult.settingWantedStateIsNotAllowed()) { - log.log(FINE, "checkDistributors: "+ checkDistributorsResult); - return checkDistributorsResult; + result = checkClusterStateAndRedundancy(nodeInfo.getNode(), clusterState.getVersion()); + if (result.notAllowed()) { + log.log(FINE, "checkDistributors: "+ result); + return result; } - return allowSettingOfWantedState(); + return allow(); + } + + /** Refuse to override whatever an operator or unknown entity is doing. */ + private static Result checkIfStateSetWithDifferentDescription(NodeInfo nodeInfo, String newDescription) { + State oldWantedState = nodeInfo.getUserWantedState().getState(); + String oldDescription = nodeInfo.getUserWantedState().getDescription(); + if (oldWantedState != UP && ! oldDescription.equals(newDescription)) + return disallow("A conflicting wanted state is already set: " + oldWantedState + ": " + oldDescription); + + return allow(); } /** * Returns a disallow-result if there is another node (in another group, if hierarchical) * that has a wanted state != UP. We disallow more than 1 suspended node/group at a time. */ - private Result anotherNodeInAnotherGroupHasWantedState(StorageNodeInfo nodeInfo) { + private Result checkIfAnotherNodeInAnotherGroupHasWantedState(StorageNodeInfo nodeInfo) { if (groupVisiting.isHierarchical()) { SettableOptional<Result> anotherNodeHasWantedState = new SettableOptional<>(); groupVisiting.visit(group -> { if (!groupContainsNode(group, nodeInfo.getNode())) { Result result = otherNodeInGroupHasWantedState(group); - if (result.settingWantedStateIsNotAllowed()) { + if (result.notAllowed()) { anotherNodeHasWantedState.set(result); // Have found a node that is suspended, halt the visiting return false; @@ -281,7 +216,7 @@ public class NodeStateChangeChecker { return true; }); - return anotherNodeHasWantedState.asOptional().orElseGet(Result::allowSettingOfWantedState); + return anotherNodeHasWantedState.asOptional().orElseGet(Result::allow); } else { // Returns a disallow-result if there is another node with a wanted state return otherNodeHasWantedState(nodeInfo); @@ -296,7 +231,7 @@ public class NodeStateChangeChecker { * if less than maxNumberOfGroupsAllowedToBeDown: return Optional.of(allowed) * else: if node is in group with nodes already down: return Optional.of(allowed), else Optional.of(disallowed) */ - private Optional<Result> otherNodesHaveWantedState(StorageNodeInfo nodeInfo, String newDescription, ClusterState clusterState) { + private Optional<Result> checkIfOtherNodesHaveWantedState(StorageNodeInfo nodeInfo, String newDescription, ClusterState clusterState) { Node node = nodeInfo.getNode(); if (groupVisiting.isHierarchical()) { @@ -309,12 +244,12 @@ public class NodeStateChangeChecker { Set<Integer> groupsWithSameStateAndDescription = groupsWithSameStateAndDescription(MAINTENANCE, newDescription); if (aGroupContainsNode(groupsWithSameStateAndDescription, node)) { log.log(FINE, "Node is in group with same state and description, allow"); - return Optional.of(allowSettingOfWantedState()); + return Optional.of(allow()); } // There are groups with nodes not up, but with another description, probably operator set if (groupsWithSameStateAndDescription.size() == 0) { - return Optional.of(createDisallowed("Wanted state already set for another node in groups: " + - sortSetIntoList(groupsWithNodesWantedStateNotUp))); + return Optional.of(disallow("Wanted state already set for another node in groups: " + + sortSetIntoList(groupsWithNodesWantedStateNotUp))); } Set<Integer> retiredAndNotUpGroups = groupsWithNotRetiredAndNotUp(clusterState); @@ -326,21 +261,25 @@ public class NodeStateChangeChecker { } if (numberOfGroupsToConsider < maxNumberOfGroupsAllowedToBeDown) { log.log(FINE, "Allow, retiredAndNotUpGroups=" + retiredAndNotUpGroups); - return Optional.of(allowSettingOfWantedState()); + return Optional.of(allow()); } - return Optional.of(createDisallowed(String.format("At most %d groups can have wanted state: %s", - maxNumberOfGroupsAllowedToBeDown, - sortSetIntoList(retiredAndNotUpGroups)))); + return Optional.of(disallow(String.format("At most %d groups can have wanted state: %s", + maxNumberOfGroupsAllowedToBeDown, + sortSetIntoList(retiredAndNotUpGroups)))); } else { // Return a disallow-result if there is another node with a wanted state var otherNodeHasWantedState = otherNodeHasWantedState(nodeInfo); - if (otherNodeHasWantedState.settingWantedStateIsNotAllowed()) + if (otherNodeHasWantedState.notAllowed()) return Optional.of(otherNodeHasWantedState); } return Optional.empty(); } + private static boolean nodeIsDown(ClusterState clusterState, NodeInfo nodeInfo) { + return clusterState.getNodeState(nodeInfo.getNode()).getState() == DOWN; + } + private ArrayList<Integer> sortSetIntoList(Set<Integer> set) { var sortedList = new ArrayList<>(set); Collections.sort(sortedList); @@ -354,55 +293,46 @@ public class NodeStateChangeChecker { StorageNodeInfo storageNodeInfo = clusterInfo.getStorageNodeInfo(index); if (storageNodeInfo == null) continue; // needed for tests only State storageNodeWantedState = storageNodeInfo.getUserWantedState().getState(); - if (storageNodeWantedState != UP) { - return createDisallowed( - "At most one group can have wanted state: Other storage node " + index + - " in group " + group.getIndex() + " has wanted state " + storageNodeWantedState); - } + if (storageNodeWantedState != UP) + return disallow("At most one group can have wanted state: Other storage node " + index + + " in group " + group.getIndex() + " has wanted state " + storageNodeWantedState); State distributorWantedState = clusterInfo.getDistributorNodeInfo(index).getUserWantedState().getState(); - if (distributorWantedState != UP) { - return createDisallowed( - "At most one group can have wanted state: Other distributor " + index + - " in group " + group.getIndex() + " has wanted state " + distributorWantedState); - } + if (distributorWantedState != UP) + return disallow("At most one group can have wanted state: Other distributor " + index + + " in group " + group.getIndex() + " has wanted state " + distributorWantedState); } - return allowSettingOfWantedState(); + return allow(); } private Result otherNodeHasWantedState(StorageNodeInfo nodeInfo) { for (var configuredNode : clusterInfo.getConfiguredNodes().values()) { int index = configuredNode.index(); - if (index == nodeInfo.getNodeIndex()) { - continue; - } + if (index == nodeInfo.getNodeIndex()) continue; State storageNodeWantedState = clusterInfo.getStorageNodeInfo(index).getUserWantedState().getState(); if (storageNodeWantedState != UP) { - return createDisallowed( - "At most one node can have a wanted state when #groups = 1: Other storage node " + + return disallow("At most one node can have a wanted state when #groups = 1: Other storage node " + index + " has wanted state " + storageNodeWantedState); } State distributorWantedState = clusterInfo.getDistributorNodeInfo(index).getUserWantedState().getState(); if (distributorWantedState != UP) { - return createDisallowed( - "At most one node can have a wanted state when #groups = 1: Other distributor " + + return disallow("At most one node can have a wanted state when #groups = 1: Other distributor " + index + " has wanted state " + distributorWantedState); } } - return allowSettingOfWantedState(); + return allow(); } private boolean anotherNodeInGroupAlreadyAllowed(StorageNodeInfo nodeInfo, String newDescription) { MutableBoolean alreadyAllowed = new MutableBoolean(false); groupVisiting.visit(group -> { - if (!groupContainsNode(group, nodeInfo.getNode())) { + if (!groupContainsNode(group, nodeInfo.getNode())) return true; - } alreadyAllowed.set(anotherNodeInGroupAlreadyAllowed(group, nodeInfo.getNode(), newDescription)); @@ -425,9 +355,8 @@ public class NodeStateChangeChecker { private static boolean groupContainsNode(Group group, Node node) { for (ConfiguredNode configuredNode : group.getNodes()) { - if (configuredNode.index() == node.getIndex()) { + if (configuredNode.index() == node.getIndex()) return true; - } } return false; @@ -449,61 +378,42 @@ public class NodeStateChangeChecker { .collect(Collectors.toList()); } - private Result checkAllNodesAreUp(ClusterState clusterState) { - // This method verifies both storage nodes and distributors are up (or retired). - // The complicated part is making a summary error message. - - for (NodeInfo storageNodeInfo : clusterInfo.getStorageNodeInfos()) { - State wantedState = storageNodeInfo.getUserWantedState().getState(); - if (wantedState != UP && wantedState != RETIRED) { - return createDisallowed("Another storage node wants state " + - wantedState.toString().toUpperCase() + ": " + storageNodeInfo.getNodeIndex()); - } - - State state = clusterState.getNodeState(storageNodeInfo.getNode()).getState(); - if (state != UP && state != RETIRED) { - return createDisallowed("Another storage node has state " + state.toString().toUpperCase() + - ": " + storageNodeInfo.getNodeIndex()); - } - } - - for (NodeInfo distributorNodeInfo : clusterInfo.getDistributorNodeInfos()) { - State wantedState = distributorNodeInfo.getUserWantedState().getState(); - if (wantedState != UP && wantedState != RETIRED) { - return createDisallowed("Another distributor wants state " + wantedState.toString().toUpperCase() + - ": " + distributorNodeInfo.getNodeIndex()); - } + /** Verifies that storage nodes and distributors are up (or retired). */ + private Result checkIfNodesAreUpOrRetired(ClusterState clusterState) { + for (NodeInfo nodeInfo : clusterInfo.getAllNodeInfos()) { + State wantedState = nodeInfo.getUserWantedState().getState(); + if (wantedState != UP && wantedState != RETIRED) + return disallow("Another " + nodeInfo.type() + " wants state " + + wantedState.toString().toUpperCase() + ": " + nodeInfo.getNodeIndex()); - State state = clusterState.getNodeState(distributorNodeInfo.getNode()).getState(); - if (state != UP && state != RETIRED) { - return createDisallowed("Another distributor has state " + state.toString().toUpperCase() + - ": " + distributorNodeInfo.getNodeIndex()); - } + State state = clusterState.getNodeState(nodeInfo.getNode()).getState(); + if (state != UP && state != RETIRED) + return disallow("Another " + nodeInfo.type() + " has state " + + state.toString().toUpperCase() + ": " + nodeInfo.getNodeIndex()); } - return allowSettingOfWantedState(); + return allow(); } - private Result checkStorageNodesForDistributor(DistributorNodeInfo distributorNodeInfo, Node node) { + private Result checkRedundancy(DistributorNodeInfo distributorNodeInfo, Node node) { List<StorageNode> storageNodes = distributorNodeInfo.getHostInfo().getDistributor().getStorageNodes(); for (StorageNode storageNode : storageNodes) { if (storageNode.getIndex() == node.getIndex()) { Integer minReplication = storageNode.getMinCurrentReplicationFactorOrNull(); // Why test on != null? Missing min-replication is OK (indicate empty/few buckets on system). if (minReplication != null && minReplication < requiredRedundancy) { - return createDisallowed("Distributor " - + distributorNodeInfo.getNodeIndex() + return disallow("Distributor " + distributorNodeInfo.getNodeIndex() + " says storage node " + node.getIndex() + " has buckets with redundancy as low as " + storageNode.getMinCurrentReplicationFactorOrNull() + ", but we require at least " + requiredRedundancy); } else { - return allowSettingOfWantedState(); + return allow(); } } } - return allowSettingOfWantedState(); + return allow(); } /** @@ -511,29 +421,29 @@ public class NodeStateChangeChecker { * @param node the node to be checked * @param clusterStateVersion the cluster state we expect distributors to have */ - private Result checkDistributors(Node node, int clusterStateVersion) { - if (clusterInfo.getDistributorNodeInfos().isEmpty()) { - return createDisallowed("Not aware of any distributors, probably not safe to upgrade?"); - } + private Result checkClusterStateAndRedundancy(Node node, int clusterStateVersion) { + if (clusterInfo.getDistributorNodeInfos().isEmpty()) + return disallow("Not aware of any distributors, probably not safe to upgrade?"); + for (DistributorNodeInfo distributorNodeInfo : clusterInfo.getDistributorNodeInfos()) { Integer distributorClusterStateVersion = distributorNodeInfo.getHostInfo().getClusterStateVersionOrNull(); - if (distributorClusterStateVersion == null) { - return createDisallowed("Distributor node " + distributorNodeInfo.getNodeIndex() - + " has not reported any cluster state version yet."); - } else if (distributorClusterStateVersion != clusterStateVersion) { - return createDisallowed("Distributor node " + distributorNodeInfo.getNodeIndex() - + " does not report same version (" - + distributorNodeInfo.getHostInfo().getClusterStateVersionOrNull() - + ") as fleetcontroller (" + clusterStateVersion + ")"); + if (distributorClusterStateVersion == null) + return disallow("Distributor node " + distributorNodeInfo.getNodeIndex() + + " has not reported any cluster state version yet."); + if (distributorClusterStateVersion != clusterStateVersion) { + return disallow("Distributor node " + distributorNodeInfo.getNodeIndex() + + " does not report same version (" + + distributorNodeInfo.getHostInfo().getClusterStateVersionOrNull() + + ") as fleetcontroller (" + clusterStateVersion + ")"); } - Result storageNodesResult = checkStorageNodesForDistributor(distributorNodeInfo, node); - if (storageNodesResult.settingWantedStateIsNotAllowed()) { + Result storageNodesResult = checkRedundancy(distributorNodeInfo, node); + if (storageNodesResult.notAllowed()) { return storageNodesResult; } } - return allowSettingOfWantedState(); + return allow(); } private Set<Integer> groupsWithUserWantedStateNotUp() { @@ -575,4 +485,50 @@ public class NodeStateChangeChecker { .collect(Collectors.toSet()); } + public static class Result { + + public enum Action { + ALLOWED, + ALREADY_SET, + DISALLOWED + } + + private final Action action; + private final String reason; + + private Result(Action action, String reason) { + this.action = action; + this.reason = reason; + } + + public static Result disallow(String reason) { + return new Result(Action.DISALLOWED, reason); + } + + public static Result allow() { + return new Result(Action.ALLOWED, "Preconditions fulfilled and new state different"); + } + + public static Result alreadySet() { + return new Result(Action.ALREADY_SET, "Basic preconditions fulfilled and new state is already effective"); + } + + public boolean allowed() { return action == Action.ALLOWED; } + + public boolean notAllowed() { return ! allowed(); } + + public boolean isAlreadySet() { + return action == Action.ALREADY_SET; + } + + public String reason() { + return reason; + } + + public String toString() { + return "action " + action + ": " + reason; + } + + } + } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java index 01a75034ddf..bfbe0f795fc 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java @@ -9,7 +9,6 @@ import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.ContentCluster; import com.yahoo.vespa.clustercontroller.core.NodeInfo; -import com.yahoo.vespa.clustercontroller.core.NodeStateChangeChecker; import com.yahoo.vespa.clustercontroller.core.RemoteClusterControllerTask; import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; import com.yahoo.vespa.clustercontroller.core.restapiv2.Id; @@ -145,7 +144,7 @@ public class SetNodeStateRequest extends Request<SetResponse> { probe); // If the state was successfully set, just return an "ok" message back. - String reason = success ? "ok" : result.getReason(); + String reason = success ? "ok" : result.reason(); return new SetResponse(reason, success); } @@ -154,19 +153,19 @@ public class SetNodeStateRequest extends Request<SetResponse> { * wanted state, or the requested state has been accepted as the new wanted state. */ private static boolean setWantedStateAccordingToResult( - NodeStateChangeChecker.Result result, + Result result, NodeState newWantedState, SetUnitStateRequest.Condition condition, NodeInfo nodeInfo, ContentCluster cluster, NodeListener stateListener, boolean probe) { - if (result.settingWantedStateIsAllowed()) { + if (result.allowed()) { setNewWantedState(nodeInfo, newWantedState, stateListener, probe); } // True if the wanted state was or has just been set to newWantedState - boolean success = result.settingWantedStateIsAllowed() || result.wantedStateAlreadySet(); + boolean success = result.allowed() || result.isAlreadySet(); if (success && condition == SetUnitStateRequest.Condition.SAFE && nodeInfo.isStorage()) { // In safe-mode, setting the storage node must be accompanied by changing the state diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java index 43687d51937..7b20fcf694a 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java @@ -115,8 +115,8 @@ public class NodeStateChangeCheckerTest { Result result = nodeStateChangeChecker.evaluateTransition( nodeDistributor, defaultAllUpClusterState(), FORCE, UP_NODE_STATE, newState); - assertTrue(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); + assertTrue(result.allowed()); + assertFalse(result.isAlreadySet()); } @ParameterizedTest @@ -127,9 +127,9 @@ public class NodeStateChangeCheckerTest { Result result = nodeStateChangeChecker.evaluateTransition( new Node(STORAGE, 10), defaultAllUpClusterState(), SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); - assertFalse(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); - assertEquals("Master cluster controller is bootstrapping and in moratorium", result.getReason()); + assertFalse(result.allowed()); + assertFalse(result.isAlreadySet()); + assertEquals("Master cluster controller is bootstrapping and in moratorium", result.reason()); } @ParameterizedTest @@ -140,9 +140,9 @@ public class NodeStateChangeCheckerTest { Result result = nodeStateChangeChecker.evaluateTransition( new Node(STORAGE, 10), defaultAllUpClusterState(), SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); - assertFalse(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); - assertEquals("Unknown node storage.10", result.getReason()); + assertFalse(result.allowed()); + assertFalse(result.isAlreadySet()); + assertEquals("Unknown node storage.10", result.reason()); } @ParameterizedTest @@ -159,10 +159,10 @@ public class NodeStateChangeCheckerTest { Result result = nodeStateChangeChecker.evaluateTransition( new Node(STORAGE, 1), clusterStateWith0InMaintenance, SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); - assertFalse(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); + assertFalse(result.allowed()); + assertFalse(result.isAlreadySet()); assertEquals("At most one node can have a wanted state when #groups = 1: Other storage node 0 has wanted state Maintenance", - result.getReason()); + result.reason()); } @Test @@ -195,9 +195,9 @@ public class NodeStateChangeCheckerTest { cluster.clusterInfo().getStorageNodeInfo(nodeIndex).setReportedState(new NodeState(STORAGE, DOWN), 0); Node node = new Node(STORAGE, nodeIndex); Result result = nodeStateChangeChecker.evaluateTransition(node, clusterState, SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); - assertFalse(result.settingWantedStateIsAllowed(), result.toString()); - assertFalse(result.wantedStateAlreadySet()); - assertEquals("At most 2 groups can have wanted state: [0, 1, 2]", result.getReason()); + assertFalse(result.allowed(), result.toString()); + assertFalse(result.isAlreadySet()); + assertEquals("At most 2 groups can have wanted state: [0, 1, 2]", result.reason()); } // Nodes in group 0 and 1 in maintenance, try to set storage node in group 2 to maintenance, should fail @@ -206,9 +206,9 @@ public class NodeStateChangeCheckerTest { int nodeIndex = 2; Node node = new Node(STORAGE, nodeIndex); Result result = nodeStateChangeChecker.evaluateTransition(node, clusterState, SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); - assertFalse(result.settingWantedStateIsAllowed(), result.toString()); - assertFalse(result.wantedStateAlreadySet()); - assertEquals("At most 2 groups can have wanted state: [0, 1]", result.getReason()); + assertFalse(result.allowed(), result.toString()); + assertFalse(result.isAlreadySet()); + assertEquals("At most 2 groups can have wanted state: [0, 1]", result.reason()); } } @@ -251,9 +251,9 @@ public class NodeStateChangeCheckerTest { int nodeIndex = 4; Node node = new Node(STORAGE, nodeIndex); Result result = nodeStateChangeChecker.evaluateTransition(node, clusterState, SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); - assertFalse(result.settingWantedStateIsAllowed(), result.toString()); - assertFalse(result.wantedStateAlreadySet()); - assertEquals("At most 2 groups can have wanted state: [0, 1]", result.getReason()); + assertFalse(result.allowed(), result.toString()); + assertFalse(result.isAlreadySet()); + assertEquals("At most 2 groups can have wanted state: [0, 1]", result.reason()); } // 2 nodes in group 0 and 1 in group 1 in maintenance, try to set storage node 3 in group 1 to maintenance @@ -270,9 +270,9 @@ public class NodeStateChangeCheckerTest { int nodeIndex = 4; Node node = new Node(STORAGE, nodeIndex); Result result = nodeStateChangeChecker.evaluateTransition(node, clusterState, SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); - assertFalse(result.settingWantedStateIsAllowed(), result.toString()); - assertFalse(result.wantedStateAlreadySet()); - assertEquals("At most 2 groups can have wanted state: [0, 1]", result.getReason()); + assertFalse(result.allowed(), result.toString()); + assertFalse(result.isAlreadySet()); + assertEquals("At most 2 groups can have wanted state: [0, 1]", result.reason()); } // 2 nodes in group 0 up again but buckets not in sync and 2 nodes in group 1 in maintenance, @@ -301,8 +301,8 @@ public class NodeStateChangeCheckerTest { int nodeIndex = 2; Node node = new Node(STORAGE, nodeIndex); Result result = nodeStateChangeChecker.evaluateTransition(node, clusterState, SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); - assertTrue(result.settingWantedStateIsAllowed(), result.toString()); - assertFalse(result.wantedStateAlreadySet()); + assertTrue(result.allowed(), result.toString()); + assertFalse(result.isAlreadySet()); } } @@ -321,10 +321,10 @@ public class NodeStateChangeCheckerTest { Result result = nodeStateChangeChecker.evaluateTransition( new Node(STORAGE, 1), clusterStateWith0InMaintenance, SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); - assertFalse(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); + assertFalse(result.allowed()); + assertFalse(result.isAlreadySet()); assertEquals("At most one node can have a wanted state when #groups = 1: Other distributor 0 has wanted state Down", - result.getReason()); + result.reason()); } @ParameterizedTest @@ -344,12 +344,12 @@ public class NodeStateChangeCheckerTest { Result result = nodeStateChangeChecker.evaluateTransition( new Node(STORAGE, 2), clusterStateWith0InMaintenance, SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); - assertFalse(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); + assertFalse(result.allowed()); + assertFalse(result.isAlreadySet()); if (maxNumberOfGroupsAllowedToBeDown >= 1) - assertEquals("Wanted state already set for another node in groups: [0]", result.getReason()); + assertEquals("Wanted state already set for another node in groups: [0]", result.reason()); else - assertEquals("At most one group can have wanted state: Other distributor 0 in group 0 has wanted state Down", result.getReason()); + assertEquals("At most one group can have wanted state: Other distributor 0 in group 0 has wanted state Down", result.reason()); } { @@ -359,11 +359,11 @@ public class NodeStateChangeCheckerTest { new Node(STORAGE, 1), clusterStateWith0InMaintenance, SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); if (maxNumberOfGroupsAllowedToBeDown >= 1) { - assertFalse(result.settingWantedStateIsAllowed(), result.getReason()); - assertEquals("Wanted state already set for another node in groups: [0]", result.getReason()); + assertFalse(result.allowed(), result.reason()); + assertEquals("Wanted state already set for another node in groups: [0]", result.reason()); } else { - assertFalse(result.settingWantedStateIsAllowed(), result.getReason()); - assertEquals("Another distributor wants state DOWN: 0", result.getReason()); + assertFalse(result.allowed(), result.reason()); + assertEquals("Another distributor wants state DOWN: 0", result.reason()); } } } @@ -385,13 +385,13 @@ public class NodeStateChangeCheckerTest { Result result = nodeStateChangeChecker.evaluateTransition( new Node(STORAGE, 2), clusterStateWith0InMaintenance, SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); - assertFalse(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); + assertFalse(result.allowed()); + assertFalse(result.isAlreadySet()); if (maxNumberOfGroupsAllowedToBeDown >= 1) - assertEquals("At most 1 groups can have wanted state: [0]", result.getReason()); + assertEquals("At most 1 groups can have wanted state: [0]", result.reason()); else assertEquals("At most one group can have wanted state: Other storage node 0 in group 0 has wanted state Maintenance", - result.getReason()); + result.reason()); } { @@ -400,8 +400,8 @@ public class NodeStateChangeCheckerTest { Result result = nodeStateChangeChecker.evaluateTransition( new Node(STORAGE, 1), clusterStateWith0InMaintenance, SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); - assertTrue(result.settingWantedStateIsAllowed(), result.getReason()); - assertFalse(result.wantedStateAlreadySet()); + assertTrue(result.allowed(), result.reason()); + assertFalse(result.isAlreadySet()); } } @@ -412,9 +412,9 @@ public class NodeStateChangeCheckerTest { Result result = nodeStateChangeChecker.evaluateTransition( nodeDistributor, defaultAllUpClusterState(), SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); - assertFalse(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); - assertTrue(result.getReason().contains("Safe-set of node state is only supported for storage nodes")); + assertFalse(result.allowed()); + assertFalse(result.isAlreadySet()); + assertTrue(result.reason().contains("Safe-set of node state is only supported for storage nodes")); } @ParameterizedTest @@ -433,17 +433,17 @@ public class NodeStateChangeCheckerTest { Result result = nodeStateChangeChecker.evaluateTransition( nodeStorage, clusterStateWith3Down, SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); - assertFalse(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); - assertEquals("Another storage node has state DOWN: 3", result.getReason()); + assertFalse(result.allowed()); + assertFalse(result.isAlreadySet()); + assertEquals("Another storage node has state DOWN: 3", result.reason()); } @ParameterizedTest @ValueSource(ints = {-1, 1}) void testCanUpgradeStorageSafeYes(int maxNumberOfGroupsAllowedToBeDown) { Result result = transitionToMaintenanceWithNoStorageNodesDown(createCluster(4, 1, maxNumberOfGroupsAllowedToBeDown), defaultAllUpClusterState()); - assertTrue(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); + assertTrue(result.allowed()); + assertFalse(result.isAlreadySet()); } @ParameterizedTest @@ -456,8 +456,8 @@ public class NodeStateChangeCheckerTest { Result result = nodeStateChangeChecker.evaluateTransition( nodeStorage, defaultAllUpClusterState(), SAFE, MAINTENANCE_NODE_STATE, UP_NODE_STATE); - assertFalse(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); + assertFalse(result.allowed()); + assertFalse(result.isAlreadySet()); } // A node may be reported as Up but have a generated state of Down if it's part of @@ -477,8 +477,8 @@ public class NodeStateChangeCheckerTest { Result result = nodeStateChangeChecker.evaluateTransition( nodeStorage, stateWithNodeDown, SAFE, MAINTENANCE_NODE_STATE, UP_NODE_STATE); - assertTrue(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); + assertTrue(result.allowed()); + assertFalse(result.isAlreadySet()); } @ParameterizedTest @@ -491,8 +491,8 @@ public class NodeStateChangeCheckerTest { Result result = nodeStateChangeChecker.evaluateTransition( nodeStorage, defaultAllUpClusterState(), SAFE, new NodeState(STORAGE, DOWN), UP_NODE_STATE); - assertTrue(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); + assertTrue(result.allowed()); + assertFalse(result.isAlreadySet()); } @ParameterizedTest @@ -505,10 +505,10 @@ public class NodeStateChangeCheckerTest { Result result = nodeStateChangeChecker.evaluateTransition( nodeStorage, defaultAllUpClusterState(), SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); - assertFalse(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); + assertFalse(result.allowed()); + assertFalse(result.isAlreadySet()); assertEquals("Distributor 0 says storage node 1 has buckets with redundancy as low as 3, but we require at least 4", - result.getReason()); + result.reason()); } @ParameterizedTest @@ -521,8 +521,8 @@ public class NodeStateChangeCheckerTest { Result result = nodeStateChangeChecker.evaluateTransition( new Node(STORAGE, 3), defaultAllUpClusterState(), SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); - assertTrue(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); + assertTrue(result.allowed()); + assertFalse(result.isAlreadySet()); } @ParameterizedTest @@ -546,8 +546,8 @@ public class NodeStateChangeCheckerTest { Result result = nodeStateChangeChecker.evaluateTransition( new Node(STORAGE, 1), defaultAllUpClusterState(), SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); - assertTrue(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); + assertTrue(result.allowed()); + assertFalse(result.isAlreadySet()); } @ParameterizedTest @@ -559,9 +559,9 @@ public class NodeStateChangeCheckerTest { Result result = nodeStateChangeChecker.evaluateTransition( nodeStorage, defaultAllUpClusterState(), SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); - assertFalse(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); - assertEquals("Distributor node 0 has not reported any cluster state version yet.", result.getReason()); + assertFalse(result.allowed()); + assertFalse(result.isAlreadySet()); + assertEquals("Distributor node 0 has not reported any cluster state version yet.", result.reason()); } private Result transitionToSameState(State state, String oldDescription, String newDescription, int maxNumberOfGroupsAllowedToBeDown) { @@ -583,23 +583,23 @@ public class NodeStateChangeCheckerTest { @ValueSource(ints = {-1, 1}) void testSettingUpWhenUpCausesAlreadySet(int maxNumberOfGroupsAllowedToBeDown) { Result result = transitionToSameState(UP, "foo", "bar", maxNumberOfGroupsAllowedToBeDown); - assertTrue(result.wantedStateAlreadySet()); + assertTrue(result.isAlreadySet()); } @ParameterizedTest @ValueSource(ints = {-1, 1}) void testSettingAlreadySetState(int maxNumberOfGroupsAllowedToBeDown) { Result result = transitionToSameState("foo", "foo", maxNumberOfGroupsAllowedToBeDown); - assertFalse(result.settingWantedStateIsAllowed()); - assertTrue(result.wantedStateAlreadySet()); + assertFalse(result.allowed()); + assertTrue(result.isAlreadySet()); } @ParameterizedTest @ValueSource(ints = {-1, 1}) void testDifferentDescriptionImpliesDenied(int maxNumberOfGroupsAllowedToBeDown) { Result result = transitionToSameState("foo", "bar", maxNumberOfGroupsAllowedToBeDown); - assertFalse(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); + assertFalse(result.allowed()); + assertFalse(result.isAlreadySet()); } private Result transitionToMaintenanceWithOneStorageNodeDown(ContentCluster cluster, ClusterState clusterState) { @@ -632,16 +632,16 @@ public class NodeStateChangeCheckerTest { @ValueSource(ints = {-1, 1}) void testCanUpgradeWhenAllUp(int maxNumberOfGroupsAllowedToBeDown) { Result result = transitionToMaintenanceWithNoStorageNodesDown(createCluster(4, maxNumberOfGroupsAllowedToBeDown), defaultAllUpClusterState()); - assertTrue(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); + assertTrue(result.allowed()); + assertFalse(result.isAlreadySet()); } @ParameterizedTest @ValueSource(ints = {-1, 1}) void testCanUpgradeWhenAllUpOrRetired(int maxNumberOfGroupsAllowedToBeDown) { Result result = transitionToMaintenanceWithNoStorageNodesDown(createCluster(4, maxNumberOfGroupsAllowedToBeDown), defaultAllUpClusterState()); - assertTrue(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); + assertTrue(result.allowed()); + assertFalse(result.isAlreadySet()); } @ParameterizedTest @@ -656,8 +656,8 @@ public class NodeStateChangeCheckerTest { clusterState.setNodeState(new Node(STORAGE, storageNodeIndex), downNodeState); Result result = transitionToMaintenanceWithOneStorageNodeDown(cluster, clusterState); - assertTrue(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); + assertTrue(result.allowed()); + assertFalse(result.isAlreadySet()); } @ParameterizedTest @@ -674,9 +674,9 @@ public class NodeStateChangeCheckerTest { clusterState.setNodeState(new Node(STORAGE, otherIndex), downNodeState); Result result = transitionToMaintenanceWithOneStorageNodeDown(cluster, clusterState); - assertFalse(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); - assertTrue(result.getReason().contains("Another storage node has state DOWN: 2")); + assertFalse(result.allowed()); + assertFalse(result.isAlreadySet()); + assertTrue(result.reason().contains("Another storage node has state DOWN: 2")); } @ParameterizedTest @@ -698,8 +698,8 @@ public class NodeStateChangeCheckerTest { Result result = nodeStateChangeChecker.evaluateTransition( nodeStorage, stateWithNodeDown, SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); - assertFalse(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); + assertFalse(result.allowed()); + assertFalse(result.isAlreadySet()); } @ParameterizedTest @@ -711,9 +711,9 @@ public class NodeStateChangeCheckerTest { currentClusterStateVersion, 0, maxNumberOfGroupsAllowedToBeDown); - assertFalse(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); - assertEquals("Only retired nodes are allowed to be set to DOWN in safe mode - is Up", result.getReason()); + assertFalse(result.allowed()); + assertFalse(result.isAlreadySet()); + assertEquals("Only retired nodes are allowed to be set to DOWN in safe mode - is Up", result.reason()); } @ParameterizedTest @@ -725,9 +725,9 @@ public class NodeStateChangeCheckerTest { currentClusterStateVersion, 1, maxNumberOfGroupsAllowedToBeDown); - assertFalse(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); - assertEquals("The storage node manages 1 buckets", result.getReason()); + assertFalse(result.allowed()); + assertFalse(result.isAlreadySet()); + assertEquals("The storage node manages 1 buckets", result.reason()); } @ParameterizedTest @@ -739,9 +739,9 @@ public class NodeStateChangeCheckerTest { currentClusterStateVersion, 0, maxNumberOfGroupsAllowedToBeDown); - assertFalse(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); - assertEquals("Reported state (Initializing) is not UP, so no bucket data is available", result.getReason()); + assertFalse(result.allowed()); + assertFalse(result.isAlreadySet()); + assertEquals("Reported state (Initializing) is not UP, so no bucket data is available", result.reason()); } @ParameterizedTest @@ -753,10 +753,10 @@ public class NodeStateChangeCheckerTest { currentClusterStateVersion - 1, 0, maxNumberOfGroupsAllowedToBeDown); - assertFalse(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); + assertFalse(result.allowed()); + assertFalse(result.isAlreadySet()); assertEquals("Cluster controller at version 2 got info for storage node 1 at a different version 1", - result.getReason()); + result.reason()); } @ParameterizedTest @@ -768,8 +768,8 @@ public class NodeStateChangeCheckerTest { currentClusterStateVersion, 0, maxNumberOfGroupsAllowedToBeDown); - assertTrue(result.settingWantedStateIsAllowed()); - assertFalse(result.wantedStateAlreadySet()); + assertTrue(result.allowed()); + assertFalse(result.isAlreadySet()); } private Result evaluateDownTransition(ClusterState clusterState, @@ -948,9 +948,9 @@ public class NodeStateChangeCheckerTest { private void checkSettingToMaintenanceIsAllowed(int nodeIndex, NodeStateChangeChecker nodeStateChangeChecker, ClusterState clusterState) { Node node = new Node(STORAGE, nodeIndex); Result result = nodeStateChangeChecker.evaluateTransition(node, clusterState, SAFE, UP_NODE_STATE, MAINTENANCE_NODE_STATE); - assertTrue(result.settingWantedStateIsAllowed(), result.toString()); - assertFalse(result.wantedStateAlreadySet()); - assertEquals("Preconditions fulfilled and new state different", result.getReason()); + assertTrue(result.allowed(), result.toString()); + assertFalse(result.isAlreadySet()); + assertEquals("Preconditions fulfilled and new state different", result.reason()); } private void setStorageNodeWantedStateToMaintenance(ContentCluster cluster, int nodeIndex) { diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequestTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequestTest.java index 6d93eadfe2a..f2f38954f55 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequestTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequestTest.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.clustercontroller.core.NodeStateChangeChecker; import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; import com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.StateRestApiException; import com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest; -import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.SetResponse; import com.yahoo.vespa.clustercontroller.utils.staterestapi.response.UnitState; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -20,6 +19,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; +import static com.yahoo.vespa.clustercontroller.core.NodeStateChangeChecker.Result; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.eq; @@ -54,7 +54,7 @@ public class SetNodeStateRequestTest { testSetStateRequest( "maintenance", State.UP, State.UP, - NodeStateChangeChecker.Result.allowSettingOfWantedState(), + Result.allow(), Optional.of(State.MAINTENANCE), Optional.of(State.DOWN)); } @@ -64,7 +64,7 @@ public class SetNodeStateRequestTest { testSetStateRequest( "maintenance", State.UP, State.UP, - NodeStateChangeChecker.Result.allowSettingOfWantedState(), + Result.allow(), Optional.empty(), Optional.empty()); } @@ -73,7 +73,7 @@ public class SetNodeStateRequestTest { testSetStateRequest( "down", State.UP, State.UP, - NodeStateChangeChecker.Result.allowSettingOfWantedState(), + Result.allow(), Optional.of(State.DOWN), Optional.of(State.DOWN)); } @@ -82,7 +82,7 @@ public class SetNodeStateRequestTest { testSetStateRequest( "up", State.MAINTENANCE, State.DOWN, - NodeStateChangeChecker.Result.allowSettingOfWantedState(), + Result.allow(), Optional.of(State.UP), Optional.of(State.UP)); } @@ -91,7 +91,7 @@ public class SetNodeStateRequestTest { testSetStateRequest( "up", State.DOWN, State.DOWN, - NodeStateChangeChecker.Result.allowSettingOfWantedState(), + Result.allow(), Optional.of(State.UP), Optional.of(State.UP)); } @@ -100,7 +100,7 @@ public class SetNodeStateRequestTest { testSetStateRequest( "maintenance", State.MAINTENANCE, State.UP, - NodeStateChangeChecker.Result.createAlreadySet(), + Result.alreadySet(), Optional.empty(), Optional.of(State.DOWN)); } @@ -109,7 +109,7 @@ public class SetNodeStateRequestTest { testSetStateRequest( "maintenance", State.MAINTENANCE, State.DOWN, - NodeStateChangeChecker.Result.createAlreadySet(), + Result.alreadySet(), Optional.empty(), Optional.empty()); } @@ -168,8 +168,8 @@ public class SetNodeStateRequestTest { } } - private SetResponse setWantedState() throws StateRestApiException { - return SetNodeStateRequest.setWantedState( + private void setWantedState() throws StateRestApiException { + SetNodeStateRequest.setWantedState( cluster, condition, newStates, diff --git a/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/RankTypeResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/RankTypeResolver.java index 6424fd8ba06..e86ac6dabfc 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/RankTypeResolver.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/RankTypeResolver.java @@ -11,7 +11,7 @@ import java.util.logging.Level; /** * Checks if fields have defined different rank types for the same - * index (typically in an index-to statement), and if they have + * index (typically in a fieldset statement), and if they have * output a warning and use the first ranktype. * * @author hmusum diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java index df3cd4103d9..92c9eccf2d3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java @@ -112,9 +112,8 @@ public class ConstantTensorJsonValidator { consumeTopObject(); return; } else if (isScalar()) { - if (top == JsonToken.VALUE_NUMBER_FLOAT || top == JsonToken.VALUE_NUMBER_INT) { - return; - } + throw new InvalidConstantTensorException( + parser, String.format("Invalid type %s: Only tensors with dimensions can be stored as file constants", tensorType.toString())); } throw new InvalidConstantTensorException( parser, String.format("Unexpected first token '%s' for constant with type %s", @@ -315,14 +314,6 @@ public class ConstantTensorJsonValidator { } } - private void assertFieldNameIs(String wantedFieldName) throws IOException { - String actualFieldName = parser.getCurrentName(); - - if (!actualFieldName.equals(wantedFieldName)) { - throw new InvalidConstantTensorException(parser, String.format("Expected field name '%s', got '%s'", wantedFieldName, actualFieldName)); - } - } - static class InvalidConstantTensorException extends IllegalArgumentException { InvalidConstantTensorException(JsonParser parser, String message) { @@ -338,19 +329,6 @@ public class ConstantTensorJsonValidator { } } - @FunctionalInterface - private interface SubroutineThrowingIOException { - void invoke() throws IOException; - } - - private void wrapIOException(SubroutineThrowingIOException lambda) { - try { - lambda.invoke(); - } catch (IOException e) { - throw new InvalidConstantTensorException(parser, e); - } - } - private void consumeValuesNesting(int level) throws IOException { assertCurrentTokenIs(JsonToken.START_ARRAY); if (level >= denseDims.size()) { diff --git a/config-model/src/test/cfg/admin/metricconfig/schemas/music.sd b/config-model/src/test/cfg/admin/metricconfig/schemas/music.sd index f90d805ce6a..71d588662a0 100644 --- a/config-model/src/test/cfg/admin/metricconfig/schemas/music.sd +++ b/config-model/src/test/cfg/admin/metricconfig/schemas/music.sd @@ -3,11 +3,9 @@ search music { document music { field f1 type string { indexing: summary | index - # index-to: f1, all } field f2 type string { indexing: summary | index - # index-to: f2, all } } } diff --git a/config-model/src/test/cfg/application/app1/schemas/music.sd b/config-model/src/test/cfg/application/app1/schemas/music.sd index 92e87848a8a..4e220f96727 100644 --- a/config-model/src/test/cfg/application/app1/schemas/music.sd +++ b/config-model/src/test/cfg/application/app1/schemas/music.sd @@ -7,13 +7,11 @@ search music { field title type string { indexing: summary | index # How this field should be indexed - # index-to: title, default # Create two indexes rank-type: about # Type of ranking settings to apply } field artist type string { indexing: summary | attribute | index - # index-to: artist, default rank-type:about } diff --git a/config-model/src/test/cfg/application/app1/schemas/product.sd b/config-model/src/test/cfg/application/app1/schemas/product.sd index 132ae15053f..70c9343d63a 100644 --- a/config-model/src/test/cfg/application/app1/schemas/product.sd +++ b/config-model/src/test/cfg/application/app1/schemas/product.sd @@ -3,7 +3,6 @@ document product { field title type string { indexing: index | summary - # index-to: title, default } field price type int { diff --git a/config-model/src/test/cfg/application/app_complicated_deployment_spec/schemas/music.sd b/config-model/src/test/cfg/application/app_complicated_deployment_spec/schemas/music.sd index 92e87848a8a..4e220f96727 100644 --- a/config-model/src/test/cfg/application/app_complicated_deployment_spec/schemas/music.sd +++ b/config-model/src/test/cfg/application/app_complicated_deployment_spec/schemas/music.sd @@ -7,13 +7,11 @@ search music { field title type string { indexing: summary | index # How this field should be indexed - # index-to: title, default # Create two indexes rank-type: about # Type of ranking settings to apply } field artist type string { indexing: summary | attribute | index - # index-to: artist, default rank-type:about } diff --git a/config-model/src/test/cfg/application/app_genericservices/schemas/music.sd b/config-model/src/test/cfg/application/app_genericservices/schemas/music.sd index 92e87848a8a..4e220f96727 100644 --- a/config-model/src/test/cfg/application/app_genericservices/schemas/music.sd +++ b/config-model/src/test/cfg/application/app_genericservices/schemas/music.sd @@ -7,13 +7,11 @@ search music { field title type string { indexing: summary | index # How this field should be indexed - # index-to: title, default # Create two indexes rank-type: about # Type of ranking settings to apply } field artist type string { indexing: summary | attribute | index - # index-to: artist, default rank-type:about } diff --git a/config-model/src/test/cfg/application/sdfilenametest/schemas/notmusic.sd b/config-model/src/test/cfg/application/sdfilenametest/schemas/notmusic.sd index 19528975587..a4cf5cef1a1 100644 --- a/config-model/src/test/cfg/application/sdfilenametest/schemas/notmusic.sd +++ b/config-model/src/test/cfg/application/sdfilenametest/schemas/notmusic.sd @@ -5,7 +5,6 @@ search music { field title type string { indexing: summary | index - # index-to: title, default } } diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/book.sd b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/book.sd index 73b540627d7..ba298f4fcba 100644 --- a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/book.sd +++ b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/book.sd @@ -2,34 +2,28 @@ search book { document book inherits base { field title type string { bolding: on - index-to: default, title indexing: index|summary rank-type: about } field dispauthor type string { bolding: on - index-to: default, dispauthor indexing: index|summary rank-type: about } field author type string { bolding: on - index-to: default, author indexing: index|summary rank-type: about } field keys type string { - index-to: default, keys indexing: index rank-type: about } field isbn type string { - index-to: default, isbn indexing: index|summary rank-type: about } field series type string { - index-to: default, series indexing: index rank-type: about } diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/music.sd b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/music.sd index 498bc79489f..21da176564b 100644 --- a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/music.sd @@ -2,11 +2,9 @@ search music { document music inherits base { field f1 type string { indexing: summary | index - index-to: f1, all } field f2 type string { indexing: summary | index - index-to: f2, all } } } diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/video.sd b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/video.sd index b010b6d9769..5462be17374 100644 --- a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/video.sd +++ b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/video.sd @@ -2,41 +2,34 @@ search video { document video inherits base { field title type string { bolding: on - index-to: default, title indexing: index|summary rank-type: about } field keys type string { - index-to: default, keys indexing: index rank-type: about } field director type string { bolding: on - index-to: default, director indexing: index|summary rank-type: about } field disp_actor type string { bolding: on - index-to: default, disp_actor indexing: index|summary rank-type: about } field actor type string { bolding: on - index-to: default, actor indexing: index|summary rank-type: about } field fmt type string { - index-to: default, fmt indexing: index|summary rank-type: about } field isbn type string { bolding: on - index-to: default, isbn indexing: index|summary rank-type: about } diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/book.sd b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/book.sd index 73b540627d7..ba298f4fcba 100644 --- a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/book.sd +++ b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/book.sd @@ -2,34 +2,28 @@ search book { document book inherits base { field title type string { bolding: on - index-to: default, title indexing: index|summary rank-type: about } field dispauthor type string { bolding: on - index-to: default, dispauthor indexing: index|summary rank-type: about } field author type string { bolding: on - index-to: default, author indexing: index|summary rank-type: about } field keys type string { - index-to: default, keys indexing: index rank-type: about } field isbn type string { - index-to: default, isbn indexing: index|summary rank-type: about } field series type string { - index-to: default, series indexing: index rank-type: about } diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/music.sd b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/music.sd index 498bc79489f..21da176564b 100644 --- a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/music.sd +++ b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/music.sd @@ -2,11 +2,9 @@ search music { document music inherits base { field f1 type string { indexing: summary | index - index-to: f1, all } field f2 type string { indexing: summary | index - index-to: f2, all } } } diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/video.sd b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/video.sd index b010b6d9769..5462be17374 100644 --- a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/video.sd +++ b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/video.sd @@ -2,41 +2,34 @@ search video { document video inherits base { field title type string { bolding: on - index-to: default, title indexing: index|summary rank-type: about } field keys type string { - index-to: default, keys indexing: index rank-type: about } field director type string { bolding: on - index-to: default, director indexing: index|summary rank-type: about } field disp_actor type string { bolding: on - index-to: default, disp_actor indexing: index|summary rank-type: about } field actor type string { bolding: on - index-to: default, actor indexing: index|summary rank-type: about } field fmt type string { - index-to: default, fmt indexing: index|summary rank-type: about } field isbn type string { bolding: on - index-to: default, isbn indexing: index|summary rank-type: about } diff --git a/config-model/src/test/cfg/routing/content_two_clusters/schemas/mobile.sd b/config-model/src/test/cfg/routing/content_two_clusters/schemas/mobile.sd index 60ea98235b0..3cc3dcf5526 100644 --- a/config-model/src/test/cfg/routing/content_two_clusters/schemas/mobile.sd +++ b/config-model/src/test/cfg/routing/content_two_clusters/schemas/mobile.sd @@ -3,11 +3,9 @@ search mobile { document mobile { field f1 type string { indexing: summary | index - # index-to: f1, all } field f2 type string { indexing: summary | index - # index-to: f2, all } } } diff --git a/config-model/src/test/cfg/routing/content_two_clusters/schemas/music.sd b/config-model/src/test/cfg/routing/content_two_clusters/schemas/music.sd index 290f8983b4d..982607955a7 100644 --- a/config-model/src/test/cfg/routing/content_two_clusters/schemas/music.sd +++ b/config-model/src/test/cfg/routing/content_two_clusters/schemas/music.sd @@ -3,11 +3,9 @@ search music { document music { field f1 type string { indexing: summary | index - # index-to: f1, all } field f2 type string { indexing: summary | index - # index-to: f2, all } } } diff --git a/config-model/src/test/cfg/routing/contentsimpleconfig/schemas/music.sd b/config-model/src/test/cfg/routing/contentsimpleconfig/schemas/music.sd index 290f8983b4d..982607955a7 100644 --- a/config-model/src/test/cfg/routing/contentsimpleconfig/schemas/music.sd +++ b/config-model/src/test/cfg/routing/contentsimpleconfig/schemas/music.sd @@ -3,11 +3,9 @@ search music { document music { field f1 type string { indexing: summary | index - # index-to: f1, all } field f2 type string { indexing: summary | index - # index-to: f2, all } } } diff --git a/config-model/src/test/cfg/routing/replacehop/schemas/music.sd b/config-model/src/test/cfg/routing/replacehop/schemas/music.sd index 274c6ca63d6..c4dcecbd6ac 100755 --- a/config-model/src/test/cfg/routing/replacehop/schemas/music.sd +++ b/config-model/src/test/cfg/routing/replacehop/schemas/music.sd @@ -3,11 +3,9 @@ search music { document music { field f1 type string { indexing: summary | index - # index-to: f1, all } field f2 type string { indexing: summary | index - # index-to: f2, all } } } diff --git a/config-model/src/test/cfg/routing/replaceroute/schemas/music.sd b/config-model/src/test/cfg/routing/replaceroute/schemas/music.sd index 274c6ca63d6..c4dcecbd6ac 100755 --- a/config-model/src/test/cfg/routing/replaceroute/schemas/music.sd +++ b/config-model/src/test/cfg/routing/replaceroute/schemas/music.sd @@ -3,11 +3,9 @@ search music { document music { field f1 type string { indexing: summary | index - # index-to: f1, all } field f2 type string { indexing: summary | index - # index-to: f2, all } } } diff --git a/config-model/src/test/cfg/search/data/travel/schemas/TTPOI.sd b/config-model/src/test/cfg/search/data/travel/schemas/TTPOI.sd index f3fe2cdf445..7895d98b2e0 100644 --- a/config-model/src/test/cfg/search/data/travel/schemas/TTPOI.sd +++ b/config-model/src/test/cfg/search/data/travel/schemas/TTPOI.sd @@ -4,13 +4,11 @@ document TTPOI { # categories associated with the POI field Categories type array<string> { indexing: summary | index - # index-to: Categories } # sub catagories associated with the POI field SubCategories type array<string> { indexing: summary | index - # index-to: SubCategories } } diff --git a/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/schemas/music.sd b/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/schemas/music.sd index 290f8983b4d..982607955a7 100644 --- a/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/schemas/music.sd +++ b/config-model/src/test/cfg/search/data/v2/inherited_rankprofiles/schemas/music.sd @@ -3,11 +3,9 @@ search music { document music { field f1 type string { indexing: summary | index - # index-to: f1, all } field f2 type string { indexing: summary | index - # index-to: f2, all } } } diff --git a/config-model/src/test/cfg/storage/app_index_higher_than_num_nodes/schemas/music.sd b/config-model/src/test/cfg/storage/app_index_higher_than_num_nodes/schemas/music.sd index 290f8983b4d..982607955a7 100644 --- a/config-model/src/test/cfg/storage/app_index_higher_than_num_nodes/schemas/music.sd +++ b/config-model/src/test/cfg/storage/app_index_higher_than_num_nodes/schemas/music.sd @@ -3,11 +3,9 @@ search music { document music { field f1 type string { indexing: summary | index - # index-to: f1, all } field f2 type string { indexing: summary | index - # index-to: f2, all } } } diff --git a/config-model/src/test/cfg/storage/clustercontroller_advanced/schemas/music.sd b/config-model/src/test/cfg/storage/clustercontroller_advanced/schemas/music.sd index 290f8983b4d..982607955a7 100644 --- a/config-model/src/test/cfg/storage/clustercontroller_advanced/schemas/music.sd +++ b/config-model/src/test/cfg/storage/clustercontroller_advanced/schemas/music.sd @@ -3,11 +3,9 @@ search music { document music { field f1 type string { indexing: summary | index - # index-to: f1, all } field f2 type string { indexing: summary | index - # index-to: f2, all } } } diff --git a/config-model/src/test/derived/music3/music3.sd b/config-model/src/test/derived/music3/music3.sd index 7123c45bac2..47867683c62 100644 --- a/config-model/src/test/derived/music3/music3.sd +++ b/config-model/src/test/derived/music3/music3.sd @@ -5,13 +5,11 @@ schema music3 { field title type string { indexing: summary | index - # index-to: title, default rank-type: about } field artist type string { indexing: summary | attribute | index - # index-to: artist, default rank-type:about } diff --git a/config-model/src/test/derived/newrank/newrank.sd b/config-model/src/test/derived/newrank/newrank.sd index 345d01bffb5..a01f292eb27 100644 --- a/config-model/src/test/derived/newrank/newrank.sd +++ b/config-model/src/test/derived/newrank/newrank.sd @@ -99,7 +99,6 @@ schema newrank{ field artist type string { indexing: summary | index - # index-to: artist, default } field artistspid type string { diff --git a/config-model/src/test/examples/attributesexactmatch.sd b/config-model/src/test/examples/attributesexactmatch.sd index 2db687cb20d..5529906adce 100644 --- a/config-model/src/test/examples/attributesexactmatch.sd +++ b/config-model/src/test/examples/attributesexactmatch.sd @@ -29,7 +29,6 @@ search music { } field genre type string { - # index-to: foo } field trumpetist type string { diff --git a/config-model/src/test/examples/casing.sd b/config-model/src/test/examples/casing.sd index b0ce0a07748..7564934949b 100644 --- a/config-model/src/test/examples/casing.sd +++ b/config-model/src/test/examples/casing.sd @@ -31,7 +31,6 @@ search music { field Genre type string { indexing: index - # index-to: Foo alias Foo: sjanger } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java index 9e8388b6442..7de3d41817a 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterMembership.java @@ -22,16 +22,28 @@ public class ClusterMembership { private ClusterMembership(String stringValue, Version vespaVersion, Optional<DockerImage> dockerImageRepo, ZoneEndpoint zoneEndpoint) { String[] components = stringValue.split("/"); - if (components.length < 4) + if (components.length < 3) throw new RuntimeException("Could not parse '" + stringValue + "' to a cluster membership. " + "Expected 'clusterType/clusterId/groupId/index[/retired][/exclusive][/stateful][/combinedId]'"); + Integer groupIndex = components[2].isEmpty() ? null : Integer.parseInt(components[2]); + Integer nodeIndex; + int missingElements = 0; + try { + nodeIndex = Integer.parseInt(components[3]); + } catch (NumberFormatException | ArrayIndexOutOfBoundsException e) { + // Legacy form missing the group component + nodeIndex = groupIndex; + groupIndex = null; + missingElements = 1; + } + boolean exclusive = false; boolean stateful = false; var combinedId = Optional.<String>empty(); boolean retired = false; - if (components.length > 4) { - for (int i = 4; i < components.length; i++) { + if (components.length > (4 - missingElements)) { + for (int i = (4 - missingElements); i < components.length; i++) { String component = components[i]; switch (component) { case "exclusive" -> exclusive = true; @@ -44,7 +56,7 @@ public class ClusterMembership { this.cluster = ClusterSpec.specification(ClusterSpec.Type.valueOf(components[0]), ClusterSpec.Id.from(components[1])) - .group(ClusterSpec.Group.from(Integer.parseInt(components[2]))) + .group(groupIndex == null ? null : ClusterSpec.Group.from(groupIndex)) .vespaVersion(vespaVersion) .exclusive(exclusive) .combinedId(combinedId.map(ClusterSpec.Id::from)) @@ -52,7 +64,7 @@ public class ClusterMembership { .loadBalancerSettings(zoneEndpoint) .stateful(stateful) .build(); - this.index = Integer.parseInt(components[3]); + this.index = nodeIndex; this.retired = retired; this.stringValue = toStringValue(); } @@ -67,7 +79,7 @@ public class ClusterMembership { protected String toStringValue() { return cluster.type().name() + "/" + cluster.id().value() + - (cluster.group().isPresent() ? "/" + cluster.group().get().index() : "") + + (cluster.group().isPresent() ? "/" + cluster.group().get().index() : "/") + "/" + index + ( cluster.isExclusive() ? "/exclusive" : "") + ( retired ? "/retired" : "") + diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java index ccc24e60edf..4a3045c9cdd 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java @@ -102,19 +102,18 @@ public final class ClusterSpec { /** Creates a ClusterSpec when requesting a cluster */ public static Builder request(Type type, Id id) { - return new Builder(type, id, false); + return new Builder(type, id); } /** Creates a ClusterSpec for an existing cluster, group id and Vespa version needs to be set */ public static Builder specification(Type type, Id id) { - return new Builder(type, id, true); + return new Builder(type, id); } public static class Builder { private final Type type; private final Id id; - private final boolean specification; private Optional<Group> groupId = Optional.empty(); private Optional<DockerImage> dockerImageRepo = Optional.empty(); @@ -124,19 +123,13 @@ public final class ClusterSpec { private ZoneEndpoint zoneEndpoint = ZoneEndpoint.defaultEndpoint; private boolean stateful; - private Builder(Type type, Id id, boolean specification) { + private Builder(Type type, Id id) { this.type = type; this.id = id; - this.specification = specification; this.stateful = type.isContent(); // Default to true for content clusters } public ClusterSpec build() { - if (specification) { - if (groupId.isEmpty()) throw new IllegalArgumentException("groupId is required to be set when creating a ClusterSpec with specification()"); - if (vespaVersion == null) throw new IllegalArgumentException("vespaVersion is required to be set when creating a ClusterSpec with specification()"); - } else - if (groupId.isPresent()) throw new IllegalArgumentException("groupId is not allowed to be set when creating a ClusterSpec with request()"); return new ClusterSpec(type, id, groupId, vespaVersion, exclusive, combinedId, dockerImageRepo, zoneEndpoint, stateful); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeAllocationException.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeAllocationException.java index 507d95c1d7b..64d028db7b0 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeAllocationException.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeAllocationException.java @@ -16,6 +16,11 @@ public class NodeAllocationException extends RuntimeException { this.retryable = retryable; } + public NodeAllocationException(String message, Throwable cause, boolean retryable) { + super(message, cause); + this.retryable = retryable; + } + public boolean retryable() { return retryable; } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java index b1195b6a54b..292aec60e39 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/ClusterMembershipTest.java @@ -100,7 +100,9 @@ public class ClusterMembershipTest { assertEquals("id1", instance.cluster().id().value()); assertFalse(instance.cluster().group().isPresent()); assertEquals(3, instance.index()); - assertEquals("container/id1/3", instance.stringValue()); + assertEquals("container/id1//3", instance.stringValue()); + // Legacy form: + assertEquals(instance, ClusterMembership.from("container/id1/3", instance.cluster().vespaVersion(), Optional.empty())); } private void assertContentService(ClusterMembership instance) { @@ -109,7 +111,7 @@ public class ClusterMembershipTest { assertFalse(instance.cluster().group().isPresent()); assertEquals(37, instance.index()); assertFalse(instance.retired()); - assertEquals("content/id1/37/stateful", instance.stringValue()); + assertEquals("content/id1//37/stateful", instance.stringValue()); } private void assertContentServiceWithGroup(ClusterMembership instance) { @@ -127,7 +129,9 @@ public class ClusterMembershipTest { assertEquals("id1", instance.cluster().id().value()); assertEquals(37, instance.index()); assertTrue(instance.retired()); - assertEquals("content/id1/37/retired/stateful", instance.stringValue()); + assertEquals("content/id1//37/retired/stateful", instance.stringValue()); + // Legacy form: + assertEquals(instance, ClusterMembership.from("content/id1/37/retired/stateful", instance.cluster().vespaVersion(), Optional.empty())); } private void assertContentServiceWithGroupAndRetire(ClusterMembership instance) { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java index 41bab257248..f8db7aadc29 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSetSubscription.java @@ -3,7 +3,6 @@ package com.yahoo.config.subscription.impl; import com.yahoo.config.ConfigInstance; import com.yahoo.config.subscription.ConfigSet; -import com.yahoo.config.subscription.ConfigSource; import com.yahoo.vespa.config.ConfigKey; import java.lang.reflect.Constructor; @@ -48,11 +47,11 @@ public class ConfigSetSubscription<T extends ConfigInstance> extends ConfigSubsc if (hasConfigChanged()) return true; if (timeout <= 0) return false; - long end = System.nanoTime() + timeout * 1_000_000; + long startNanos = System.nanoTime(); do { sleep(); if (hasConfigChanged()) return true; - } while (System.nanoTime() < end); + } while (System.nanoTime() - startNanos < timeout * 1_000_000); return false; } diff --git a/configserver/src/test/apps/app-jdisc-only-restart/schemas/music.sd b/configserver/src/test/apps/app-jdisc-only-restart/schemas/music.sd index a45c62ccdc7..71cfc346117 100644 --- a/configserver/src/test/apps/app-jdisc-only-restart/schemas/music.sd +++ b/configserver/src/test/apps/app-jdisc-only-restart/schemas/music.sd @@ -7,13 +7,11 @@ search music { field title type string { indexing: summary | index # How this field should be indexed - # index-to: title, default # Create two indexes weight: 75 # Ranking importancy of this field, used by the built in nativeRank feature } field artist type string { indexing: summary | attribute | index - # index-to: artist, default weight: 25 } diff --git a/configserver/src/test/apps/app-jdisc-only/schemas/music.sd b/configserver/src/test/apps/app-jdisc-only/schemas/music.sd index a45c62ccdc7..71cfc346117 100644 --- a/configserver/src/test/apps/app-jdisc-only/schemas/music.sd +++ b/configserver/src/test/apps/app-jdisc-only/schemas/music.sd @@ -7,13 +7,11 @@ search music { field title type string { indexing: summary | index # How this field should be indexed - # index-to: title, default # Create two indexes weight: 75 # Ranking importancy of this field, used by the built in nativeRank feature } field artist type string { indexing: summary | attribute | index - # index-to: artist, default weight: 25 } diff --git a/configserver/src/test/apps/app-major-version-7/schemas/music.sd b/configserver/src/test/apps/app-major-version-7/schemas/music.sd index f4b11d1e8e4..85db4873ba1 100644 --- a/configserver/src/test/apps/app-major-version-7/schemas/music.sd +++ b/configserver/src/test/apps/app-major-version-7/schemas/music.sd @@ -7,13 +7,11 @@ search music { field title type string { indexing: summary | index # How this field should be indexed - # index-to: title, default # Create two indexes weight: 75 # Ranking importancy of this field, used by the built in nativeRank feature } field artist type string { indexing: summary | attribute | index - # index-to: artist, default weight: 25 } diff --git a/configserver/src/test/apps/app/schemas/music.sd b/configserver/src/test/apps/app/schemas/music.sd index f4b11d1e8e4..85db4873ba1 100644 --- a/configserver/src/test/apps/app/schemas/music.sd +++ b/configserver/src/test/apps/app/schemas/music.sd @@ -7,13 +7,11 @@ search music { field title type string { indexing: summary | index # How this field should be indexed - # index-to: title, default # Create two indexes weight: 75 # Ranking importancy of this field, used by the built in nativeRank feature } field artist type string { indexing: summary | attribute | index - # index-to: artist, default weight: 25 } diff --git a/configserver/src/test/apps/content/schemas/music.sd b/configserver/src/test/apps/content/schemas/music.sd index f4b11d1e8e4..85db4873ba1 100644 --- a/configserver/src/test/apps/content/schemas/music.sd +++ b/configserver/src/test/apps/content/schemas/music.sd @@ -7,13 +7,11 @@ search music { field title type string { indexing: summary | index # How this field should be indexed - # index-to: title, default # Create two indexes weight: 75 # Ranking importancy of this field, used by the built in nativeRank feature } field artist type string { indexing: summary | attribute | index - # index-to: artist, default weight: 25 } diff --git a/configserver/src/test/apps/content2/schemas/music.sd b/configserver/src/test/apps/content2/schemas/music.sd index f4b11d1e8e4..85db4873ba1 100644 --- a/configserver/src/test/apps/content2/schemas/music.sd +++ b/configserver/src/test/apps/content2/schemas/music.sd @@ -7,13 +7,11 @@ search music { field title type string { indexing: summary | index # How this field should be indexed - # index-to: title, default # Create two indexes weight: 75 # Ranking importancy of this field, used by the built in nativeRank feature } field artist type string { indexing: summary | attribute | index - # index-to: artist, default weight: 25 } diff --git a/configserver/src/test/apps/deprecated-features-app/searchdefinitions/music.sd b/configserver/src/test/apps/deprecated-features-app/searchdefinitions/music.sd index a45c62ccdc7..71cfc346117 100644 --- a/configserver/src/test/apps/deprecated-features-app/searchdefinitions/music.sd +++ b/configserver/src/test/apps/deprecated-features-app/searchdefinitions/music.sd @@ -7,13 +7,11 @@ search music { field title type string { indexing: summary | index # How this field should be indexed - # index-to: title, default # Create two indexes weight: 75 # Ranking importancy of this field, used by the built in nativeRank feature } field artist type string { indexing: summary | attribute | index - # index-to: artist, default weight: 25 } diff --git a/configserver/src/test/apps/hosted-no-write-access-control/schemas/music.sd b/configserver/src/test/apps/hosted-no-write-access-control/schemas/music.sd index cb4b860e019..49475dc7f77 100644 --- a/configserver/src/test/apps/hosted-no-write-access-control/schemas/music.sd +++ b/configserver/src/test/apps/hosted-no-write-access-control/schemas/music.sd @@ -3,7 +3,6 @@ search music { document music { field title type string { indexing: index | summary - # index-to: default } } } diff --git a/configserver/src/test/apps/legacy-flag/schemas/music.sd b/configserver/src/test/apps/legacy-flag/schemas/music.sd index f4b11d1e8e4..85db4873ba1 100644 --- a/configserver/src/test/apps/legacy-flag/schemas/music.sd +++ b/configserver/src/test/apps/legacy-flag/schemas/music.sd @@ -7,13 +7,11 @@ search music { field title type string { indexing: summary | index # How this field should be indexed - # index-to: title, default # Create two indexes weight: 75 # Ranking importancy of this field, used by the built in nativeRank feature } field artist type string { indexing: summary | attribute | index - # index-to: artist, default weight: 25 } diff --git a/configserver/src/test/apps/zkapp/schemas/music.sd b/configserver/src/test/apps/zkapp/schemas/music.sd index 7616e2370b4..cc7844e76d5 100644 --- a/configserver/src/test/apps/zkapp/schemas/music.sd +++ b/configserver/src/test/apps/zkapp/schemas/music.sd @@ -7,13 +7,11 @@ search music { field title type string { indexing: summary | index # How this field should be indexed - # index-to: title, default # Create two indexes rank-type: about # Type of ranking settings to apply } field artist type string { indexing: summary | attribute | index - # index-to: artist, default rank-type:about } diff --git a/configserver/src/test/apps/zkapp/schemas/product.sd b/configserver/src/test/apps/zkapp/schemas/product.sd index 132ae15053f..70c9343d63a 100644 --- a/configserver/src/test/apps/zkapp/schemas/product.sd +++ b/configserver/src/test/apps/zkapp/schemas/product.sd @@ -3,7 +3,6 @@ document product { field title type string { indexing: index | summary - # index-to: title, default } field price type int { diff --git a/configserver/src/test/apps/zkfeed/schemas/product.sd b/configserver/src/test/apps/zkfeed/schemas/product.sd index 132ae15053f..70c9343d63a 100644 --- a/configserver/src/test/apps/zkfeed/schemas/product.sd +++ b/configserver/src/test/apps/zkfeed/schemas/product.sd @@ -3,7 +3,6 @@ document product { field title type string { indexing: index | summary - # index-to: title, default } field price type int { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java index d6fb6de6354..b488662591a 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InvokerFactory.java @@ -61,16 +61,10 @@ public abstract class InvokerFactory { List<SearchInvoker> invokers = new ArrayList<>(nodes.size()); Set<Integer> failed = null; for (Node node : nodes) { - boolean nodeAdded = false; - if (node.isWorking() != Boolean.FALSE) { - Optional<SearchInvoker> invoker = createNodeSearchInvoker(searcher, query, maxHits, node); - if (invoker.isPresent()) { - invokers.add(invoker.get()); - nodeAdded = true; - } - } - - if ( ! nodeAdded) { + if ( node.isWorking() == Boolean.FALSE + || createNodeSearchInvoker(searcher, query, maxHits, node) + .map(invoker -> { invokers.add(invoker); return invoker; }) + .isEmpty()) { if (failed == null) { failed = new HashSet<>(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java index 0d8e7745f65..186e6838a71 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java @@ -85,10 +85,7 @@ public class ApplicationPackageValidator { private void validateDeprecatedElements(ApplicationPackage applicationPackage) { int wantedMajor = applicationPackage.compileVersion().map(Version::getMajor) .or(() -> applicationPackage.deploymentSpec().majorVersion()) - .or(() -> controller.readVersionStatus().controllerVersion() - .map(VespaVersion::versionNumber) - .map(Version::getMajor)) - .orElseThrow(() -> new IllegalArgumentException("Could not determine wanted major version")); + .orElseGet(() -> controller.readSystemVersion().getMajor()); for (var deprecatedElement : applicationPackage.deploymentSpec().deprecatedElements()) { if (deprecatedElement.majorVersion() >= wantedMajor) continue; throw new IllegalArgumentException(deprecatedElement.humanReadableString()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index 30c832a7747..66fb1fe615b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -332,26 +332,24 @@ public class RoutingPolicies { } } if (!aliasTargets.isEmpty()) { - nameServiceForwarderIn(targetZone).createAlias( + nameServiceForwarder(applicationEndpoint).createAlias( RecordName.from(applicationEndpoint.dnsName()), aliasTargets, Priority.normal, owner); } if (!directTargets.isEmpty()) { - nameServiceForwarderIn(targetZone).createDirect( + nameServiceForwarder(applicationEndpoint).createDirect( RecordName.from(applicationEndpoint.dnsName()), directTargets, Priority.normal, owner); } }); // Remove DNS records for inactive targets inactiveTargetsByEndpoint.forEach((applicationEndpoint, targets) -> { - // Where multiple zones are permitted, they all have the same routing policy, and nameServiceForwarder. - ZoneId targetZone = applicationEndpoint.targets().iterator().next().deployment().zoneId(); targets.forEach(target -> { if (!target.deployment().equals(deployment)) return; // Do not update target not matching this deployment - nameServiceForwarderIn(targetZone).removeRecords(target.type(), - RecordName.from(applicationEndpoint.dnsName()), - target.data(), - Priority.normal, - owner); + nameServiceForwarder(applicationEndpoint).removeRecords(target.type(), + RecordName.from(applicationEndpoint.dnsName()), + target.data(), + Priority.normal, + owner); }); }); } @@ -394,13 +392,14 @@ public class RoutingPolicies { /** Update zone DNS record for given policy */ private void updateZoneDnsOf(RoutingPolicy policy, LoadBalancer loadBalancer, DeploymentId deploymentId) { + RoutingMethod routingMethod = controller.zoneRegistry().routingMethod(deploymentId.zoneId()); boolean addTokenEndpoint = controller.routing().tokenEndpointEnabled(deploymentId.applicationId()); - for (var endpoint : policy.zoneEndpointsIn(controller.system(), RoutingMethod.exclusive, addTokenEndpoint)) { + for (var endpoint : policy.zoneEndpointsIn(controller.system(), routingMethod, addTokenEndpoint)) { var name = RecordName.from(endpoint.dnsName()); var record = policy.canonicalName().isPresent() ? new Record(Record.Type.CNAME, name, RecordData.fqdn(policy.canonicalName().get().value())) : new Record(Record.Type.A, name, RecordData.from(policy.ipAddress().orElseThrow())); - nameServiceForwarderIn(policy.id().zone()).createRecord(record, Priority.normal, ownerOf(deploymentId)); + nameServiceForwarder(endpoint).createRecord(record, Priority.normal, ownerOf(deploymentId)); setPrivateDns(endpoint, loadBalancer, deploymentId); } } @@ -413,13 +412,14 @@ public class RoutingPolicies { case mtls -> false; }; if (skipBasedOnAuthMethod) return; + if (endpoint.routingMethod() != RoutingMethod.exclusive) return; // Not supported for this routing method controller.serviceRegistry().vpcEndpointService() .setPrivateDns(DomainName.of(endpoint.dnsName()), new ClusterId(deploymentId, endpoint.cluster()), loadBalancer.cloudAccount()) .ifPresent(challenge -> { try (Mutex lock = db.lockNameServiceQueue()) { - nameServiceForwarderIn(deploymentId.zoneId()).createTxt(challenge.name(), List.of(challenge.data()), Priority.high, ownerOf(deploymentId)); + controller.nameServiceForwarder().createTxt(challenge.name(), List.of(challenge.data()), Priority.high, ownerOf(deploymentId)); db.writeDnsChallenge(challenge); } }); @@ -458,8 +458,7 @@ public class RoutingPolicies { } private void removeDnsChallenge(DnsChallenge challenge) { - nameServiceForwarderIn(challenge.clusterId().deploymentId().zoneId()) - .removeRecords(Type.TXT, challenge.name(), Priority.normal, ownerOf(challenge.clusterId().deploymentId())); + controller.nameServiceForwarder().removeRecords(Type.TXT, challenge.name(), Priority.normal, ownerOf(challenge.clusterId().deploymentId())); db.deleteDnsChallenge(challenge.clusterId()); } @@ -469,17 +468,18 @@ public class RoutingPolicies { * @return the updated policies */ private RoutingPolicyList removePoliciesUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList instancePolicies, @SuppressWarnings("unused") Mutex lock) { + RoutingMethod routingMethod = controller.zoneRegistry().routingMethod(allocation.deployment.zoneId()); boolean addTokenEndpoint = controller.routing().tokenEndpointEnabled(allocation.deployment.applicationId()); Map<RoutingPolicyId, RoutingPolicy> newPolicies = new LinkedHashMap<>(instancePolicies.asMap()); Set<RoutingPolicyId> activeIds = allocation.asPolicyIds(); RoutingPolicyList removable = instancePolicies.deployment(allocation.deployment) .not().matching(policy -> activeIds.contains(policy.id())); for (var policy : removable) { - for (var endpoint : policy.zoneEndpointsIn(controller.system(), RoutingMethod.exclusive, addTokenEndpoint)) { - nameServiceForwarderIn(allocation.deployment.zoneId()).removeRecords(Record.Type.CNAME, - RecordName.from(endpoint.dnsName()), - Priority.normal, - ownerOf(allocation)); + for (var endpoint : policy.zoneEndpointsIn(controller.system(), routingMethod, addTokenEndpoint)) { + nameServiceForwarder(endpoint).removeRecords(Record.Type.CNAME, + RecordName.from(endpoint.dnsName()), + Priority.normal, + ownerOf(allocation)); } newPolicies.remove(policy.id()); } @@ -497,12 +497,11 @@ public class RoutingPolicies { EndpointList endpoints = controller.routing().readDeclaredEndpointsOf(id.instance()) .not().requiresRotation() .named(id.endpointId(), Endpoint.Scope.global); - NameServiceForwarder forwarder = nameServiceForwarderIn(allocation.deployment.zoneId()); // This removes all ALIAS records having this DNS name. There is no attempt to delete only the entry for the // affected zone. Instead, the correct set of records is (re)created by updateGlobalDnsOf - endpoints.forEach(endpoint -> forwarder.removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), - Priority.normal, - ownerOf(allocation))); + endpoints.forEach(endpoint -> nameServiceForwarder(endpoint).removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), + Priority.normal, + ownerOf(allocation))); } } @@ -520,8 +519,8 @@ public class RoutingPolicies { List<RoutingPolicy> policies = routingTable.get(id); for (var policy : policies) { if (!policy.appliesTo(allocation.deployment)) continue; - NameServiceForwarder forwarder = nameServiceForwarderIn(policy.id().zone()); for (Endpoint endpoint : endpoints) { + NameServiceForwarder forwarder = nameServiceForwarder(endpoint); if (policy.canonicalName().isPresent()) { forwarder.removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), @@ -690,11 +689,11 @@ public class RoutingPolicies { .collect(Collectors.toUnmodifiableSet()); } - /** Returns the name updater to use for given zone */ - private NameServiceForwarder nameServiceForwarderIn(ZoneId zone) { - return switch (controller.zoneRegistry().routingMethod(zone)) { + /** Returns the name updater to use for given endpoint */ + private NameServiceForwarder nameServiceForwarder(Endpoint endpoint) { + return switch (endpoint.routingMethod()) { case exclusive -> controller.nameServiceForwarder(); - case sharedLayer4 -> new NameServiceDiscarder(controller.curator()); + case sharedLayer4 -> endpoint.generated().isPresent() ? controller.nameServiceForwarder() : new NameServiceDiscarder(controller.curator()); }; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index f6ed8fd7323..0a6d2a3b106 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -370,14 +370,22 @@ public class RoutingPoliciesTest { } @Test - void zone_routing_policies_without_dns_update() { + void zone_routing_policies_with_shared_routing() { var tester = new RoutingPoliciesTester(new DeploymentTester(), false); var context = tester.newDeploymentContext("tenant1", "app1", "default"); tester.provisionLoadBalancers(1, context.instanceId(), true, zone1, zone2); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); assertEquals(0, tester.controllerTester().controller().curator().readNameServiceQueue().requests().size()); + // Ordinary endpoints are not created in DNS assertEquals(List.of(), tester.recordNames()); assertEquals(2, tester.policiesOf(context.instanceId()).size()); + // Generated endpoints are created in DNS + tester.controllerTester().flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); + addCertificateToPool("cafed00d", UnassignedCertificate.State.ready, tester); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + assertEquals(List.of("b22ab332.cafed00d.z.vespa.oath.cloud", + "d71005bf.cafed00d.z.vespa.oath.cloud"), + tester.recordNames()); } @Test diff --git a/documentgen-test/etc/complex/music.sd b/documentgen-test/etc/complex/music.sd index e91adeed039..11abbaa154e 100644 --- a/documentgen-test/etc/complex/music.sd +++ b/documentgen-test/etc/complex/music.sd @@ -3,19 +3,16 @@ search music { document music inherits common { field artist type string { bolding: on - # index-to: default, artist indexing: index|summary } field disp_song type string { indexing: summary } field song type string { - # index-to: default, song indexing: index } field isbn type string { bolding: on - # index-to: default, isbn indexing: index|summary } field year type int { diff --git a/documentgen-test/etc/complex/music2.sd b/documentgen-test/etc/complex/music2.sd index 34419eeea6a..64e91caf25b 100644 --- a/documentgen-test/etc/complex/music2.sd +++ b/documentgen-test/etc/complex/music2.sd @@ -3,19 +3,16 @@ search music2 { document music2 inherits common { field artist type string { bolding: on - # index-to: default, artist indexing: index|summary } field disp_song type string { indexing: summary } field song type string { - # index-to: default, song indexing: index } field isbn type string { bolding: on - # index-to: default, isbn indexing: index|summary } field year type int { diff --git a/documentgen-test/etc/complex/video.sd b/documentgen-test/etc/complex/video.sd index 5ca283eaa96..95cb4228ad8 100644 --- a/documentgen-test/etc/complex/video.sd +++ b/documentgen-test/etc/complex/video.sd @@ -3,26 +3,21 @@ search video { document video inherits common { field director type string { bolding: on - # index-to: default, director indexing: index|summary } field disp_actor type string { bolding: on - # index-to: default, disp_actor indexing: index|summary } field actor type string { bolding: on - # index-to: default, actor indexing: index|summary } field fmt type string { - # index-to: default, fmt indexing: index|summary } field isbn type string { bolding: on - # index-to: default, isbn indexing: index|summary } field year type int { diff --git a/eval/src/tests/eval/value_cache/dense.json b/eval/src/tests/eval/value_cache/dense.json index 2263053f01f..f310ee9dc32 100644 --- a/eval/src/tests/eval/value_cache/dense.json +++ b/eval/src/tests/eval/value_cache/dense.json @@ -1,8 +1,8 @@ { "dimensions": ["x","y"], "cells": [ - { "address": { "x": "0", "y": "0" }, "value": 1.0 }, - { "address": { "x": "0", "y": "1" }, "value": 2.0 }, - { "address": { "x": "1", "y": "0" }, "value": 3.0 }, + { "address": { "x": 0, "y": 0 }, "value": 1.0 }, + { "address": { "y": 1, "x": 0 }, "value": 2.0 }, + { "address": { "x": "1", "y": 0 }, "value": 3.0 }, { "address": { "x": "1", "y": "1" }, "value": 4.0 }] } diff --git a/eval/src/tests/eval/value_cache/sparse-short1.json b/eval/src/tests/eval/value_cache/sparse-short1.json index 741a2160898..5b6aa6d6104 100644 --- a/eval/src/tests/eval/value_cache/sparse-short1.json +++ b/eval/src/tests/eval/value_cache/sparse-short1.json @@ -1,5 +1,5 @@ { - "foo": 1.0, + "foo": 1, "cells": 2.0, "values": 0.5, "blocks": 1.5 diff --git a/eval/src/tests/eval/value_cache/sparse-short2.json b/eval/src/tests/eval/value_cache/sparse-short2.json index 7eb377968e4..552fec39bcc 100644 --- a/eval/src/tests/eval/value_cache/sparse-short2.json +++ b/eval/src/tests/eval/value_cache/sparse-short2.json @@ -1,6 +1,6 @@ { "cells": { - "foo": 1.0, + "foo": 1, "cells": 2.0, "values": 0.5, "blocks": 1.5 diff --git a/eval/src/tests/eval/value_cache/sparse.json b/eval/src/tests/eval/value_cache/sparse.json index a80e7906286..f52ad888c61 100644 --- a/eval/src/tests/eval/value_cache/sparse.json +++ b/eval/src/tests/eval/value_cache/sparse.json @@ -2,5 +2,6 @@ "dimensions": ["x","y"], "cells": [ { "address": { "x": "foo", "y": "bar" }, "value": 1.0 }, + { "address": { "x": 17, "y": 42 }, "value": 1742.0 }, { "address": { "x": "bar", "y": "foo" }, "value": 2.0 }] } diff --git a/eval/src/tests/eval/value_cache/sparse.json.lz4 b/eval/src/tests/eval/value_cache/sparse.json.lz4 Binary files differindex 0de6fae56e1..4064222d403 100644 --- a/eval/src/tests/eval/value_cache/sparse.json.lz4 +++ b/eval/src/tests/eval/value_cache/sparse.json.lz4 diff --git a/eval/src/tests/eval/value_cache/tensor_loader_test.cpp b/eval/src/tests/eval/value_cache/tensor_loader_test.cpp index c10da861c83..ba2412d6f70 100644 --- a/eval/src/tests/eval/value_cache/tensor_loader_test.cpp +++ b/eval/src/tests/eval/value_cache/tensor_loader_test.cpp @@ -28,6 +28,7 @@ TensorSpec make_simple_dense_tensor() { TensorSpec make_sparse_tensor() { return TensorSpec("tensor(x{},y{})") + .add({{"x", "17"}, {"y", "42"}}, 1742.0) .add({{"x", "foo"}, {"y", "bar"}}, 1.0) .add({{"x", "bar"}, {"y", "foo"}}, 2.0); } @@ -74,6 +75,10 @@ TEST_F("require that dense tensors can be loaded", ConstantTensorLoader(factory) TEST_DO(verify_tensor(make_dense_tensor(), f1.create(TEST_PATH("dense.json"), "tensor(x[2],y[2])"))); } +TEST_F("require that sparse tensors can be loaded", ConstantTensorLoader(factory)) { + TEST_DO(verify_tensor(make_sparse_tensor(), f1.create(TEST_PATH("sparse.json"), "tensor(x{},y{})"))); +} + TEST_F("require that mixed tensors can be loaded", ConstantTensorLoader(factory)) { TEST_DO(verify_tensor(make_mixed_tensor(), f1.create(TEST_PATH("mixed.json"), "tensor(x{},y[2])"))); } diff --git a/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp b/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp index 059bf3c535d..74965b79bbc 100644 --- a/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp +++ b/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp @@ -20,6 +20,42 @@ using ObjectTraverser = slime::ObjectTraverser; namespace { +struct Target { + const ValueType tensor_type; + TensorSpec spec; + void check_add(TensorSpec::Address address, double value) { + for (const auto &dim : tensor_type.dimensions()) { + const auto & it = address.find(dim.name); + if (it == address.end()) { + LOG(error, "Missing dimension '%s' in address for constant tensor", dim.name.c_str()); + throw std::exception(); + } + if (it->second.is_mapped() != dim.is_mapped()) { + LOG(error, "Mismatch mapped/indexed for '%s' in address", dim.name.c_str()); + throw std::exception(); + } + if (dim.is_indexed()) { + if (it->second.index >= dim.size) { + LOG(error, "Index %zu out of range for dimension %s[%u]", + it->second.index, dim.name.c_str(), dim.size); + throw std::exception(); + } + } + } + if (address.size() != tensor_type.dimensions().size()) { + for (const auto & [name, label] : address) { + if (tensor_type.dimension_index(name) == ValueType::Dimension::npos) { + LOG(error, "Extra dimension '%s' in address for constant tensor", name.c_str()); + } + } + LOG(error, "Wrong number %zu of dimensions in address for constant tensor, wanted %zu", + address.size(), tensor_type.dimensions().size()); + throw std::exception(); + } + spec.add(address, value); + } +}; + struct AddressExtractor : ObjectTraverser { const std::set<vespalib::string> &indexed; TensorSpec::Address &address; @@ -28,14 +64,38 @@ struct AddressExtractor : ObjectTraverser { : indexed(indexed_in), address(address_out) {} void field(const Memory &symbol, const Inspector &inspector) override { vespalib::string dimension = symbol.make_string(); - vespalib::string label = inspector.asString().make_string(); - if (dimension.empty() || label.empty()) { + if (dimension.empty()) { + LOG(warning, "missing 'dimension' in address"); + throw std::exception(); + } + if (inspector.type().getId() == vespalib::slime::LONG::ID) { + size_t index = inspector.asLong(); + if (indexed.contains(dimension)) { + address.emplace(dimension, TensorSpec::Label(index)); + } else { + auto label = std::to_string(index); + address.emplace(dimension, TensorSpec::Label(label)); + } return; } + vespalib::string label = inspector.asString().make_string(); + if (label.empty()) { + auto got = inspector.toString(); + int sz = got.size(); + if (sz > 0) --sz; + LOG(error, "missing 'label' in address, got '%.*s'", sz, got.c_str()); + throw std::exception(); + } if (indexed.find(dimension) == indexed.end()) { address.emplace(dimension, TensorSpec::Label(label)); } else { - size_t index = strtoull(label.c_str(), nullptr, 10); + const char *str_beg = label.c_str(); + char *str_end = const_cast<char *>(str_beg); + size_t index = strtoull(str_beg, &str_end, 10); + if (str_end == str_beg || *str_end != '\0') { + LOG(error, "bad index: '%s' cannot be parsed as an unsigned integer", str_beg); + throw std::exception(); + } address.emplace(dimension, TensorSpec::Label(index)); } } @@ -43,46 +103,41 @@ struct AddressExtractor : ObjectTraverser { struct SingleMappedExtractor : ObjectTraverser { const vespalib::string &dimension; - TensorSpec &spec; - SingleMappedExtractor(const vespalib::string &dimension_in, TensorSpec &spec_in) + Target ⌖ + SingleMappedExtractor(const vespalib::string &dimension_in, Target &target_in) : dimension(dimension_in), - spec(spec_in) + target(target_in) {} void field(const Memory &symbol, const Inspector &inspector) override { vespalib::string label = symbol.make_string(); double value = inspector.asDouble(); TensorSpec::Address address; address.emplace(dimension, label); - spec.add(address, value); + target.check_add(address, value); } }; -void decodeSingleMappedForm(const Inspector &root, const ValueType &value_type, TensorSpec &spec) { - auto extractor = SingleMappedExtractor(value_type.dimensions()[0].name, spec); +void decodeSingleMappedForm(const Inspector &root, const ValueType &value_type, Target &target) { + auto extractor = SingleMappedExtractor(value_type.dimensions()[0].name, target); root.traverse(extractor); } -void decodeSingleDenseForm(const Inspector &values, const ValueType &value_type, TensorSpec &spec) { +void decodeSingleDenseForm(const Inspector &values, const ValueType &value_type, Target &target) { const auto &dimension = value_type.dimensions()[0].name; for (size_t i = 0; i < values.entries(); ++i) { TensorSpec::Address address; address.emplace(dimension, TensorSpec::Label(i)); - spec.add(address, values[i].asDouble()); + target.check_add(address, values[i].asDouble()); } } struct DenseValuesDecoder { const std::vector<ValueType::Dimension> _idims; - TensorSpec &_target; - DenseValuesDecoder(std::vector<ValueType::Dimension> idims, TensorSpec &target) - : _idims(std::move(idims)), - _target(target) - { - } + Target &_target; void decode(const Inspector &input, const TensorSpec::Address &address, size_t dim_idx) { if (dim_idx == _idims.size()) { - _target.add(address, input.asDouble()); + _target.check_add(address, input.asDouble()); } else { const auto &dimension = _idims[dim_idx]; if (input.entries() != dimension.size) { @@ -97,9 +152,9 @@ struct DenseValuesDecoder { } }; -void decodeDenseValues(const Inspector &values, const ValueType &value_type, TensorSpec &spec) { +void decodeDenseValues(const Inspector &values, const ValueType &value_type, Target &target) { TensorSpec::Address address; - DenseValuesDecoder decoder(value_type.indexed_dimensions(), spec); + DenseValuesDecoder decoder{value_type.indexed_dimensions(), target}; decoder.decode(values, address, 0); } @@ -113,12 +168,12 @@ struct TraverserCallback : ObjectTraverser { } }; -void decodeSingleMappedBlocks(const Inspector &blocks, const ValueType &value_type, TensorSpec &spec) { +void decodeSingleMappedBlocks(const Inspector &blocks, const ValueType &value_type, Target &target) { if (value_type.count_mapped_dimensions() != 1) { return; // TODO handle mismatch } vespalib::string dim_name = value_type.mapped_dimensions()[0].name; - DenseValuesDecoder decoder(value_type.indexed_dimensions(), spec); + DenseValuesDecoder decoder{value_type.indexed_dimensions(), target}; auto lambda = [&](vespalib::string label, const Inspector &input) { TensorSpec::Address address; address.emplace(dim_name, std::move(label)); @@ -128,13 +183,13 @@ void decodeSingleMappedBlocks(const Inspector &blocks, const ValueType &value_ty blocks.traverse(cb); } -void decodeAddressedBlocks(const Inspector &blocks, const ValueType &value_type, TensorSpec &spec) { +void decodeAddressedBlocks(const Inspector &blocks, const ValueType &value_type, Target &target) { const auto & idims = value_type.indexed_dimensions(); std::set<vespalib::string> indexed; for (const auto &dimension: idims) { indexed.insert(dimension.name); } - DenseValuesDecoder decoder(value_type.indexed_dimensions(), spec); + DenseValuesDecoder decoder{value_type.indexed_dimensions(), target}; for (size_t i = 0; i < blocks.entries(); ++i) { TensorSpec::Address address; AddressExtractor extractor(indexed, address); @@ -143,7 +198,7 @@ void decodeAddressedBlocks(const Inspector &blocks, const ValueType &value_type, } } -void decodeLiteralForm(const Inspector &cells, const ValueType &value_type, TensorSpec &spec) { +void decodeLiteralForm(const Inspector &cells, const ValueType &value_type, Target &target) { std::set<vespalib::string> indexed; for (const auto &dimension: value_type.dimensions()) { if (dimension.is_indexed()) { @@ -154,7 +209,7 @@ void decodeLiteralForm(const Inspector &cells, const ValueType &value_type, Tens TensorSpec::Address address; AddressExtractor extractor(indexed, address); cells[i]["address"].traverse(extractor); - spec.add(address, cells[i]["value"].asDouble()); + target.check_add(address, cells[i]["value"].asDouble()); } } @@ -207,7 +262,7 @@ ConstantTensorLoader::create(const vespalib::string &path, const vespalib::strin } Slime slime; decode_json(path, slime); - TensorSpec spec(type); + Target target{value_type, TensorSpec(type)}; bool isSingleDenseType = value_type.is_dense() && (value_type.count_indexed_dimensions() == 1); bool isSingleMappedType = value_type.is_sparse() && (value_type.count_mapped_dimensions() == 1); const Inspector &root = slime.get(); @@ -216,31 +271,31 @@ ConstantTensorLoader::create(const vespalib::string &path, const vespalib::strin const Inspector &values = root["values"]; const Inspector &blocks = root["blocks"]; if (cells.type().getId() == vespalib::slime::ARRAY::ID) { - decodeLiteralForm(cells, value_type, spec); + decodeLiteralForm(cells, value_type, target); } else if (cells.type().getId() == vespalib::slime::OBJECT::ID) { if (isSingleMappedType) { - decodeSingleMappedForm(cells, value_type, spec); + decodeSingleMappedForm(cells, value_type, target); } } else if (values.type().getId() == vespalib::slime::ARRAY::ID) { - decodeDenseValues(values, value_type, spec); + decodeDenseValues(values, value_type, target); } else if (blocks.type().getId() == vespalib::slime::OBJECT::ID) { - decodeSingleMappedBlocks(blocks, value_type, spec); + decodeSingleMappedBlocks(blocks, value_type, target); } else if (blocks.type().getId() == vespalib::slime::ARRAY::ID) { - decodeAddressedBlocks(blocks, value_type, spec); + decodeAddressedBlocks(blocks, value_type, target); } else if (isSingleMappedType) { - decodeSingleMappedForm(root, value_type, spec); + decodeSingleMappedForm(root, value_type, target); } } else if (root.type().getId() == vespalib::slime::ARRAY::ID && isSingleDenseType) { - decodeSingleDenseForm(root, value_type, spec); + decodeSingleDenseForm(root, value_type, target); } try { - return std::make_unique<SimpleConstantValue>(value_from_spec(spec, _factory)); + return std::make_unique<SimpleConstantValue>(value_from_spec(target.spec, _factory)); } catch (std::exception &) { return std::make_unique<BadConstantValue>(); } diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ExportPackagesIT.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ExportPackagesIT.java index a2aade05059..4c33bbb563f 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ExportPackagesIT.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ExportPackagesIT.java @@ -1,5 +1,6 @@ package com.yahoo.jdisc.core; +import com.google.common.collect.Sets; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -7,11 +8,13 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; import java.nio.file.Paths; -import java.util.Arrays; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Properties; import java.util.Set; -import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -49,6 +52,60 @@ public class ExportPackagesIT { "javax.activation.jar" ).map(f -> JAR_PATH + f).toList(); + private static final Pattern PACKAGE_PATTERN = Pattern.compile("([^;,]+);\\s*version=\"([^\"]*)\"(?:,\\s*([^;,]+);\\s*uses:=\"([^\"]*)\")?"); + + record PackageInfo(String packageName, String version, List<String> clauses) implements Comparable<PackageInfo> { + + PackageInfo withoutVersion() { + return new PackageInfo(packageName, "", clauses); + } + + @Override + public String toString() { + return packageName + ":" + version; + } + + @Override + public int compareTo(PackageInfo o) { + int pkg = packageName.compareTo(o.packageName); + return (pkg != 0) ? pkg : version.compareTo(o.version); + } + } + + record PackageSet(List<PackageInfo> packages) { + PackageSet removeJavaVersion() { + return new PackageSet(packages.stream() + .map(p -> p.version.contains(".JavaSE_") ? p.withoutVersion() : p) + .toList()); + } + + PackageSet removeNewPackageOnJava20() { + return new PackageSet(packages.stream() + .filter(p -> ! p.packageName.contains("java.lang.foreign")) + .filter(p -> ! p.packageName.contains("com.sun.jna")) + .toList()); + } + + boolean isEquivalentTo(PackageSet other) { + var thisPackages = new HashSet<>(removeJavaVersion().removeNewPackageOnJava20().packages); + var otherPackages = new HashSet<>(other.removeJavaVersion().removeNewPackageOnJava20().packages); + return thisPackages.equals(otherPackages); + } + + PackageSet minus(PackageSet other) { + var thisPackages = new HashSet<>(removeJavaVersion().removeNewPackageOnJava20().packages); + var otherPackages = new HashSet<>(other.removeJavaVersion().removeNewPackageOnJava20().packages); + Set<PackageInfo> diff = Sets.difference(thisPackages, otherPackages); + return new PackageSet(diff.stream().sorted().toList()); + } + + @Override + public String toString() { + return packages.stream().map(PackageInfo::toString) + .collect(Collectors.joining(",\n ", " [", "]")); + } + } + @TempDir public static File tempFolder; @@ -62,60 +119,60 @@ public class ExportPackagesIT { String expectedValue = expectedProperties.getProperty(ExportPackages.EXPORT_PACKAGES); assertNotNull(expectedValue, "Missing exportPackages property in file."); - Set<String> actualPackages = removeNewPackageOnJava20(removeJavaVersion(getPackages(actualValue))); - Set<String> expectedPackages = removeNewPackageOnJava20(removeJavaVersion(getPackages(expectedValue))); - if (!actualPackages.equals(expectedPackages)) { + var expectedPackages = parsePackages(expectedValue).removeJavaVersion(); + var actualPackages = parsePackages(actualValue).removeJavaVersion() + .removeNewPackageOnJava20(); + + if (!actualPackages.isEquivalentTo(expectedPackages)) { StringBuilder message = getDiff(actualPackages, expectedPackages); message.append("\n\nIf this test fails due to an intentional change in exported packages, run the following command:\n") .append("$ cp jdisc_core/target/classes/exportPackages.properties jdisc_core/src/test/resources/") .append("\n\nNote that removing exported packages usually requires a new major version of Vespa.\n"); fail(message.toString()); } + // TODO: check that actualValue equals expectedValue. Problem is that exportPackages.properties is not deterministic. } - private static Set<String> removeJavaVersion(Set<String> packages) { - return packages.stream().map(p -> p.replaceAll(".JavaSE_\\d+", "")).collect(Collectors.toSet()); - } - - private static Set<String> removeNewPackageOnJava20(Set<String> packages) { - return packages.stream() - .filter(p -> ! p.contains("java.lang.foreign")) - .filter(p -> ! p.contains("com.sun.jna")) - .collect(Collectors.toSet()); - } - - private static StringBuilder getDiff(Set<String> actual, Set<String> expected) { + private static StringBuilder getDiff(PackageSet actual, PackageSet expected) { StringBuilder sb = new StringBuilder(); - Set<String> onlyInActual = onlyInSet1(actual, expected); - if (! onlyInActual.isEmpty()) { + + var onlyInActual = actual.minus(expected); + if (! onlyInActual.packages().isEmpty()) { sb.append("\nexportPackages.properties contained ") - .append(onlyInActual.size()) + .append(onlyInActual.packages.size()) .append(" unexpected packages:\n") - .append(onlyInActual.stream().collect(Collectors.joining(",\n ", " [", "]"))); + .append(onlyInActual); } - Set<String> onlyInExpected = onlyInSet1(expected, actual); - if (! onlyInExpected.isEmpty()) { + var onlyInExpected = expected.minus(actual); + if (! onlyInExpected.packages.isEmpty()) { sb.append("\nexportPackages.properties did not contain ") - .append(onlyInExpected.size()) + .append(onlyInExpected.packages.size()) .append(" expected packages:\n") - .append(onlyInExpected.stream().collect(Collectors.joining(",\n ", " [", "]"))); + .append(onlyInExpected); } return sb; } - // Returns a sorted set for readability. - private static Set<String> onlyInSet1(Set<String> set1, Set<String> set2) { - return set1.stream() - .filter(s -> ! set2.contains(s)) - .collect(Collectors.toCollection(TreeSet::new)); - } + public static PackageSet parsePackages(String input) { + List<PackageInfo> packages = new ArrayList<>(); - private static Set<String> getPackages(String propertyValue) { - return Arrays.stream(propertyValue.split(",")) - .map(String::trim) - .filter(s -> ! s.isEmpty()) - .collect(Collectors.toSet()); + Matcher matcher = PACKAGE_PATTERN.matcher(input); + while (matcher.find()) { + String packageName = matcher.group(1); + String version = matcher.group(2); + String dependencyPackage = matcher.group(3); + String dependencyClause = matcher.group(4); + + List<String> clauses = new ArrayList<>(); + if (dependencyPackage != null && dependencyClause != null) { + clauses.add(dependencyPackage + ";" + dependencyClause); + } + + PackageInfo packageInfo = new PackageInfo(packageName, version, clauses); + packages.add(packageInfo); + } + return new PackageSet(packages); } private static Properties getPropertiesFromFile(File file) throws IOException { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/LockedNodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/LockedNodeList.java index 9bc18533ddf..e760e36f90b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/LockedNodeList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/LockedNodeList.java @@ -24,7 +24,7 @@ public final class LockedNodeList extends NodeList { this.lock = Objects.requireNonNull(lock, "lock must be non-null"); } - /** Returns a new LockedNodeList with the for the same lock. */ + /** Returns a new LockedNodeList with the same lock. */ public LockedNodeList childList(List<Node> nodes) { return new LockedNodeList(nodes, lock); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java index 2a0b4f02b20..331759127e4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java @@ -268,11 +268,9 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { // build() requires a version, even though it is not (should not be) used .vespaVersion(Vtag.currentVersion) .build(); - NodeSpec nodeSpec = NodeSpec.from(clusterCapacity.count(), nodeResources, false, true, + NodeSpec nodeSpec = NodeSpec.from(clusterCapacity.count(), 1, nodeResources, false, true, nodeRepository().zone().cloud().account(), Duration.ZERO); - int wantedGroups = 1; - - NodePrioritizer prioritizer = new NodePrioritizer(allNodes, applicationId, clusterSpec, nodeSpec, wantedGroups, + NodePrioritizer prioritizer = new NodePrioritizer(allNodes, applicationId, clusterSpec, nodeSpec, true, nodeRepository().nameResolver(), nodeRepository().nodes(), nodeRepository().resourcesCalculator(), nodeRepository().spareCount(), nodeSpec.cloudAccount().isExclave(nodeRepository().zone())); List<NodeCandidate> nodeCandidates = prioritizer.collect(List.of()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index eec195ccfcb..0bb045dc6a1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -24,7 +24,6 @@ import com.yahoo.vespa.hosted.provision.applications.Applications; import com.yahoo.vespa.hosted.provision.maintenance.NodeFailer; import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; import com.yahoo.vespa.hosted.provision.persistence.CuratorDb; -import com.yahoo.vespa.hosted.provision.provisioning.HostIpConfig; import com.yahoo.vespa.orchestrator.HostNameNotFoundException; import com.yahoo.vespa.orchestrator.Orchestrator; @@ -36,7 +35,6 @@ import java.util.Collection; import java.util.Comparator; import java.util.EnumSet; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.NavigableSet; import java.util.Optional; @@ -48,6 +46,7 @@ import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; +import static com.yahoo.collections.Iterables.reversed; import static com.yahoo.vespa.hosted.provision.restapi.NodePatcher.DROP_DOCUMENTS_REPORT; import static java.util.Comparator.comparing; import static java.util.stream.Collectors.groupingBy; @@ -968,7 +967,7 @@ public class Nodes { // If the first node is now earlier in lock order than some other locks we have, we need to close those and re-acquire them. Node next = unlocked.pollFirst(); Set<NodeMutex> outOfOrder = locked.tailSet(new NodeMutex(next, () -> { }), false); - NodeMutexes.close(outOfOrder.iterator()); + NodeMutexes.close(outOfOrder); for (NodeMutex node : outOfOrder) unlocked.add(node.node()); outOfOrder.clear(); @@ -1002,15 +1001,25 @@ public class Nodes { } finally { // If we didn't manage to lock all nodes, we must close the ones we did lock before we throw. - NodeMutexes.close(locked.iterator()); + NodeMutexes.close(locked); } } /** A node with their locks, acquired in a universal order. */ public record NodeMutexes(List<NodeMutex> nodes) implements AutoCloseable { - @Override public void close() { close(nodes.iterator()); } - private static void close(Iterator<NodeMutex> nodes) { - if (nodes.hasNext()) try (NodeMutex node = nodes.next()) { close(nodes); } + @Override public void close() { close(nodes); } + private static void close(Collection<NodeMutex> nodes) { + RuntimeException thrown = null; + for (NodeMutex node : reversed(List.copyOf(nodes))) { + try { + node.close(); + } + catch (RuntimeException e) { + if (thrown == null) thrown = e; + else thrown.addSuppressed(e); + } + } + if (thrown != null) throw thrown; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java index c25f33bc8c2..9adff9f9d7a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Activator.java @@ -14,7 +14,6 @@ import com.yahoo.vespa.hosted.provision.NodeMutex; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; -import com.yahoo.vespa.hosted.provision.autoscale.Autoscaling; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; @@ -71,8 +70,8 @@ class Activator { NodeList allNodes = nodeRepository.nodes().list(); NodeList applicationNodes = allNodes.owner(application); - NodeList reserved = updatePortsFrom(hosts, applicationNodes.state(Node.State.reserved) - .matching(node -> hostnames.contains(node.hostname()))); + NodeList reserved = applicationNodes.state(Node.State.reserved).matching(node -> hostnames.contains(node.hostname())); + reserved = updatePortsFrom(hosts, reserved); nodeRepository.nodes().reserve(reserved.asList()); // Re-reserve nodes to avoid reservation expiry NodeList oldActive = applicationNodes.state(Node.State.active); // All nodes active now diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupIndices.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupIndices.java new file mode 100644 index 00000000000..44f371be293 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupIndices.java @@ -0,0 +1,163 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.node.Agent; + +import java.time.Clock; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; + +/** + * Knows how to assign a group index to a number of nodes (some of which have an index already), + * such that the nodes are placed in the desired groups with minimal group movement. + * + * @author bratseth + */ +class GroupIndices { + + private final NodeSpec requested; + private final NodeList allNodes; + private final Clock clock; + + GroupIndices(NodeSpec requested, NodeList allNodes, Clock clock) { + if (requested.groups() > 1 && requested.count().isEmpty()) + throw new IllegalArgumentException("Unlimited nodes cannot be grouped"); + this.requested = requested; + this.allNodes = allNodes; + this.clock = clock; + } + + Collection<NodeCandidate> assignTo(Collection<NodeCandidate> nodes) { + int[] countInGroup = countInEachGroup(nodes); + nodes = byUnretiringPriority(nodes).stream().map(node -> unretireNodeInExpandedGroup(node, countInGroup)).toList(); + nodes = nodes.stream().map(node -> assignGroupToNewNode(node, countInGroup)).toList(); + nodes = byUnretiringPriority(nodes).stream().map(node -> moveNodeInSurplusGroup(node, countInGroup)).toList(); + nodes = byRetiringPriority(nodes).stream().map(node -> retireSurplusNodeInGroup(node, countInGroup)).toList(); + nodes = nodes.stream().filter(node -> ! shouldRemove(node)).toList(); + return nodes; + } + + /** Prefer to retire nodes we want the least */ + private List<NodeCandidate> byRetiringPriority(Collection<NodeCandidate> candidates) { + return candidates.stream().sorted(Comparator.reverseOrder()).toList(); + } + + /** Prefer to unretire nodes we don't want to retire, and otherwise those with lower index */ + private List<NodeCandidate> byUnretiringPriority(Collection<NodeCandidate> candidates) { + return candidates.stream() + .sorted(Comparator.comparing(NodeCandidate::wantToRetire) + .thenComparing(n -> n.allocation().get().membership().index())) + .toList(); + } + + private int[] countInEachGroup(Collection<NodeCandidate> nodes) { + int[] countInGroup = new int[requested.groups()]; + for (var node : nodes) { + if (node.allocation().get().membership().retired()) continue; + var currentGroup = node.allocation().get().membership().cluster().group(); + if (currentGroup.isEmpty()) continue; + if (currentGroup.get().index() >= requested.groups()) continue; + countInGroup[currentGroup.get().index()]++; + } + return countInGroup; + } + + /** Assign a group to new or to be reactivated nodes. */ + private NodeCandidate assignGroupToNewNode(NodeCandidate node, int[] countInGroup) { + if (node.state() == Node.State.active && node.allocation().get().membership().retired()) return node; + if (node.state() == Node.State.active && node.allocation().get().membership().cluster().group().isPresent()) return node; + return inFirstGroupWithDeficiency(node, countInGroup); + } + + private NodeCandidate moveNodeInSurplusGroup(NodeCandidate node, int[] countInGroup) { + var currentGroup = node.allocation().get().membership().cluster().group(); + if (currentGroup.isEmpty()) return node; // Shouldn't happen + if (currentGroup.get().index() < requested.groups()) return node; + return inFirstGroupWithDeficiency(node, countInGroup); + } + + private NodeCandidate retireSurplusNodeInGroup(NodeCandidate node, int[] countInGroup) { + if (node.allocation().get().membership().retired()) return node; + var currentGroup = node.allocation().get().membership().cluster().group(); + if (currentGroup.isEmpty()) return node; + if (currentGroup.get().index() >= requested.groups()) return node; + if (requested.count().isEmpty()) return node; // Can't retire + if (countInGroup[currentGroup.get().index()] <= requested.count().get() / requested.groups()) return node; + countInGroup[currentGroup.get().index()]--; + return node.withNode(node.toNode().retire(Agent.application, clock.instant())); + } + + /** Unretire nodes that are already in the correct group when the group is deficient. */ + private NodeCandidate unretireNodeInExpandedGroup(NodeCandidate node, int[] countInGroup) { + if ( ! node.allocation().get().membership().retired()) return node; + var currentGroup = node.allocation().get().membership().cluster().group(); + if (currentGroup.isEmpty()) return node; + if (currentGroup.get().index() >= requested.groups()) return node; + if (node.preferToRetire() || node.wantToRetire()) return node; + if (requested.count().isPresent() && countInGroup[currentGroup.get().index()] >= requested.count().get() / requested.groups()) return node; + node = unretire(node); + if (node.allocation().get().membership().retired()) return node; + countInGroup[currentGroup.get().index()]++; + return node; + } + + private NodeCandidate inFirstGroupWithDeficiency(NodeCandidate node, int[] countInGroup) { + for (int group = 0; group < requested.groups(); group++) { + if (requested.count().isEmpty() || countInGroup[group] < requested.count().get() / requested.groups()) { + return inGroup(group, node, countInGroup); + } + } + return node; + } + + private boolean shouldRemove(NodeCandidate node) { + var currentGroup = node.allocation().get().membership().cluster().group(); + if (currentGroup.isEmpty()) return true; // new and not assigned an index: Not needed + return currentGroup.get().index() >= requested.groups(); + } + + private NodeCandidate inGroup(int group, NodeCandidate node, int[] countInGroup) { + node = unretire(node); + if (node.allocation().get().membership().retired()) return node; + var membership = node.allocation().get().membership(); + var currentGroup = membership.cluster().group(); + countInGroup[group]++; + if ( ! currentGroup.isEmpty() && currentGroup.get().index() < requested.groups()) + countInGroup[membership.cluster().group().get().index()]--; + return node.withNode(node.toNode().with(node.allocation().get().with(membership.with(membership.cluster().with(Optional.of(ClusterSpec.Group.from(group))))))); + } + + /** Attempt to unretire the given node if it is retired. */ + private NodeCandidate unretire(NodeCandidate node) { + if (node.retiredNow()) return node; + if ( ! node.allocation().get().membership().retired()) return node; + if ( ! hasCompatibleResources(node) ) return node; + var parent = node.parentHostname().flatMap(hostname -> allNodes.node(hostname)); + if (parent.isPresent() && (parent.get().status().wantToRetire() || parent.get().status().preferToRetire())) return node; + node = node.withNode(); + if ( ! requested.isCompatible(node.resources())) + node = node.withNode(resize(node.toNode())); + return node.withNode(node.toNode().unretire()); + } + + private Node resize(Node node) { + NodeResources hostResources = allNodes.parentOf(node).get().flavor().resources(); + return node.with(new Flavor(requested.resources().get() + .with(hostResources.diskSpeed()) + .with(hostResources.storageType()) + .with(hostResources.architecture())), + Agent.application, clock.instant()); + } + + private boolean hasCompatibleResources(NodeCandidate candidate) { + return requested.isCompatible(candidate.resources()) || candidate.isResizable; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 0c4838abe4d..e6b47d38779 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -61,14 +61,14 @@ public class GroupPreparer { // but it may not change the set of active nodes, as the active nodes must stay in sync with the // active config model which is changed on activate public PrepareResult prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, - List<Node> surplusActiveNodes, NodeIndices indices, int wantedGroups, + List<Node> surplusActiveNodes, NodeIndices indices, LockedNodeList allNodes) { log.log(Level.FINE, () -> "Preparing " + cluster.type().name() + " " + cluster.id() + " with requested resources " + requestedNodes.resources().orElse(NodeResources.unspecified())); // Try preparing in memory without global unallocated lock. Most of the time there should be no changes, // and we can return nodes previously allocated. NodeAllocation probeAllocation = prepareAllocation(application, cluster, requestedNodes, surplusActiveNodes, - indices::probeNext, wantedGroups, allNodes); + indices::probeNext, allNodes); if (probeAllocation.fulfilledAndNoChanges()) { List<Node> acceptedNodes = probeAllocation.finalNodes(); surplusActiveNodes.removeAll(acceptedNodes); @@ -77,7 +77,7 @@ public class GroupPreparer { } else { // There were some changes, so re-do the allocation with locks indices.resetProbe(); - List<Node> prepared = prepareWithLocks(application, cluster, requestedNodes, surplusActiveNodes, indices, wantedGroups); + List<Node> prepared = prepareWithLocks(application, cluster, requestedNodes, surplusActiveNodes, indices); return new PrepareResult(prepared, createUnlockedNodeList()); } } @@ -87,12 +87,12 @@ public class GroupPreparer { /// Note that this will write to the node repo. private List<Node> prepareWithLocks(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, - List<Node> surplusActiveNodes, NodeIndices indices, int wantedGroups) { + List<Node> surplusActiveNodes, NodeIndices indices) { try (Mutex lock = nodeRepository.applications().lock(application); Mutex allocationLock = nodeRepository.nodes().lockUnallocated()) { LockedNodeList allNodes = nodeRepository.nodes().list(allocationLock); NodeAllocation allocation = prepareAllocation(application, cluster, requestedNodes, surplusActiveNodes, - indices::next, wantedGroups, allNodes); + indices::next, allNodes); NodeType hostType = allocation.nodeType().hostType(); if (canProvisionDynamically(hostType) && allocation.hostDeficit().isPresent()) { HostSharing sharing = hostSharing(cluster, hostType); @@ -134,27 +134,25 @@ public class GroupPreparer { // Non-dynamically provisioned zone with a deficit because we just now retired some nodes. // Try again, but without retiring indices.resetProbe(); - List<Node> accepted = prepareWithLocks(application, cluster, cns.withoutRetiring(), surplusActiveNodes, indices, wantedGroups); + List<Node> accepted = prepareWithLocks(application, cluster, cns.withoutRetiring(), surplusActiveNodes, indices); log.warning("Prepared " + application + " " + cluster.id() + " without retirement due to lack of capacity"); return accepted; } if (! allocation.fulfilled() && requestedNodes.canFail()) - throw new NodeAllocationException((cluster.group().isPresent() ? "Node allocation failure on " + cluster.group().get() - : "") + allocation.allocationFailureDetails(), - true); + throw new NodeAllocationException(allocation.allocationFailureDetails(), true); // Carry out and return allocation + List<Node> acceptedNodes = allocation.finalNodes(); nodeRepository.nodes().reserve(allocation.reservableNodes()); nodeRepository.nodes().addReservedNodes(new LockedNodeList(allocation.newNodes(), allocationLock)); - List<Node> acceptedNodes = allocation.finalNodes(); surplusActiveNodes.removeAll(acceptedNodes); return acceptedNodes; } } private NodeAllocation prepareAllocation(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, - List<Node> surplusActiveNodes, Supplier<Integer> nextIndex, int wantedGroups, + List<Node> surplusActiveNodes, Supplier<Integer> nextIndex, LockedNodeList allNodes) { NodeAllocation allocation = new NodeAllocation(allNodes, application, cluster, requestedNodes, nextIndex, nodeRepository); @@ -162,7 +160,6 @@ public class GroupPreparer { application, cluster, requestedNodes, - wantedGroups, nodeRepository.zone().cloud().dynamicProvisioning(), nodeRepository.nameResolver(), nodeRepository.nodes(), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index a2e0e59e329..40e5909d4d9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -19,7 +19,6 @@ import com.yahoo.vespa.hosted.provision.node.Allocation; import java.util.ArrayList; import java.util.Collection; -import java.util.Comparator; import java.util.EnumSet; import java.util.HashSet; import java.util.LinkedHashMap; @@ -60,7 +59,7 @@ class NodeAllocation { /** The number of already allocated nodes of compatible size */ private int acceptedAndCompatible = 0; - /** The number of already allocated nodes which can be made compatible*/ + /** The number of already allocated nodes which can be made compatible */ private int acceptedAndCompatibleOrResizable = 0; /** The number of nodes rejected because of clashing parentHostname */ @@ -120,7 +119,6 @@ class NodeAllocation { ClusterMembership membership = allocation.membership(); if ( ! allocation.owner().equals(application)) continue; // wrong application if ( ! membership.cluster().satisfies(cluster)) continue; // wrong cluster id/type - if ((! candidate.isSurplus || saturated()) && ! membership.cluster().group().equals(cluster.group())) continue; // wrong group, and we can't or have no reason to change it if ( candidate.state() == Node.State.active && allocation.removable()) continue; // don't accept; causes removal if ( candidate.state() == Node.State.active && candidate.wantToFail()) continue; // don't accept; causes failing if ( indexes.contains(membership.index())) continue; // duplicate index (just to be sure) @@ -175,6 +173,7 @@ class NodeAllocation { if (candidate.preferToRetire() && candidate.replaceableBy(candidates)) return Retirement.softRequest; if (violatesExclusivity(candidate)) return Retirement.violatesExclusivity; if (requiredHostFlavor.isPresent() && ! candidate.parent.map(node -> node.flavor().name()).equals(requiredHostFlavor)) return Retirement.violatesHostFlavor; + if (candidate.violatesSpares) return Retirement.violatesSpares; return Retirement.none; } @@ -243,12 +242,10 @@ class NodeAllocation { */ private boolean acceptIncompatible(NodeCandidate candidate) { if (candidate.state() != Node.State.active) return false; - if (! candidate.allocation().get().membership().cluster().group().equals(cluster.group())) return false; if (candidate.allocation().get().membership().retired()) return true; // don't second-guess if already retired if ( ! requestedNodes.considerRetiring()) // the node is active and we are not allowed to remove gracefully, so keep return true; - return cluster.isStateful() || (cluster.type() == ClusterSpec.Type.container && !hasCompatibleResources(candidate)); } @@ -259,7 +256,6 @@ class NodeAllocation { private Node acceptNode(NodeCandidate candidate, Retirement retirement, boolean resizeable) { Node node = candidate.toNode(); - if (node.allocation().isPresent()) // Record the currently requested resources node = node.with(node.allocation().get().withRequestedResources(requestedNodes.resources().orElse(node.resources()))); @@ -268,10 +264,11 @@ class NodeAllocation { // We want to allocate new nodes rather than unretiring with resize, so count without those // for the purpose of deciding when to stop accepting nodes (saturation) if (node.allocation().isEmpty() - || ! ( requestedNodes.needsResize(node) && - (node.allocation().get().membership().retired() || ! requestedNodes.considerRetiring()))) { + || (canBeUsedInGroupWithDeficiency(node) && + ! ( requestedNodes.needsResize(node) && (node.allocation().get().membership().retired() || ! requestedNodes.considerRetiring())))) { acceptedAndCompatible++; } + if (hasCompatibleResources(candidate)) acceptedAndCompatibleOrResizable++; @@ -289,15 +286,28 @@ class NodeAllocation { node = node.retire(nodeRepository.clock().instant()); } if ( ! node.allocation().get().membership().cluster().equals(cluster)) { - // group may be different - node = setCluster(cluster, node); + // Cluster has the updated settings but do not set a group + node = setCluster(cluster.with(node.allocation().get().membership().cluster().group()), node); } - candidate = candidate.withNode(node); + candidate = candidate.withNode(node, retirement != Retirement.none && retirement != Retirement.alreadyRetired ); indexes.add(node.allocation().get().membership().index()); nodes.put(node.hostname(), candidate); return node; } + private boolean canBeUsedInGroupWithDeficiency(Node node) { + if (requestedNodes.count().isEmpty()) return true; + if (node.allocation().isEmpty()) return true; + var group = node.allocation().get().membership().cluster().group(); + if (group.isEmpty()) return true; + long nodesInGroup = nodes.values().stream().filter(n -> groupOf(n).equals(group)).count(); + return nodesInGroup < requestedNodes.count().get() / requestedNodes.groups(); + } + + private Optional<ClusterSpec.Group> groupOf(NodeCandidate candidate) { + return candidate.allocation().flatMap(a -> a.membership().cluster().group()); + } + private Node resize(Node node) { NodeResources hostResources = allNodes.parentOf(node).get().flavor().resources(); return node.with(new Flavor(requestedNodes.resources().get() @@ -391,52 +401,21 @@ class NodeAllocation { return requestedNodes.type(); } - /** - * Make the number of <i>non-retired</i> nodes in the list equal to the requested number - * of nodes, and retire the rest of the list. Only retire currently active nodes. - * Prefer to retire nodes of the wrong flavor. - * Make as few changes to the retired set as possible. - * - * @return the final list of nodes - */ List<Node> finalNodes() { - int wantToRetireCount = (int) matching(NodeCandidate::wantToRetire).count(); - int currentRetiredCount = (int) matching(node -> node.allocation().get().membership().retired()).count(); - int deltaRetiredCount = requestedNodes.idealRetiredCount(nodes.size(), wantToRetireCount, currentRetiredCount); - - if (deltaRetiredCount > 0) { // retire until deltaRetiredCount is 0 - for (NodeCandidate candidate : byRetiringPriority(nodes.values())) { - if ( ! candidate.allocation().get().membership().retired() && candidate.state() == Node.State.active) { - candidate = candidate.withNode(); - candidate = candidate.withNode(candidate.toNode().retire(Agent.application, nodeRepository.clock().instant())); - nodes.put(candidate.toNode().hostname(), candidate); - if (--deltaRetiredCount == 0) break; - } - } - } - else if (deltaRetiredCount < 0) { // unretire until deltaRetiredCount is 0 - for (NodeCandidate candidate : byUnretiringPriority(nodes.values())) { - if (candidate.allocation().get().membership().retired() && hasCompatibleResources(candidate) ) { - candidate = candidate.withNode(); - if (candidate.isResizable) - candidate = candidate.withNode(resize(candidate.toNode())); - candidate = candidate.withNode(candidate.toNode().unretire()); - nodes.put(candidate.toNode().hostname(), candidate); - if (++deltaRetiredCount == 0) break; - } - } - } - + // Set whether the node is exclusive for (NodeCandidate candidate : nodes.values()) { - // Set whether the node is exclusive candidate = candidate.withNode(); Allocation allocation = candidate.allocation().get(); candidate = candidate.withNode(candidate.toNode().with(allocation.with(allocation.membership() - .with(allocation.membership().cluster().exclusive(cluster.isExclusive()))))); + .with(allocation.membership().cluster().exclusive(cluster.isExclusive()))))); nodes.put(candidate.toNode().hostname(), candidate); } - return nodes.values().stream().map(NodeCandidate::toNode).toList(); + GroupIndices groupIndices = new GroupIndices(requestedNodes, allNodes, nodeRepository.clock()); + Collection<NodeCandidate> finalNodes = groupIndices.assignTo(nodes.values()); + nodes.clear(); + finalNodes.forEach(candidate -> nodes.put(candidate.toNode().hostname(), candidate)); + return finalNodes.stream().map(NodeCandidate::toNode).toList(); } List<Node> reservableNodes() { @@ -461,19 +440,6 @@ class NodeAllocation { return allNodes.nodeType(nodeType()).size(); } - /** Prefer to retire nodes we want the least */ - private List<NodeCandidate> byRetiringPriority(Collection<NodeCandidate> candidates) { - return candidates.stream().sorted(Comparator.reverseOrder()).toList(); - } - - /** Prefer to unretire nodes we don't want to retire, and otherwise those with lower index */ - private List<NodeCandidate> byUnretiringPriority(Collection<NodeCandidate> candidates) { - return candidates.stream() - .sorted(Comparator.comparing(NodeCandidate::wantToRetire) - .thenComparing(n -> n.allocation().get().membership().index())) - .toList(); - } - String allocationFailureDetails() { List<String> reasons = new ArrayList<>(); if (rejectedDueToExclusivity > 0) @@ -486,7 +452,7 @@ class NodeAllocation { reasons.add("insufficient real resources on hosts"); if (reasons.isEmpty()) return ""; - return ": Not enough suitable nodes available due to " + String.join(", ", reasons); + return "Not enough suitable nodes available due to " + String.join(", ", reasons); } private static Integer parseIndex(String hostname) { @@ -510,6 +476,7 @@ class NodeAllocation { violatesExclusivity("node violates host exclusivity"), violatesHostFlavor("node violates host flavor"), violatesHostFlavorGeneration("node violates host flavor generation"), + violatesSpares("node is assigned to a host we want to use as a spare"), none(""); private final String description; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java index 8462e23fbfd..adc04c491e2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java @@ -81,6 +81,9 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat public abstract boolean preferToRetire(); + /** Returns true if we have decided to retire this node as part of this deployment */ + public boolean retiredNow() { return false; } + public abstract boolean wantToFail(); public abstract Flavor flavor(); @@ -217,7 +220,12 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat /** Returns a copy of this with node set to given value */ NodeCandidate withNode(Node node) { - return new ConcreteNodeCandidate(node, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizable); + return withNode(node, retiredNow()); + } + + /** Returns a copy of this with node set to given value */ + NodeCandidate withNode(Node node, boolean retiredNow) { + return new ConcreteNodeCandidate(node, retiredNow, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizable); } /** Returns the switch priority, based on switch exclusivity, of this compared to other */ @@ -260,7 +268,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat boolean isSurplus, boolean isNew, boolean isResizeable) { - return new ConcreteNodeCandidate(node, freeParentCapacity, Optional.of(parent), violatesSpares, true, isSurplus, isNew, isResizeable); + return new ConcreteNodeCandidate(node, false, freeParentCapacity, Optional.of(parent), violatesSpares, true, isSurplus, isNew, isResizeable); } public static NodeCandidate createNewChild(NodeResources resources, @@ -274,26 +282,33 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat } public static NodeCandidate createNewExclusiveChild(Node node, Node parent) { - return new ConcreteNodeCandidate(node, node.resources(), Optional.of(parent), false, true, false, true, false); + return new ConcreteNodeCandidate(node, false, node.resources(), Optional.of(parent), false, true, false, true, false); } public static NodeCandidate createStandalone(Node node, boolean isSurplus, boolean isNew) { - return new ConcreteNodeCandidate(node, node.resources(), Optional.empty(), false, true, isSurplus, isNew, false); + return new ConcreteNodeCandidate(node, false, node.resources(), Optional.empty(), false, true, isSurplus, isNew, false); } /** A candidate backed by a node */ static class ConcreteNodeCandidate extends NodeCandidate { private final Node node; + private final boolean retiredNow; - ConcreteNodeCandidate(Node node, NodeResources freeParentCapacity, Optional<Node> parent, + ConcreteNodeCandidate(Node node, + boolean retiredNow, + NodeResources freeParentCapacity, Optional<Node> parent, boolean violatesSpares, boolean exclusiveSwitch, boolean isSurplus, boolean isNew, boolean isResizeable) { super(freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizeable); + this.retiredNow = retiredNow; this.node = Objects.requireNonNull(node, "Node cannot be null"); } @Override + public boolean retiredNow() { return retiredNow; } + + @Override public NodeResources resources() { return node.resources(); } @Override @@ -322,7 +337,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat @Override public NodeCandidate allocate(ApplicationId owner, ClusterMembership membership, NodeResources requestedResources, Instant at) { - return new ConcreteNodeCandidate(node.allocate(owner, membership, requestedResources, at), + return new ConcreteNodeCandidate(node.allocate(owner, membership, requestedResources, at), retiredNow, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizable); } @@ -332,7 +347,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat @Override public NodeCandidate withExclusiveSwitch(boolean exclusiveSwitch) { - return new ConcreteNodeCandidate(node, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, + return new ConcreteNodeCandidate(node, retiredNow, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizable); } @@ -439,7 +454,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat NodeType.tenant) .cloudAccount(parent.get().cloudAccount()) .build(); - return new ConcreteNodeCandidate(node, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizable); + return new ConcreteNodeCandidate(node, false, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizable); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index 4f21c8dcd50..9f00e5fdbba 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -46,7 +46,7 @@ public class NodePrioritizer { private final boolean enclave; public NodePrioritizer(LockedNodeList allNodes, ApplicationId application, ClusterSpec clusterSpec, NodeSpec nodeSpec, - int wantedGroups, boolean dynamicProvisioning, NameResolver nameResolver, Nodes nodes, + boolean dynamicProvisioning, NameResolver nameResolver, Nodes nodes, HostResourcesCalculator hostResourcesCalculator, int spareCount, boolean enclave) { this.allNodes = allNodes; this.calculator = hostResourcesCalculator; @@ -70,12 +70,9 @@ public class NodePrioritizer { .stream()) .distinct() .count(); - this.topologyChange = currentGroups != wantedGroups; + this.topologyChange = currentGroups != requestedNodes.groups(); - this.currentClusterSize = (int) nonRetiredNodesInCluster.state(Node.State.active).stream() - .map(node -> node.allocation().flatMap(alloc -> alloc.membership().cluster().group())) - .filter(clusterSpec.group()::equals) - .count(); + this.currentClusterSize = (int) nonRetiredNodesInCluster.state(Node.State.active).stream().count(); // In dynamically provisioned zones, we can always take spare hosts since we can provision new on-demand, // NodeCandidate::compareTo will ensure that they will not be used until there is no room elsewhere. diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index ad91bdef478..c29c51ccbd5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -96,19 +96,17 @@ public class NodeRepositoryProvisioner implements Provisioner { validate(actual, target, cluster, application); logIfDownscaled(requested.minResources().nodes(), actual.minResources().nodes(), cluster, logger); - groups = target.groups(); resources = getNodeResources(cluster, target.nodeResources(), application); - nodeSpec = NodeSpec.from(target.nodes(), resources, cluster.isExclusive(), actual.canFail(), + nodeSpec = NodeSpec.from(target.nodes(), target.groups(), resources, cluster.isExclusive(), actual.canFail(), requested.cloudAccount().orElse(nodeRepository.zone().cloud().account()), requested.clusterInfo().hostTTL()); } else { - groups = 1; // type request with multiple groups is not supported cluster = cluster.withExclusivity(true); resources = getNodeResources(cluster, requested.minResources().nodeResources(), application); nodeSpec = NodeSpec.from(requested.type(), nodeRepository.zone().cloud().account()); } - return asSortedHosts(preparer.prepare(application, cluster, nodeSpec, groups), + return asSortedHosts(preparer.prepare(application, cluster, nodeSpec), requireCompatibleResources(resources, cluster)); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java index f32928a9ec4..f4b2c4ceee0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java @@ -35,21 +35,20 @@ public interface NodeSpec { return fulfilledDeficitCount(count) == 0; } + /** Returns the total number of nodes this is requesting, or empty if not specified */ + Optional<Integer> count(); + + int groups(); + /** Returns whether this should throw an exception if the requested nodes are not fully available */ boolean canFail(); /** Returns whether we should retire nodes at all when fulfilling this spec */ boolean considerRetiring(); - /** Returns the ideal number of nodes that should be retired to fulfill this spec */ - int idealRetiredCount(int acceptedCount, int wantToRetireCount, int currentRetiredCount); - /** Returns number of additional nodes needed for this spec to be fulfilled given the current node count */ int fulfilledDeficitCount(int count); - /** Returns a specification of a fraction of all the nodes of this. It is assumed the argument is a valid divisor. */ - NodeSpec fraction(int divisor); - /** Returns the resources requested by this or empty if none are explicitly requested */ Optional<NodeResources> resources(); @@ -77,9 +76,9 @@ public interface NodeSpec { return false; } - static NodeSpec from(int nodeCount, NodeResources resources, boolean exclusive, boolean canFail, + static NodeSpec from(int nodeCount, int groupCount, NodeResources resources, boolean exclusive, boolean canFail, CloudAccount cloudAccount, Duration hostTTL) { - return new CountNodeSpec(nodeCount, resources, exclusive, canFail, canFail, cloudAccount, hostTTL); + return new CountNodeSpec(nodeCount, groupCount, resources, exclusive, canFail, canFail, cloudAccount, hostTTL); } static NodeSpec from(NodeType type, CloudAccount cloudAccount) { @@ -90,6 +89,7 @@ public interface NodeSpec { class CountNodeSpec implements NodeSpec { private final int count; + private final int groups; private final NodeResources requestedNodeResources; private final boolean exclusive; private final boolean canFail; @@ -97,9 +97,10 @@ public interface NodeSpec { private final CloudAccount cloudAccount; private final Duration hostTTL; - private CountNodeSpec(int count, NodeResources resources, boolean exclusive, boolean canFail, + private CountNodeSpec(int count, int groups, NodeResources resources, boolean exclusive, boolean canFail, boolean considerRetiring, CloudAccount cloudAccount, Duration hostTTL) { this.count = count; + this.groups = groups; this.requestedNodeResources = Objects.requireNonNull(resources, "Resources must be specified"); this.exclusive = exclusive; this.canFail = canFail; @@ -112,6 +113,12 @@ public interface NodeSpec { } @Override + public Optional<Integer> count() { return Optional.of(count); } + + @Override + public int groups() { return groups; } + + @Override public Optional<NodeResources> resources() { return Optional.of(requestedNodeResources); } @@ -136,22 +143,12 @@ public interface NodeSpec { } @Override - public int idealRetiredCount(int acceptedCount, int wantToRetireCount, int currentRetiredCount) { - return acceptedCount - this.count - currentRetiredCount; - } - - @Override public int fulfilledDeficitCount(int count) { return Math.max(this.count - count, 0); } - @Override - public NodeSpec fraction(int divisor) { - return new CountNodeSpec(count/divisor, requestedNodeResources, exclusive, canFail, considerRetiring, cloudAccount, hostTTL); - } - public NodeSpec withoutRetiring() { - return new CountNodeSpec(count, requestedNodeResources, exclusive, canFail, false, cloudAccount, hostTTL); + return new CountNodeSpec(count, groups, requestedNodeResources, exclusive, canFail, false, cloudAccount, hostTTL); } @Override @@ -163,7 +160,6 @@ public interface NodeSpec { public boolean canResize(NodeResources currentNodeResources, NodeResources currentSpareHostResources, ClusterSpec.Type type, boolean hasTopologyChange, int currentClusterSize) { if (exclusive) return false; // exclusive resources must match the host - // Never allow in-place resize when also changing topology or decreasing cluster size if (hasTopologyChange || count < currentClusterSize) return false; @@ -192,7 +188,10 @@ public interface NodeSpec { public Duration hostTTL() { return hostTTL; } @Override - public String toString() { return "request for " + count + " nodes with " + requestedNodeResources; } + public String toString() { + return "request for " + count + " nodes" + + ( groups > 1 ? " (in " + groups + " groups)" : "") + + " with " + requestedNodeResources; } } @@ -211,6 +210,12 @@ public interface NodeSpec { } @Override + public Optional<Integer> count() { return Optional.empty(); } + + @Override + public int groups() { return 1; } + + @Override public NodeType type() { return type; } @Override @@ -226,20 +231,12 @@ public interface NodeSpec { public boolean considerRetiring() { return true; } @Override - public int idealRetiredCount(int acceptedCount, int wantToRetireCount, int currentRetiredCount) { - return wantToRetireCount - currentRetiredCount; - } - - @Override public int fulfilledDeficitCount(int count) { // If no wanted count is specified for this node type, then any count fulfills the deficit return Math.max(0, WANTED_NODE_COUNT.getOrDefault(type, 0) - count); } @Override - public NodeSpec fraction(int divisor) { return this; } - - @Override public Optional<NodeResources> resources() { return Optional.empty(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index 42b9e53dd8a..25efcabfe8e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -9,8 +9,8 @@ import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.yolean.Exceptions; +import java.time.Clock; import java.util.ArrayList; import java.util.List; import java.util.ListIterator; @@ -33,17 +33,15 @@ class Preparer { } /** Prepare all required resources for the given application and cluster */ - public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, int wantedGroups) { + public List<Node> prepare(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes) { try { - var nodes = prepareNodes(application, cluster, requestedNodes, wantedGroups); + var nodes = prepareNodes(application, cluster, requestedNodes); prepareLoadBalancer(application, cluster, requestedNodes); return nodes; } catch (NodeAllocationException e) { - e.printStackTrace(); throw new NodeAllocationException("Could not satisfy " + requestedNodes + - ( wantedGroups > 1 ? " (in " + wantedGroups + " groups)" : "") + - " in " + application + " " + cluster + ": " + Exceptions.toMessageString(e), + " in " + application + " " + cluster, e, e.retryable()); } } @@ -56,34 +54,29 @@ class Preparer { // Note: This operation may make persisted changes to the set of reserved and inactive nodes, // but it may not change the set of active nodes, as the active nodes must stay in sync with the // active config model which is changed on activate - private List<Node> prepareNodes(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes, - int wantedGroups) { + private List<Node> prepareNodes(ApplicationId application, ClusterSpec cluster, NodeSpec requestedNodes) { LockedNodeList allNodes = groupPreparer.createUnlockedNodeList(); - NodeList appNodes = allNodes.owner(application); - List<Node> surplusNodes = findNodesInRemovableGroups(appNodes, cluster, wantedGroups); + NodeList clusterNodes = allNodes.owner(application); + List<Node> surplusNodes = findNodesInRemovableGroups(clusterNodes, requestedNodes.groups()); - List<Integer> usedIndices = appNodes.cluster(cluster.id()).mapToList(node -> node.allocation().get().membership().index()); + List<Integer> usedIndices = clusterNodes.mapToList(node -> node.allocation().get().membership().index()); NodeIndices indices = new NodeIndices(usedIndices); List<Node> acceptedNodes = new ArrayList<>(); - for (int groupIndex = 0; groupIndex < wantedGroups; groupIndex++) { - ClusterSpec clusterGroup = cluster.with(Optional.of(ClusterSpec.Group.from(groupIndex))); - GroupPreparer.PrepareResult result = groupPreparer.prepare(application, clusterGroup, - requestedNodes.fraction(wantedGroups), - surplusNodes, indices, wantedGroups, - allNodes); - allNodes = result.allNodes(); // Might have changed - List<Node> accepted = result.prepared(); - if (requestedNodes.rejectNonActiveParent()) { - NodeList activeHosts = allNodes.state(Node.State.active).parents().nodeType(requestedNodes.type().hostType()); - accepted = accepted.stream() - .filter(node -> node.parentHostname().isEmpty() || activeHosts.parentOf(node).isPresent()) - .toList(); - } - - replace(acceptedNodes, accepted); + GroupPreparer.PrepareResult result = groupPreparer.prepare(application, cluster, + requestedNodes, + surplusNodes, indices, + allNodes); + List<Node> accepted = result.prepared(); + if (requestedNodes.rejectNonActiveParent()) { + NodeList activeHosts = result.allNodes().state(Node.State.active).parents().nodeType(requestedNodes.type().hostType()); + accepted = accepted.stream() + .filter(node -> node.parentHostname().isEmpty() || activeHosts.parentOf(node).isPresent()) + .toList(); } - moveToActiveGroup(surplusNodes, wantedGroups, cluster.group()); + + replace(acceptedNodes, accepted); + moveToActiveGroup(surplusNodes, requestedNodes.groups(), cluster.group()); acceptedNodes.removeAll(surplusNodes); return acceptedNodes; } @@ -97,18 +90,16 @@ class Preparer { * Returns a list of the nodes which are * in groups with index number above or equal the group count */ - private List<Node> findNodesInRemovableGroups(NodeList appNodes, ClusterSpec requestedCluster, int wantedGroups) { + private List<Node> findNodesInRemovableGroups(NodeList clusterNodes, int wantedGroups) { List<Node> surplusNodes = new ArrayList<>(); - for (Node node : appNodes.state(Node.State.active)) { + for (Node node : clusterNodes.state(Node.State.active)) { ClusterSpec nodeCluster = node.allocation().get().membership().cluster(); - if ( ! nodeCluster.id().equals(requestedCluster.id())) continue; - if ( ! nodeCluster.type().equals(requestedCluster.type())) continue; if (nodeCluster.group().get().index() >= wantedGroups) surplusNodes.add(node); } return surplusNodes; } - + /** Move nodes from unwanted groups to wanted groups to avoid lingering groups consisting of retired nodes */ private void moveToActiveGroup(List<Node> surplusNodes, int wantedGroups, Optional<ClusterSpec.Group> targetGroup) { for (ListIterator<Node> i = surplusNodes.listIterator(); i.hasNext(); ) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java index 478b201d71b..7cf1b0d5177 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java @@ -94,7 +94,6 @@ public class DynamicAllocationTest { hostsWithChildren.add(node.parentHostname().get()); } assertEquals(4 - spareCount, hostsWithChildren.size()); - } /** @@ -342,8 +341,8 @@ public class DynamicAllocationTest { tester.activate(application, hosts); NodeList activeNodes = tester.nodeRepository().nodes().list().owner(application); - assertEquals(Set.of("127.0.127.2", "::2"), activeNodes.asList().get(0).ipConfig().primary()); - assertEquals(Set.of("127.0.127.13", "::d"), activeNodes.asList().get(1).ipConfig().primary()); + assertEquals(Set.of("127.0.127.2", "::2"), activeNodes.asList().get(1).ipConfig().primary()); + assertEquals(Set.of("127.0.127.13", "::d"), activeNodes.asList().get(0).ipConfig().primary()); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java index c99728b714b..5539bb0cb6e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java @@ -334,7 +334,7 @@ public class DynamicProvisioningTest { .flagSource(flagSource) .build(); - ApplicationId app = ProvisioningTester.applicationId(); + ApplicationId app = ProvisioningTester.applicationId("a1"); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("8").build(); Capacity capacity = Capacity.from(new ClusterResources(4, 2, new NodeResources(2, 4, 50, 0.1, DiskSpeed.any, StorageType.any, Architecture.any))); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java index de9b3a4db33..4799d3b5577 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java @@ -27,7 +27,6 @@ import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.testutils.InMemoryProvisionLogger; import java.time.Duration; -import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.Set; @@ -36,12 +35,12 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** - * A provisioniong tester which + * A provisioning tester which * - Supports dynamic provisioning (only). * - Optionally replicates the actual AWS setup and logic used on Vespa Cloud. * - Supports autoscaling testing. * - * TODO: All provisioning testing should migrate to use this, and then the provisionging tester should be collapsed + * TODO: All provisioning testing should migrate to use this, and then the provisioning tester should be collapsed * into this. ... or we should just use autoscalingtester for everything. * * @author bratseth @@ -80,13 +79,6 @@ public class DynamicProvisioningTester { capacityPolicies = new CapacityPolicies(provisioningTester.nodeRepository()); } - private static List<Flavor> toFlavors(List<NodeResources> resources) { - List<Flavor> flavors = new ArrayList<>(); - for (int i = 0; i < resources.size(); i++) - flavors.add(new Flavor("flavor" + i, resources.get(i))); - return flavors; - } - public InMemoryProvisionLogger provisionLogger() { return provisioningTester.provisionLogger(); } public static Fixture.Builder fixture() { return new Fixture.Builder(); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java index 0bb6dc61d1b..54f0507831d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java @@ -57,7 +57,7 @@ public class InPlaceResizeProvisionTest { private final ProvisioningTester tester = new ProvisioningTester.Builder() .flagSource(flagSource) .zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - private final ApplicationId app = ProvisioningTester.applicationId(); + private final ApplicationId app = ProvisioningTester.applicationId("a1"); @Test public void single_group_same_cluster_size_resource_increase() { @@ -167,8 +167,6 @@ public class InPlaceResizeProvisionTest { assertEquals(0, listCluster(content1).retired().size()); } - - /** In this scenario there should be no resizing */ @Test public void increase_size_decrease_resources() { addParentHosts(14, largeResources.with(fast)); @@ -198,15 +196,15 @@ public class InPlaceResizeProvisionTest { assertSizeAndResources(listCluster(content1).retired(), 4, resources); assertSizeAndResources(listCluster(content1).not().retired(), 8, halvedResources); - // ... same with setting a node to want to retire - Node nodeToWantoToRetire = listCluster(content1).not().retired().asList().get(0); - try (NodeMutex lock = tester.nodeRepository().nodes().lockAndGetRequired(nodeToWantoToRetire)) { + // Here we'll unretire and resize one of the previously retired nodes as there is no rule against it + Node nodeToWantToRetire = listCluster(content1).not().retired().asList().get(0); + try (NodeMutex lock = tester.nodeRepository().nodes().lockAndGetRequired(nodeToWantToRetire)) { tester.nodeRepository().nodes().write(lock.node().withWantToRetire(true, Agent.system, tester.clock().instant()), lock); } new PrepareHelper(tester, app).prepare(content1, 8, 1, halvedResources).activate(); - assertTrue(listCluster(content1).retired().stream().anyMatch(n -> n.equals(nodeToWantoToRetire))); - assertEquals(5, listCluster(content1).retired().size()); + assertTrue(listCluster(content1).retired().stream().anyMatch(n -> n.equals(nodeToWantToRetire))); + assertEquals(4, listCluster(content1).retired().size()); assertSizeAndResources(listCluster(content1).not().retired(), 8, halvedResources); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java index 32db213c445..c82b29c7d65 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java @@ -24,17 +24,17 @@ public class NodeCandidateTest { @Test public void testOrdering() { List<NodeCandidate> expected = List.of( - new NodeCandidate.ConcreteNodeCandidate(node("01", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.empty(), false, true, true, false, false), - new NodeCandidate.ConcreteNodeCandidate(node("02", Node.State.active), new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, false, false), - new NodeCandidate.ConcreteNodeCandidate(node("04", Node.State.reserved), new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, false, false), - new NodeCandidate.ConcreteNodeCandidate(node("03", Node.State.inactive), new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, false, false), - new NodeCandidate.ConcreteNodeCandidate(node("05", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.active)), true, true, false, true, false), - new NodeCandidate.ConcreteNodeCandidate(node("06", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.ready)), true, true, false, true, false), - new NodeCandidate.ConcreteNodeCandidate(node("07", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.provisioned)), true, true, false, true, false), - new NodeCandidate.ConcreteNodeCandidate(node("08", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.failed)), true, true, false, true, false), - new NodeCandidate.ConcreteNodeCandidate(node("09", Node.State.ready), new NodeResources(1, 1, 1, 1), Optional.empty(), true, true, false, true, false), - new NodeCandidate.ConcreteNodeCandidate(node("10", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, true, false), - new NodeCandidate.ConcreteNodeCandidate(node("11", Node.State.ready), new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, true, false) + new NodeCandidate.ConcreteNodeCandidate(node("01", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.empty(), false, true, true, false, false), + new NodeCandidate.ConcreteNodeCandidate(node("02", Node.State.active), false, new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, false, false), + new NodeCandidate.ConcreteNodeCandidate(node("04", Node.State.reserved), false, new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, false, false), + new NodeCandidate.ConcreteNodeCandidate(node("03", Node.State.inactive), false, new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, false, false), + new NodeCandidate.ConcreteNodeCandidate(node("05", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.active)), true, true, false, true, false), + new NodeCandidate.ConcreteNodeCandidate(node("06", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.ready)), true, true, false, true, false), + new NodeCandidate.ConcreteNodeCandidate(node("07", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.provisioned)), true, true, false, true, false), + new NodeCandidate.ConcreteNodeCandidate(node("08", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.of(node("host1", Node.State.failed)), true, true, false, true, false), + new NodeCandidate.ConcreteNodeCandidate(node("09", Node.State.ready), false, new NodeResources(1, 1, 1, 1), Optional.empty(), true, true, false, true, false), + new NodeCandidate.ConcreteNodeCandidate(node("10", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, true, false), + new NodeCandidate.ConcreteNodeCandidate(node("11", Node.State.ready), false, new NodeResources(2, 2, 2, 2), Optional.empty(), true, true, false, true, false) ); assertOrder(expected); } @@ -148,7 +148,7 @@ public class NodeCandidateTest { Node parent = Node.create(hostname + "parent", hostname, new Flavor(totalHostResources), Node.State.ready, NodeType.host) .ipConfig(IP.Config.of(Set.of("::1"), Set.of("::2"))) .build(); - return new NodeCandidate.ConcreteNodeCandidate(node, totalHostResources.subtract(allocatedHostResources), Optional.of(parent), + return new NodeCandidate.ConcreteNodeCandidate(node, false, totalHostResources.subtract(allocatedHostResources), Optional.of(parent), false, exclusiveSwitch, false, true, false); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index a76b576e430..cb4644f179f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -241,7 +241,7 @@ public class ProvisioningTest { public void application_deployment_variable_application_size() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - ApplicationId application1 = ProvisioningTester.applicationId(); + ApplicationId application1 = ProvisioningTester.applicationId("a1"); tester.makeReadyHosts(30, defaultResources); tester.activateTenantHosts(); @@ -821,7 +821,7 @@ public class ProvisioningTest { public void highest_node_indexes_are_retired_first() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - ApplicationId application1 = ProvisioningTester.applicationId(); + ApplicationId application1 = ProvisioningTester.applicationId("a1"); tester.makeReadyHosts(14, defaultResources).activateTenantHosts(); @@ -833,17 +833,19 @@ public class ProvisioningTest { SystemState state2 = prepare(application1, 2, 2, 2, 2, defaultResources, tester); tester.activate(application1, state2.allHosts); - // content0 - assertFalse(state2.hostByMembership("content0", 0, 0).membership().get().retired()); - assertFalse(state2.hostByMembership("content0", 0, 1).membership().get().retired()); - assertTrue( state2.hostByMembership("content0", 0, 2).membership().get().retired()); - assertTrue( state2.hostByMembership("content0", 0, 3).membership().get().retired()); - - // content1 - assertFalse(state2.hostByMembership("content1", 0, 0).membership().get().retired()); - assertFalse(state2.hostByMembership("content1", 0, 1).membership().get().retired()); - assertTrue( state2.hostByMembership("content1", 0, 2).membership().get().retired()); - assertTrue( state2.hostByMembership("content1", 0, 3).membership().get().retired()); + List<Integer> unretiredInContent0Indices = state2.content0.stream().filter(h -> ! h.membership().get().retired()).map(h -> h.membership().get().index()).toList(); + for (var host : state2.content0) { + if ( ! host.membership().get().retired()) continue; + for (int unretiredIndex : unretiredInContent0Indices) + assertTrue(host.membership().get().index() > unretiredIndex); + } + + List<Integer> unretiredInContent1Indices = state2.content1.stream().filter(h -> ! h.membership().get().retired()).map(h -> h.membership().get().index()).toList(); + for (var host : state2.content1) { + if ( ! host.membership().get().retired()) continue; + for (int unretiredIndex : unretiredInContent1Indices) + assertTrue(host.membership().get().index() > unretiredIndex); + } } @Test @@ -857,7 +859,7 @@ public class ProvisioningTest { tester.deploy(application, spec, Capacity.from(new ClusterResources(6, 1, defaultResources))); - // Pick out a random application node and make it's parent larger, this will make it the spare host + // Pick out a random application node and make its parent larger, this will make it the spare host NodeList nodes = tester.nodeRepository().nodes().list(); Node randomNode = nodes.owner(application).shuffle(new Random()).first().get(); tester.nodeRepository().nodes().write(nodes.parentOf(randomNode).get() diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java index a6a988052e6..6ec189d98c3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java @@ -23,6 +23,7 @@ import com.yahoo.config.provision.Zone; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.yolean.Exceptions; import org.junit.Test; import java.util.HashSet; @@ -442,9 +443,8 @@ public class VirtualNodeProvisioningTest { "Could not satisfy request for 3 nodes with " + "[vcpu: 2.0, memory: 4.0 Gb, disk: 100.0 Gb, bandwidth: 1.0 Gbps, architecture: any] " + "in tenant2.app2 container cluster 'my-container' 6.39: " + - "Node allocation failure on group 0: " + "Not enough suitable nodes available due to host exclusivity constraints", - e.getMessage()); + Exceptions.toMessageString(e)); } // Adding 3 nodes of another application for the same tenant works @@ -469,8 +469,8 @@ public class VirtualNodeProvisioningTest { assertEquals("Could not satisfy request for 2 nodes with " + "[vcpu: 1.0, memory: 4.0 Gb, disk: 100.0 Gb, bandwidth: 1.0 Gbps, storage type: remote, architecture: any] " + "in tenant.app1 content cluster 'my-content'" + - " 6.42: Node allocation failure on group 0", - e.getMessage()); + " 6.42", + Exceptions.toMessageString(e)); } } diff --git a/parent/pom.xml b/parent/pom.xml index 2224a4b213a..88248571ee7 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -252,7 +252,7 @@ <plugin> <groupId>com.helger.maven</groupId> <artifactId>ph-javacc-maven-plugin</artifactId> - <version>4.1.2</version> + <version>4.1.5</version> <executions> <execution> <phase>generate-sources</phase> diff --git a/renovate.json b/renovate.json index 3c8eace23da..6b37e2049cb 100644 --- a/renovate.json +++ b/renovate.json @@ -3,6 +3,7 @@ "extends": [ "config:base" ], + "dependencyDashboardApproval": true, "transitiveRemediation": true, "prHourlyLimit": 10, "prConcurrentLimit": 10, diff --git a/screwdriver.yaml b/screwdriver.yaml index 10ae44aaefb..79a1569633f 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -10,6 +10,7 @@ shared: environment: USER_SHELL_BIN: bash annotations: + screwdriver.cd/restrictPR: fork restore-cache: &restore-cache restore-cache: | (cd /tmp && if [[ -f $MAIN_CACHE_FILE ]]; then tar xf $MAIN_CACHE_FILE; fi) diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json index 7d6f2f8790c..a8e028ff6ad 100644 --- a/searchlib/abi-spec.json +++ b/searchlib/abi-spec.json @@ -822,46 +822,43 @@ "abstract" ], "methods" : [ - "public void setTabSize(int)", - "public int getTabSize()", - "protected void expandBuff(boolean)", + "public void <init>(int, int, int)", + "public final void reInit(int, int, int)", "protected abstract int streamRead(char[], int, int)", "protected abstract void streamClose()", + "protected int getBufSizeAfterExpansion()", + "protected void expandBuff(boolean)", + "protected final void internalAdjustBuffSize()", "protected void fillBuff()", - "public char beginToken()", - "protected void updateLineColumn(char)", + "protected final void internalSetBufLineColumn(int, int)", + "protected final void internalUpdateLineColumn(char)", "public char readChar()", + "public char beginToken()", "public int getBeginColumn()", "public int getBeginLine()", "public int getEndColumn()", "public int getEndLine()", "public void backup(int)", - "public void <init>(int, int, int)", - "public void reInit(int, int, int)", "public java.lang.String getImage()", "public char[] getSuffix(int)", "public void done()", - "public void adjustBeginLineColumn(int, int)", - "public void setTrackLineColumn(boolean)", - "public boolean isTrackLineColumn()" + "public final int getTabSize()", + "public final void setTabSize(int)", + "public final void adjustBeginLineColumn(int, int)", + "protected final int getLine()", + "protected final int getColumn()", + "public final boolean isTrackLineColumn()", + "public final void setTrackLineColumn(boolean)" ], "fields" : [ "public static final int DEFAULT_BUF_SIZE", - "protected int bufpos", + "protected char[] buffer", "protected int bufsize", + "protected int bufpos", "protected int available", "protected int tokenBegin", - "protected int[] bufline", - "protected int[] bufcolumn", - "protected int column", - "protected int line", - "protected boolean prevCharIsCR", - "protected boolean prevCharIsLF", - "protected char[] buffer", - "protected int maxNextCharInd", "protected int inBuf", - "protected char[] nextCharBuf", - "protected int nextCharInd" + "protected int maxNextCharInd" ] }, "com.yahoo.searchlib.rankingexpression.parser.CharStream" : { @@ -883,10 +880,10 @@ "public abstract java.lang.String getImage()", "public abstract char[] getSuffix(int)", "public abstract void done()", - "public abstract void setTabSize(int)", "public abstract int getTabSize()", - "public abstract void setTrackLineColumn(boolean)", - "public abstract boolean isTrackLineColumn()" + "public abstract void setTabSize(int)", + "public abstract boolean isTrackLineColumn()", + "public abstract void setTrackLineColumn(boolean)" ], "fields" : [ ] }, @@ -989,9 +986,7 @@ "public final com.yahoo.tensor.functions.Slice$DimensionValue dimensionValue(java.util.Optional)", "public final java.lang.String label()", "public final java.lang.String string()", - "public void <init>(java.io.InputStream)", "public void <init>(java.io.InputStream, java.lang.String)", - "public void ReInit(java.io.InputStream)", "public void ReInit(java.io.InputStream, java.lang.String)", "public void <init>(java.io.Reader)", "public void ReInit(java.io.Reader)", @@ -1156,26 +1151,22 @@ "public" ], "methods" : [ - "protected int streamRead(char[], int, int)", - "protected void streamClose()", - "protected void fillBuff()", - "public char readChar()", "public void <init>(java.io.Reader, int, int, int)", "public void <init>(java.io.Reader, int, int)", "public void <init>(java.io.Reader)", - "public void reInit(java.io.Reader)", - "public void reInit(java.io.Reader, int, int)", "public void reInit(java.io.Reader, int, int, int)", + "public void reInit(java.io.Reader, int, int)", + "public void reInit(java.io.Reader)", "public void <init>(java.io.InputStream, java.lang.String, int, int, int)", "public void <init>(java.io.InputStream, java.lang.String, int, int)", "public void <init>(java.io.InputStream, java.lang.String)", "public void reInit(java.io.InputStream, java.lang.String)", "public void reInit(java.io.InputStream, java.lang.String, int, int)", - "public void reInit(java.io.InputStream, java.lang.String, int, int, int)" + "public void reInit(java.io.InputStream, java.lang.String, int, int, int)", + "protected int streamRead(char[], int, int)", + "protected void streamClose()" ], - "fields" : [ - "protected java.io.Reader inputStream" - ] + "fields" : [ ] }, "com.yahoo.searchlib.rankingexpression.parser.Token" : { "superClass" : "java.lang.Object", diff --git a/searchlib/src/tests/diskindex/diskindex/diskindex_test.cpp b/searchlib/src/tests/diskindex/diskindex/diskindex_test.cpp index d153481ef36..f87096aa1e3 100644 --- a/searchlib/src/tests/diskindex/diskindex/diskindex_test.cpp +++ b/searchlib/src/tests/diskindex/diskindex/diskindex_test.cpp @@ -228,9 +228,8 @@ DiskIndexTest::requireThatWeCanReadPostingList() { // field 'f1' LookupResult::UP r = _index->lookup(0, "w1"); PostingListHandle::UP h = _index->readPostingList(*r); - SearchIterator * sb = h->createIterator(r->counts, mda); + auto sb = h->createIterator(r->counts, mda); EXPECT_EQ(SimpleResult({1,3}), SimpleResult().search(*sb)); - delete sb; } } diff --git a/searchlib/src/tests/queryeval/filter_search/filter_search_test.cpp b/searchlib/src/tests/queryeval/filter_search/filter_search_test.cpp index ea4753ab847..8f2f8f2e96b 100644 --- a/searchlib/src/tests/queryeval/filter_search/filter_search_test.cpp +++ b/searchlib/src/tests/queryeval/filter_search/filter_search_test.cpp @@ -273,7 +273,9 @@ struct WeightedSetTermAdapter { WeightedSetTermAdapter(); ~WeightedSetTermAdapter(); void addChild(std::unique_ptr<Blueprint> child) { - blueprint.addTerm(std::move(child), 100); + Blueprint::HitEstimate estimate = blueprint.getState().estimate(); + blueprint.addTerm(std::move(child), 100, estimate); + blueprint.complete(estimate); } auto createFilterSearch(bool strict, Constraint constraint) const { return blueprint.createFilterSearch(strict, constraint); @@ -292,7 +294,9 @@ struct DotProductAdapter { void addChild(std::unique_ptr<Blueprint> child) { auto child_field = blueprint.getNextChildField(field); auto term = std::make_unique<LeafProxy>(child_field, std::move(child)); - blueprint.addTerm(std::move(term), 100); + Blueprint::HitEstimate estimate = blueprint.getState().estimate(); + blueprint.addTerm(std::move(term), 100, estimate); + blueprint.complete(estimate); } auto createFilterSearch(bool strict, Constraint constraint) const { return blueprint.createFilterSearch(strict, constraint); @@ -310,7 +314,9 @@ struct ParallelWeakAndAdapter { void addChild(std::unique_ptr<Blueprint> child) { auto child_field = blueprint.getNextChildField(field); auto term = std::make_unique<LeafProxy>(child_field, std::move(child)); - blueprint.addTerm(std::move(term), 100); + Blueprint::HitEstimate estimate = blueprint.getState().estimate(); + blueprint.addTerm(std::move(term), 100, estimate); + blueprint.complete(estimate); } auto createFilterSearch(bool strict, Constraint constraint) const { return blueprint.createFilterSearch(strict, constraint); diff --git a/searchlib/src/tests/queryeval/weighted_set_term/weighted_set_term_test.cpp b/searchlib/src/tests/queryeval/weighted_set_term/weighted_set_term_test.cpp index 90e16d4feff..f93aa537625 100644 --- a/searchlib/src/tests/queryeval/weighted_set_term/weighted_set_term_test.cpp +++ b/searchlib/src/tests/queryeval/weighted_set_term/weighted_set_term_test.cpp @@ -312,9 +312,11 @@ TEST("require that children get a common (yet separate) term field match data") auto top_handle = layout.allocTermField(42); FieldSpec top_spec("foo", 42, top_handle); WeightedSetTermBlueprint blueprint(top_spec); + queryeval::Blueprint::HitEstimate estimate; for (size_t i = 0; i < 5; ++i) { - blueprint.addTerm(vmd.create(blueprint.getNextChildField(top_spec)), 1); + blueprint.addTerm(vmd.create(blueprint.getNextChildField(top_spec)), 1, estimate); } + blueprint.complete(estimate); auto match_data = layout.createMatchData(); auto search = blueprint.createSearch(*match_data, true); auto top_tfmd = match_data->resolveTermField(top_handle); diff --git a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp index 532d645524b..152fcef5e8b 100644 --- a/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attribute_blueprint_factory.cpp @@ -412,7 +412,6 @@ template <typename SearchType> class DirectWeightedSetBlueprint : public ComplexLeafBlueprint { private: - HitEstimate _estimate; std::vector<int32_t> _weights; std::vector<IDocumentWeightAttribute::LookupResult> _terms; const IAttributeVector &_iattr; @@ -422,7 +421,6 @@ private: public: DirectWeightedSetBlueprint(const FieldSpec &field, const IAttributeVector &iattr, const IDocumentWeightAttribute &attr, size_t size_hint) : ComplexLeafBlueprint(field), - _estimate(), _weights(), _terms(), _iattr(iattr), @@ -435,20 +433,22 @@ public: } ~DirectWeightedSetBlueprint() override; - void addTerm(const IDocumentWeightAttribute::LookupKey & key, int32_t weight) { + void addTerm(const IDocumentWeightAttribute::LookupKey & key, int32_t weight, HitEstimate & estimate) { IDocumentWeightAttribute::LookupResult result = _attr.lookup(key, _dictionary_snapshot); HitEstimate childEst(result.posting_size, (result.posting_size == 0)); if (!childEst.empty) { - if (_estimate.empty) { - _estimate = childEst; + if (estimate.empty) { + estimate = childEst; } else { - _estimate.estHits += childEst.estHits; + estimate.estHits += childEst.estHits; } - setEstimate(_estimate); _weights.push_back(weight); _terms.push_back(result); } } + void complete(HitEstimate estimate) { + setEstimate(estimate); + } SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool) const override; @@ -506,7 +506,6 @@ DirectWeightedSetBlueprint<SearchType>::createFilterSearch(bool, FilterConstrain class DirectWandBlueprint : public queryeval::ComplexLeafBlueprint { private: - HitEstimate _estimate; mutable queryeval::SharedWeakAndPriorityQueue _scores; const queryeval::wand::score_t _scoreThreshold; double _thresholdBoostFactor; @@ -520,7 +519,6 @@ public: DirectWandBlueprint(const FieldSpec &field, const IDocumentWeightAttribute &attr, uint32_t scoresToTrack, queryeval::wand::score_t scoreThreshold, double thresholdBoostFactor, size_t size_hint) : ComplexLeafBlueprint(field), - _estimate(), _scores(scoresToTrack), _scoreThreshold(scoreThreshold), _thresholdBoostFactor(thresholdBoostFactor), @@ -536,20 +534,22 @@ public: ~DirectWandBlueprint() override; - void addTerm(const IDocumentWeightAttribute::LookupKey & key, int32_t weight) { + void addTerm(const IDocumentWeightAttribute::LookupKey & key, int32_t weight, HitEstimate & estimate) { IDocumentWeightAttribute::LookupResult result = _attr.lookup(key, _dictionary_snapshot); HitEstimate childEst(result.posting_size, (result.posting_size == 0)); if (!childEst.empty) { - if (_estimate.empty) { - _estimate = childEst; + if (estimate.empty) { + estimate = childEst; } else { - _estimate.estHits += childEst.estHits; + estimate.estHits += childEst.estHits; } - setEstimate(_estimate); _weights.push_back(weight); _terms.push_back(result); } } + void complete(HitEstimate estimate) { + setEstimate(estimate); + } SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool strict) const override { assert(tfmda.size() == 1); @@ -857,9 +857,11 @@ template <typename WS> void CreateBlueprintVisitor::createDirectWeightedSet(WS *bp, MultiTerm &n) { Blueprint::UP result(bp); + Blueprint::HitEstimate estimate; for (uint32_t i(0); i < n.getNumTerms(); i++) { - bp->addTerm(LookupKey(n, i), n.weight(i).percent()); + bp->addTerm(LookupKey(n, i), n.weight(i).percent(), estimate); } + bp->complete(estimate); setResult(std::move(result)); } @@ -869,11 +871,13 @@ CreateBlueprintVisitor::createShallowWeightedSet(WS *bp, MultiTerm &n, const Fie Blueprint::UP result(bp); SearchContextParams scParams = createContextParams(); bp->reserve(n.getNumTerms()); + Blueprint::HitEstimate estimate; for (uint32_t i(0); i < n.getNumTerms(); i++) { FieldSpec childfs = bp->getNextChildField(fs); auto term = n.getAsString(i); - bp->addTerm(std::make_unique<AttributeFieldBlueprint>(childfs, _attr, extractTerm(term.first, isInteger), scParams.useBitVector(childfs.isFilter())), term.second.percent()); + bp->addTerm(std::make_unique<AttributeFieldBlueprint>(childfs, _attr, extractTerm(term.first, isInteger), scParams.useBitVector(childfs.isFilter())), term.second.percent(), estimate); } + bp->complete(estimate); setResult(std::move(result)); } diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.cpp index 4ed72c2f8c3..8a987c49544 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.cpp @@ -53,7 +53,7 @@ ZcPosOccRandRead::~ZcPosOccRandRead() } -search::queryeval::SearchIterator * +std::unique_ptr<search::queryeval::SearchIterator> ZcPosOccRandRead:: createIterator(const PostingListCounts &counts, const PostingListHandle &handle, @@ -67,7 +67,7 @@ createIterator(const PostingListCounts &counts, assert(handle._bitOffsetMem <= handle._bitOffset); if (handle._bitLength == 0) { - return new search::queryeval::EmptySearch; + return std::make_unique<search::queryeval::EmptySearch>(); } const char *cmem = static_cast<const char *>(handle._mem); @@ -80,7 +80,7 @@ createIterator(const PostingListCounts &counts, handle._bitOffsetMem) & 63; Position start(mem, bitOffset); - return create_zc_posocc_iterator(true, counts, start, handle._bitLength, _posting_params, _fieldsParams, matchData).release(); + return create_zc_posocc_iterator(true, counts, start, handle._bitLength, _posting_params, _fieldsParams, matchData); } diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.h b/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.h index f23af15f72d..db7806beadd 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.h +++ b/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.h @@ -33,7 +33,7 @@ public: * Create iterator for single word. Semantic lifetime of counts and * handle must exceed lifetime of iterator. */ - queryeval::SearchIterator * + std::unique_ptr<queryeval::SearchIterator> createIterator(const PostingListCounts &counts, const PostingListHandle &handle, const fef::TermFieldMatchDataArray &matchData, bool usebitVector) const override; diff --git a/searchlib/src/vespa/searchlib/index/postinglistfile.cpp b/searchlib/src/vespa/searchlib/index/postinglistfile.cpp index acb1d40e353..7bb724f0fe6 100644 --- a/searchlib/src/vespa/searchlib/index/postinglistfile.cpp +++ b/searchlib/src/vespa/searchlib/index/postinglistfile.cpp @@ -3,6 +3,7 @@ #include "postinglistfile.h" #include "postinglistparams.h" #include <vespa/fastos/file.h> +#include <vespa/searchlib/queryeval/searchiterator.h> namespace search::index { @@ -94,7 +95,7 @@ PostingListFileRandReadPassThrough::~PostingListFileRandReadPassThrough() } } -search::queryeval::SearchIterator * +std::unique_ptr<search::queryeval::SearchIterator> PostingListFileRandReadPassThrough:: createIterator(const PostingListCounts &counts, const PostingListHandle &handle, diff --git a/searchlib/src/vespa/searchlib/index/postinglistfile.h b/searchlib/src/vespa/searchlib/index/postinglistfile.h index 17bfb47b812..93d0dd362f7 100644 --- a/searchlib/src/vespa/searchlib/index/postinglistfile.h +++ b/searchlib/src/vespa/searchlib/index/postinglistfile.h @@ -147,7 +147,7 @@ public: * didn't cover the whole word, probably need access to higher level * API above caches. */ - virtual search::queryeval::SearchIterator * + virtual std::unique_ptr<search::queryeval::SearchIterator> createIterator(const PostingListCounts &counts, const PostingListHandle &handle, const search::fef::TermFieldMatchDataArray &matchData, @@ -194,7 +194,7 @@ public: PostingListFileRandReadPassThrough(PostingListFileRandRead *lower, bool ownLower); ~PostingListFileRandReadPassThrough(); - search::queryeval::SearchIterator * + std::unique_ptr<search::queryeval::SearchIterator> createIterator(const PostingListCounts &counts, const PostingListHandle &handle, const search::fef::TermFieldMatchDataArray &matchData, diff --git a/searchlib/src/vespa/searchlib/index/postinglisthandle.cpp b/searchlib/src/vespa/searchlib/index/postinglisthandle.cpp index 82737531d69..c8cccd89207 100644 --- a/searchlib/src/vespa/searchlib/index/postinglisthandle.cpp +++ b/searchlib/src/vespa/searchlib/index/postinglisthandle.cpp @@ -2,10 +2,11 @@ #include "postinglisthandle.h" #include "postinglistfile.h" +#include <vespa/searchlib/queryeval/searchiterator.h> namespace search::index { -search::queryeval::SearchIterator * +std::unique_ptr<search::queryeval::SearchIterator> PostingListHandle::createIterator(const PostingListCounts &counts, const search::fef::TermFieldMatchDataArray &matchData, bool useBitVector) const diff --git a/searchlib/src/vespa/searchlib/index/postinglisthandle.h b/searchlib/src/vespa/searchlib/index/postinglisthandle.h index 9a4ec212636..1f3a72a876f 100644 --- a/searchlib/src/vespa/searchlib/index/postinglisthandle.h +++ b/searchlib/src/vespa/searchlib/index/postinglisthandle.h @@ -61,7 +61,7 @@ public: * didn't cover the whole word, probably need access to higher level * API above caches. */ - search::queryeval::SearchIterator * + std::unique_ptr<search::queryeval::SearchIterator> createIterator(const PostingListCounts &counts, const search::fef::TermFieldMatchDataArray &matchData, bool useBitVector=false) const; diff --git a/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.cpp b/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.cpp index a2d244250cf..bb44eaa0f3d 100644 --- a/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.cpp @@ -76,12 +76,14 @@ template <typename WS, typename NODE> void CreateBlueprintVisitorHelper::createWeightedSet(std::unique_ptr<WS> bp, NODE &n) { bp->reserve(n.getNumTerms()); + Blueprint::HitEstimate estimate; for (size_t i = 0; i < n.getNumTerms(); ++i) { auto term = n.getAsString(i); query::SimpleStringTerm node(term.first, n.getView(), 0, term.second); // TODO Temporary FieldSpec field = bp->getNextChildField(_field); - bp->addTerm(_searchable.createBlueprint(_requestContext, field, node), term.second.percent()); + bp->addTerm(_searchable.createBlueprint(_requestContext, field, node), term.second.percent(), estimate); } + bp->complete(estimate); setResult(std::move(bp)); } void diff --git a/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp index de5bdc33e3c..3e85ae4d00a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.cpp @@ -9,12 +9,10 @@ namespace search::queryeval { DotProductBlueprint::DotProductBlueprint(const FieldSpec &field) : ComplexLeafBlueprint(field), - _estimate(), _layout(), _weights(), _terms() -{ -} +{ } DotProductBlueprint::~DotProductBlueprint() = default; @@ -32,16 +30,15 @@ DotProductBlueprint::reserve(size_t num_children) { } void -DotProductBlueprint::addTerm(Blueprint::UP term, int32_t weight) +DotProductBlueprint::addTerm(Blueprint::UP term, int32_t weight, HitEstimate & estimate) { HitEstimate childEst = term->getState().estimate(); if (! childEst.empty) { - if (_estimate.empty) { - _estimate = childEst; + if (estimate.empty) { + estimate = childEst; } else { - _estimate.estHits += childEst.estHits; + estimate.estHits += childEst.estHits; } - setEstimate(_estimate); } _weights.push_back(weight); _terms.push_back(std::move(term)); diff --git a/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.h index 2975958b5af..18770691350 100644 --- a/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/dot_product_blueprint.h @@ -11,7 +11,6 @@ namespace search::queryeval { class DotProductBlueprint : public ComplexLeafBlueprint { - HitEstimate _estimate; fef::MatchDataLayout _layout; std::vector<int32_t> _weights; std::vector<Blueprint::UP> _terms; @@ -27,7 +26,10 @@ public: // used by create visitor void reserve(size_t num_children); - void addTerm(Blueprint::UP term, int32_t weight); + void addTerm(Blueprint::UP term, int32_t weight, HitEstimate & estimate); + void complete(HitEstimate estimate) { + setEstimate(estimate); + } SearchIteratorUP createLeafSearch(const search::fef::TermFieldMatchDataArray &tfmda, bool strict) const override; SearchIteratorUP createFilterSearch(bool strict, FilterConstraint constraint) const override; diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp index b4b55098eaa..e303e0b16d9 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.cpp @@ -22,7 +22,6 @@ ParallelWeakAndBlueprint::ParallelWeakAndBlueprint(const FieldSpec &field, _scoreThreshold(scoreThreshold), _thresholdBoostFactor(thresholdBoostFactor), _scoresAdjustFrequency(DEFAULT_PARALLEL_WAND_SCORES_ADJUST_FREQUENCY), - _estimate(), _layout(), _weights(), _terms() @@ -40,7 +39,6 @@ ParallelWeakAndBlueprint::ParallelWeakAndBlueprint(const FieldSpec &field, _scoreThreshold(scoreThreshold), _thresholdBoostFactor(thresholdBoostFactor), _scoresAdjustFrequency(scoresAdjustFrequency), - _estimate(), _layout(), _weights(), _terms() @@ -62,20 +60,18 @@ ParallelWeakAndBlueprint::reserve(size_t num_children) { } void -ParallelWeakAndBlueprint::addTerm(Blueprint::UP term, int32_t weight) +ParallelWeakAndBlueprint::addTerm(Blueprint::UP term, int32_t weight, HitEstimate & estimate) { HitEstimate childEst = term->getState().estimate(); if (!childEst.empty) { - if (_estimate.empty) { - _estimate = childEst; + if (estimate.empty) { + estimate = childEst; } else { - _estimate.estHits += childEst.estHits; + estimate.estHits += childEst.estHits; } - setEstimate(_estimate); } _weights.push_back(weight); _terms.push_back(std::move(term)); - set_tree_size(_terms.size() + 1); } SearchIterator::UP diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.h index a2c13f12485..cb4d44f4497 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_blueprint.h @@ -26,7 +26,6 @@ private: const wand::score_t _scoreThreshold; double _thresholdBoostFactor; const uint32_t _scoresAdjustFrequency; - HitEstimate _estimate; fef::MatchDataLayout _layout; std::vector<int32_t> _weights; std::vector<Blueprint::UP> _terms; @@ -57,7 +56,11 @@ public: // Used by create visitor void reserve(size_t num_children); - void addTerm(Blueprint::UP term, int32_t weight); + void addTerm(Blueprint::UP term, int32_t weight, HitEstimate & estimate); + void complete(HitEstimate estimate) { + setEstimate(estimate); + set_tree_size(_terms.size() + 1); + } SearchIterator::UP createLeafSearch(const fef::TermFieldMatchDataArray &tfmda, bool strict) const override; std::unique_ptr<SearchIterator> createFilterSearch(bool strict, FilterConstraint constraint) const override; diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.cpp b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.cpp index 1a7e91b2d1a..8540752e320 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.cpp @@ -79,12 +79,12 @@ public: _localScores() { } - virtual size_t get_num_terms() const override { return _terms.size(); } - virtual int32_t get_term_weight(size_t idx) const override { return _terms.weight(idx); } - virtual score_t get_max_score(size_t idx) const override { return _terms.maxScore(idx); } - virtual const MatchParams &getMatchParams() const override { return _matchParams; } + size_t get_num_terms() const override { return _terms.size(); } + int32_t get_term_weight(size_t idx) const override { return _terms.weight(idx); } + score_t get_max_score(size_t idx) const override { return _terms.maxScore(idx); } + const MatchParams &getMatchParams() const override { return _matchParams; } - virtual void doSeek(uint32_t docid) override { + void doSeek(uint32_t docid) override { updateThreshold(_matchParams.scores.getMinScore()); if (IS_STRICT) { seek_strict(docid); @@ -92,7 +92,7 @@ public: seek_unstrict(docid); } } - virtual void doUnpack(uint32_t docid) override { + void doUnpack(uint32_t docid) override { score_t score = _algo.get_full_score(_terms, _heaps, DotProductScorer()); _localScores.push_back(score); if (_localScores.size() == _matchParams.scoresAdjustFrequency) { @@ -101,14 +101,14 @@ public: } _tfmd.setRawScore(docid, score); } - virtual void visitMembers(vespalib::ObjectVisitor &visitor) const override { + void visitMembers(vespalib::ObjectVisitor &visitor) const override { _terms.visit_members(visitor); } void initRange(uint32_t begin, uint32_t end) override { ParallelWeakAndSearch::initRange(begin, end); _algo.init_range(_terms, _heaps, begin, end); } - Trinary is_strict() const override { return IS_STRICT ? Trinary::True : Trinary::False; } + Trinary is_strict() const final { return IS_STRICT ? Trinary::True : Trinary::False; } }; namespace { diff --git a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp index ee55a89dcdc..4e06f170253 100644 --- a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.cpp @@ -62,7 +62,6 @@ WeightedSetTermMatchingElementsSearch::initRange(uint32_t begin_id, uint32_t end WeightedSetTermBlueprint::WeightedSetTermBlueprint(const FieldSpec &field) : ComplexLeafBlueprint(field), - _estimate(), _layout(), _children_field(field.getName(), field.getFieldId(), _layout.allocTermField(field.getFieldId()), field.isFilter()), _weights(), @@ -81,16 +80,15 @@ WeightedSetTermBlueprint::reserve(size_t num_children) { } void -WeightedSetTermBlueprint::addTerm(Blueprint::UP term, int32_t weight) +WeightedSetTermBlueprint::addTerm(Blueprint::UP term, int32_t weight, HitEstimate & estimate) { HitEstimate childEst = term->getState().estimate(); if (! childEst.empty) { - if (_estimate.empty) { - _estimate = childEst; + if (estimate.empty) { + estimate = childEst; } else { - _estimate.estHits += childEst.estHits; + estimate.estHits += childEst.estHits; } - setEstimate(_estimate); } _weights.push_back(weight); _terms.push_back(std::move(term)); diff --git a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.h index 3827dc8a35f..b40ab421890 100644 --- a/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/weighted_set_term_blueprint.h @@ -12,7 +12,6 @@ namespace search::queryeval { class WeightedSetTermBlueprint : public ComplexLeafBlueprint { - HitEstimate _estimate; fef::MatchDataLayout _layout; FieldSpec _children_field; std::vector<int32_t> _weights; @@ -31,7 +30,10 @@ public: // used by create visitor void reserve(size_t num_children); - void addTerm(Blueprint::UP term, int32_t weight); + void addTerm(Blueprint::UP term, int32_t weight, HitEstimate & estimate); + void complete(HitEstimate estimate) { + setEstimate(estimate); + } SearchIteratorUP createLeafSearch(const fef::TermFieldMatchDataArray &tfmda, bool strict) const override; SearchIteratorUP createFilterSearch(bool strict, FilterConstraint constraint) const override; diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fake_match_loop.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fake_match_loop.cpp index 95f07bc3191..bb55593f8e3 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fake_match_loop.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fake_match_loop.cpp @@ -32,7 +32,7 @@ public: _tfmda.add(&_md); _md.setNeedNormalFeatures(posting.enable_unpack_normal_features()); _md.setNeedInterleavedFeatures(posting.enable_unpack_interleaved_features()); - _itr.reset(posting.createIterator(_tfmda)); + _itr = posting.createIterator(_tfmda); } ~IteratorState() {} diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp index 9521bed7827..358008f389a 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp @@ -717,23 +717,16 @@ FakeFilterOccEGCompressed64ArrayIterator<bigEndian>::doUnpack(uint32_t docId) } -search::queryeval::SearchIterator * +std::unique_ptr<search::queryeval::SearchIterator> FakeEGCompr64FilterOcc:: createIterator(const fef::TermFieldMatchDataArray &matchData) const { const uint64_t *arr = _compressed.first; - if (_bigEndian) - return new FakeFilterOccEGCompressed64ArrayIterator<true>(arr, - 0, - _hitDocs, - _lastDocId, - matchData); - else - return new FakeFilterOccEGCompressed64ArrayIterator<false>(arr, - 0, - _hitDocs, - _lastDocId, - matchData); + if (_bigEndian) { + return std::make_unique<FakeFilterOccEGCompressed64ArrayIterator<true>>(arr, 0, _hitDocs, _lastDocId, matchData); + } else { + return std::make_unique<FakeFilterOccEGCompressed64ArrayIterator<false>>(arr, 0, _hitDocs, _lastDocId, matchData); + } } @@ -766,7 +759,7 @@ class FakeEGCompr64SkipFilterOcc : public FakeEGCompr64FilterOcc public: FakeEGCompr64SkipFilterOcc(const FakeWord &fw); ~FakeEGCompr64SkipFilterOcc(); - search::queryeval::SearchIterator *createIterator(const fef::TermFieldMatchDataArray &matchData) const override; + std::unique_ptr<search::queryeval::SearchIterator> createIterator(const fef::TermFieldMatchDataArray &matchData) const override; }; @@ -1451,7 +1444,7 @@ FakeFilterOccEGCompressed64SkipArrayIterator<doSkip>::doUnpack(uint32_t docId) template <bool doSkip> -search::queryeval::SearchIterator * +std::unique_ptr<search::queryeval::SearchIterator> FakeEGCompr64SkipFilterOcc<doSkip>:: createIterator(const fef::TermFieldMatchDataArray &matchData) const { @@ -1478,15 +1471,16 @@ createIterator(const fef::TermFieldMatchDataArray &matchData) const const uint64_t *l2SkipArr = _l2SkipCompressed.first; const uint64_t *l3SkipArr = _l3SkipCompressed.first; const uint64_t *l4SkipArr = _l4SkipCompressed.first; - return new FakeFilterOccEGCompressed64SkipArrayIterator<doSkip>(docIdBits.getCompr(), - docIdBits.getBitOffset(), - _lastDocId, - l1SkipArr, 0, - l2SkipArr, 0, - l3SkipArr, 0, - l4SkipArr, 0, - getName(), - matchData); + return std::make_unique<FakeFilterOccEGCompressed64SkipArrayIterator<doSkip>> + (docIdBits.getCompr(), + docIdBits.getBitOffset(), + _lastDocId, + l1SkipArr, 0, + l2SkipArr, 0, + l3SkipArr, 0, + l4SkipArr, 0, + getName(), + matchData); } } diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.h b/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.h index 6e398a2f0b0..2ef91c70921 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.h @@ -58,7 +58,7 @@ public: int lowLevelSinglePostingScanUnpack() const override; int lowLevelAndPairPostingScan(const FakePosting &rhs) const override; int lowLevelAndPairPostingScanUnpack(const FakePosting &rhs) const override; - queryeval::SearchIterator *createIterator(const fef::TermFieldMatchDataArray &matchData) const override; + std::unique_ptr<queryeval::SearchIterator> createIterator(const fef::TermFieldMatchDataArray &matchData) const override; }; } diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakefilterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakefilterocc.cpp index a412a779006..54710a85a04 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakefilterocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakefilterocc.cpp @@ -176,13 +176,11 @@ FakeFilterOccArrayIterator::doUnpack(uint32_t docId) } -search::queryeval::SearchIterator * +std::unique_ptr<search::queryeval::SearchIterator> FakeFilterOcc:: createIterator(const fef::TermFieldMatchDataArray &matchData) const { - return new FakeFilterOccArrayIterator(&*_uncompressed.begin(), - &*_uncompressed.end(), - matchData); + return std::make_unique<FakeFilterOccArrayIterator>(&*_uncompressed.begin(), &*_uncompressed.end(), matchData); } } diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakefilterocc.h b/searchlib/src/vespa/searchlib/test/fakedata/fakefilterocc.h index ed0855cfb87..c05dc9db342 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakefilterocc.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakefilterocc.h @@ -30,7 +30,7 @@ public: int lowLevelSinglePostingScanUnpack() const override; int lowLevelAndPairPostingScan(const FakePosting &rhs) const override; int lowLevelAndPairPostingScanUnpack(const FakePosting &rhs) const override; - queryeval::SearchIterator * createIterator(const fef::TermFieldMatchDataArray &matchData) const override; + std::unique_ptr<queryeval::SearchIterator> createIterator(const fef::TermFieldMatchDataArray &matchData) const override; }; } // namespace fakedata diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp index 99d0fb3b3f1..48820b58a7c 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp @@ -129,14 +129,14 @@ lowLevelAndPairPostingScanUnpack(const FakePosting &rhs) const } -search::queryeval::SearchIterator * +std::unique_ptr<search::queryeval::SearchIterator> FakeMemTreeOcc:: createIterator(const fef::TermFieldMatchDataArray &matchData) const { return memoryindex::make_search_iterator<false>(_tree.begin(_allocator), _mgr._featureStore, _packedIndex, - matchData).release(); + matchData); } diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h index ea8699d94b2..28698e29cf9 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h @@ -176,7 +176,7 @@ public: int lowLevelSinglePostingScanUnpack() const override; int lowLevelAndPairPostingScan(const FakePosting &rhs) const override; int lowLevelAndPairPostingScanUnpack(const FakePosting &rhs) const override; - queryeval::SearchIterator *createIterator(const fef::TermFieldMatchDataArray &matchData) const override; + std::unique_ptr<queryeval::SearchIterator> createIterator(const fef::TermFieldMatchDataArray &matchData) const override; }; } diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeposting.h b/searchlib/src/vespa/searchlib/test/fakedata/fakeposting.h index 7e32fcc31ad..56a54b2cf85 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeposting.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeposting.h @@ -72,7 +72,7 @@ public: /* * Iterator factory, for current query evaluation framework. */ - virtual search::queryeval::SearchIterator * + virtual std::unique_ptr<search::queryeval::SearchIterator> createIterator(const fef::TermFieldMatchDataArray &matchData) const = 0; const std::string &getName() const diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakezcbfilterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakezcbfilterocc.cpp index dc6f546fad0..2f9714f1638 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakezcbfilterocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakezcbfilterocc.cpp @@ -237,14 +237,12 @@ FakeFilterOccZCBArrayIterator::doUnpack(uint32_t docId) } -search::queryeval::SearchIterator * +std::unique_ptr<search::queryeval::SearchIterator> FakeZcbFilterOcc:: createIterator(const fef::TermFieldMatchDataArray &matchData) const { const uint8_t *arr = &*_compressed.begin(); - return new FakeFilterOccZCBArrayIterator(arr, - _hitDocs, - matchData); + return std::make_unique<FakeFilterOccZCBArrayIterator>(arr, _hitDocs, matchData); } } diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakezcbfilterocc.h b/searchlib/src/vespa/searchlib/test/fakedata/fakezcbfilterocc.h index 87d25cb6761..599b9c83d76 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakezcbfilterocc.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakezcbfilterocc.h @@ -29,7 +29,7 @@ public: int lowLevelSinglePostingScanUnpack() const override; int lowLevelAndPairPostingScan(const FakePosting &rhs) const override; int lowLevelAndPairPostingScanUnpack(const FakePosting &rhs) const override; - queryeval::SearchIterator *createIterator(const fef::TermFieldMatchDataArray &matchData) const override; + std::unique_ptr<queryeval::SearchIterator> createIterator(const fef::TermFieldMatchDataArray &matchData) const override; }; } // namespace fakedata diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp index 809746a87e6..dc2791fa4f6 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp @@ -502,11 +502,11 @@ FakeFilterOccZCArrayIterator::doUnpack(uint32_t docId) } -SearchIterator * +std::unique_ptr<SearchIterator> FakeZcFilterOcc:: createIterator(const TermFieldMatchDataArray &matchData) const { - return new FakeFilterOccZCArrayIterator(_compressed.first, 0, _posting_params._doc_id_limit, matchData); + return std::make_unique<FakeFilterOccZCArrayIterator>(_compressed.first, 0, _posting_params._doc_id_limit, matchData); } class FakeZcSkipFilterOcc : public FakeZcFilterOcc @@ -516,7 +516,7 @@ public: FakeZcSkipFilterOcc(const FakeWord &fw); ~FakeZcSkipFilterOcc() override; - SearchIterator *createIterator(const TermFieldMatchDataArray &matchData) const override; + std::unique_ptr<SearchIterator> createIterator(const TermFieldMatchDataArray &matchData) const override; }; static FPFactoryInit @@ -534,10 +534,10 @@ FakeZcSkipFilterOcc::FakeZcSkipFilterOcc(const FakeWord &fw) FakeZcSkipFilterOcc::~FakeZcSkipFilterOcc() = default; -SearchIterator * +std::unique_ptr<SearchIterator> FakeZcSkipFilterOcc::createIterator(const TermFieldMatchDataArray &matchData) const { - return create_zc_posocc_iterator(true, _counts, Position(_compressed.first, 0), _compressedBits, _posting_params, _fieldsParams, matchData).release(); + return create_zc_posocc_iterator(true, _counts, Position(_compressed.first, 0), _compressedBits, _posting_params, _fieldsParams, matchData); } @@ -550,7 +550,7 @@ public: ~FakeEGCompr64PosOcc() override; size_t bitSize() const override; bool hasWordPositions() const override; - SearchIterator *createIterator(const TermFieldMatchDataArray &matchData) const override; + std::unique_ptr<SearchIterator> createIterator(const TermFieldMatchDataArray &matchData) const override; }; @@ -587,11 +587,11 @@ FakeEGCompr64PosOcc<bigEndian>::hasWordPositions() const template <bool bigEndian> -SearchIterator * +std::unique_ptr<SearchIterator> FakeEGCompr64PosOcc<bigEndian>:: createIterator(const TermFieldMatchDataArray &matchData) const { - return create_zc_posocc_iterator(bigEndian, _counts, Position(_compressed.first, 0), _compressedBits, _posting_params, _fieldsParams, matchData).release(); + return create_zc_posocc_iterator(bigEndian, _counts, Position(_compressed.first, 0), _compressedBits, _posting_params, _fieldsParams, matchData); } @@ -604,7 +604,7 @@ public: ~FakeEG2Compr64PosOcc() override; size_t bitSize() const override; bool hasWordPositions() const override; - SearchIterator *createIterator(const fef::TermFieldMatchDataArray &matchData) const override; + std::unique_ptr<SearchIterator> createIterator(const fef::TermFieldMatchDataArray &matchData) const override; }; @@ -642,11 +642,11 @@ FakeEG2Compr64PosOcc<bigEndian>::hasWordPositions() const template <bool bigEndian> -SearchIterator * +std::unique_ptr<SearchIterator> FakeEG2Compr64PosOcc<bigEndian>:: createIterator(const TermFieldMatchDataArray &matchData) const { - return create_zc_posocc_iterator(bigEndian, _counts, Position(_compressed.first, 0), _compressedBits, _posting_params, _fieldsParams, matchData).release(); + return create_zc_posocc_iterator(bigEndian, _counts, Position(_compressed.first, 0), _compressedBits, _posting_params, _fieldsParams, matchData); } @@ -660,7 +660,7 @@ public: size_t bitSize() const override; bool hasWordPositions() const override; - SearchIterator *createIterator(const TermFieldMatchDataArray &matchData) const override; + std::unique_ptr<SearchIterator> createIterator(const TermFieldMatchDataArray &matchData) const override; }; @@ -699,11 +699,11 @@ FakeZcSkipPosOcc<bigEndian>::hasWordPositions() const template <bool bigEndian> -SearchIterator * +std::unique_ptr<SearchIterator> FakeZcSkipPosOcc<bigEndian>:: createIterator(const TermFieldMatchDataArray &matchData) const { - return create_zc_posocc_iterator(bigEndian, _counts, Position(_compressed.first, 0), _compressedBits, _posting_params, _fieldsParams, matchData).release(); + return create_zc_posocc_iterator(bigEndian, _counts, Position(_compressed.first, 0), _compressedBits, _posting_params, _fieldsParams, matchData); } @@ -720,7 +720,7 @@ public: ~FakeZc4SkipPosOcc() override; size_t bitSize() const override; bool hasWordPositions() const override; - SearchIterator *createIterator(const TermFieldMatchDataArray &matchData) const override; + std::unique_ptr<SearchIterator> createIterator(const TermFieldMatchDataArray &matchData) const override; bool enable_unpack_normal_features() const override { return _unpack_normal_features; } bool enable_unpack_interleaved_features() const override { return _unpack_interleaved_features; } }; @@ -766,7 +766,7 @@ FakeZc4SkipPosOcc<bigEndian>::hasWordPositions() const template <bool bigEndian> -SearchIterator * +std::unique_ptr<SearchIterator> FakeZc4SkipPosOcc<bigEndian>:: createIterator(const TermFieldMatchDataArray &matchData) const { @@ -777,7 +777,7 @@ createIterator(const TermFieldMatchDataArray &matchData) const assert(!_unpack_normal_features); assert(!_unpack_interleaved_features); } -return create_zc_posocc_iterator(bigEndian, _counts, Position(_compressed.first, 0), _compressedBits, _posting_params, _fieldsParams, matchData).release(); + return create_zc_posocc_iterator(bigEndian, _counts, Position(_compressed.first, 0), _compressedBits, _posting_params, _fieldsParams, matchData); } template <bool bigEndian> diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.h b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.h index bcdd780e1e4..7d0670f993b 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.h @@ -65,7 +65,7 @@ public: int lowLevelSinglePostingScanUnpack() const override; int lowLevelAndPairPostingScan(const FakePosting &rhs) const override; int lowLevelAndPairPostingScanUnpack(const FakePosting &rhs) const override; - queryeval::SearchIterator *createIterator(const fef::TermFieldMatchDataArray &matchData) const override; + std::unique_ptr<queryeval::SearchIterator> createIterator(const fef::TermFieldMatchDataArray &matchData) const override; }; } diff --git a/vdslib/src/main/java/com/yahoo/vdslib/BucketDistribution.java b/vdslib/src/main/java/com/yahoo/vdslib/BucketDistribution.java index c49fdb93d20..ba17b947bb8 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/BucketDistribution.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/BucketDistribution.java @@ -4,6 +4,7 @@ package com.yahoo.vdslib; import com.yahoo.document.BucketId; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -14,16 +15,16 @@ import java.util.logging.Logger; public class BucketDistribution { // A logger object to enable proper logging. - private static Logger log = Logger.getLogger(BucketDistribution.class.getName()); + private static final Logger log = Logger.getLogger(BucketDistribution.class.getName()); // A map from bucket id to column index. - private int[] bucketToColumn; + private final int[] bucketToColumn; // The number of columns to distribute to. private int numColumns; // The number of bits to use for bucket identification. - private int numBucketBits; + private final int numBucketBits; /** * Constructs a new bucket distribution object with a given number of columns and buckets. @@ -68,7 +69,7 @@ public class BucketDistribution { * @return The bucket distribution. */ private static List<Integer> getBucketCount(int numColumns, int numBucketBits) { - List<Integer> ret = new ArrayList<Integer>(numColumns); + List<Integer> ret = new ArrayList<>(numColumns); int cnt = getNumBuckets(numBucketBits) / numColumns; int rst = getNumBuckets(numBucketBits) % numColumns; for (int i = 0; i < numColumns; ++i) { @@ -100,9 +101,7 @@ public class BucketDistribution { * that it all buckets point to that single column. */ public void reset() { - for (int i = 0; i < bucketToColumn.length; ++i) { - bucketToColumn[i] = 0; - } + Arrays.fill(bucketToColumn, 0); numColumns = 1; } @@ -152,32 +151,6 @@ public class BucketDistribution { } /** - * Sets the number of buckets to use for this document distribution object. This will reset and setup this object - * from scratch. The original number of columns is maintained. - * - * @param numBucketBits The new number of bits to use for bucket id. - */ - public synchronized void setNumBucketBits(int numBucketBits) { - if (numBucketBits == this.numBucketBits) { - return; - } - this.numBucketBits = numBucketBits; - bucketToColumn = new int[getNumBuckets(numBucketBits)]; - int numColumns = this.numColumns; - reset(); - setNumColumns(numColumns); - } - - /** - * Returns the number of bits used for bucket identifiers. - * - * @return The number of bits. - */ - public int getNumBucketBits() { - return numBucketBits; - } - - /** * Returns the number of buckets available using the configured number of bucket bits. * * @return The number of buckets. diff --git a/vdslib/src/main/java/com/yahoo/vdslib/DocumentSummary.java b/vdslib/src/main/java/com/yahoo/vdslib/DocumentSummary.java index ab5fe0d9a86..4371e19d090 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/DocumentSummary.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/DocumentSummary.java @@ -5,7 +5,8 @@ import com.yahoo.vespa.objects.BufferSerializer; import com.yahoo.vespa.objects.Deserializer; import java.nio.ByteOrder; -import java.io.UnsupportedEncodingException; + +import static java.nio.charset.StandardCharsets.UTF_8; public class DocumentSummary { @@ -27,23 +28,14 @@ public class DocumentSummary { int summarySize = buf.getInt(null); int end = start; while (cArr[end++] != 0); - try { - byte [] sb = new byte [summarySize]; - System.arraycopy(cArr, end, sb, 0, summarySize); - summaries[i] = new Summary(new String(cArr, start, end-start-1, "utf-8"), sb); - start = end + summarySize; - } catch (UnsupportedEncodingException e) { - throw new RuntimeException("UTF-8 apparently not supported"); - } + byte [] sb = new byte [summarySize]; + System.arraycopy(cArr, end, sb, 0, summarySize); + summaries[i] = new Summary(new String(cArr, start, end-start-1, UTF_8), sb); + start = end + summarySize; } } } - /** Constructs a new message from a byte buffer. */ - public DocumentSummary(byte[] buffer) { - this(BufferSerializer.wrap(buffer)); - } - final public int getSummaryCount() { return summaries.length; } final public Summary getSummary(int hitNo) { return summaries[hitNo]; } @@ -63,7 +55,6 @@ public class DocumentSummary { final public String getDocId() { return docId; } final public byte [] getSummary() { return summary; } - final public void setSummary(byte [] summary) { this.summary = summary; } public int compareTo(Summary s) { return getDocId().compareTo(s.getDocId()); diff --git a/vdslib/src/main/java/com/yahoo/vdslib/MetaEntry.java b/vdslib/src/main/java/com/yahoo/vdslib/MetaEntry.java deleted file mode 100644 index a05f746fe48..00000000000 --- a/vdslib/src/main/java/com/yahoo/vdslib/MetaEntry.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vdslib; - -import com.yahoo.io.GrowableByteBuffer; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -public class MetaEntry { - public static int REMOVE_ENTRY = 1; - public static int BODY_STRIPPED = 2; - public static int BODY_IN_HEADER = 4; - public static int UPDATE_ENTRY = 8; - public static int COMPRESSED = 16; - - public static int SIZE = 32; - - public long timestamp = 0; - public int headerPos = 0; - public int headerLen = 0; - public int bodyPos = 0; - public int bodyLen = 0; - public byte flags = 0; - - public MetaEntry() { - } - - public MetaEntry(byte[] buffer, int position) { - ByteBuffer buf = ByteBuffer.wrap(buffer, position, SIZE); - buf.order(ByteOrder.LITTLE_ENDIAN); - - timestamp = buf.getLong(); - headerPos = buf.getInt(); - headerLen = buf.getInt(); - bodyPos = buf.getInt(); - bodyLen = buf.getInt(); - flags = buf.get(); - } - - public void serialize(GrowableByteBuffer buf) { - ByteOrder originalOrder = buf.order(); - buf.order(ByteOrder.LITTLE_ENDIAN); - buf.putLong(timestamp); // 8 - buf.putInt(headerPos); // 12 - buf.putInt(headerLen); // 16 - buf.putInt(bodyPos); // 20 - buf.putInt(bodyLen); // 24 - buf.putInt(flags); // 28 (written as little-endian int, this is on purpose) - buf.putInt(0); // 32 - buf.order(originalOrder); - } -} diff --git a/vdslib/src/main/java/com/yahoo/vdslib/SearchResult.java b/vdslib/src/main/java/com/yahoo/vdslib/SearchResult.java index b7c9b1b71b5..c89abf87970 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/SearchResult.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/SearchResult.java @@ -1,19 +1,22 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vdslib; + import com.yahoo.data.access.helpers.MatchFeatureData; import com.yahoo.vespa.objects.BufferSerializer; import com.yahoo.vespa.objects.Deserializer; -import java.io.UnsupportedEncodingException; import java.nio.ByteOrder; import java.util.ArrayList; import java.util.Map; import java.util.Optional; import java.util.TreeMap; +import static java.nio.charset.StandardCharsets.UTF_8; + public class SearchResult { + public static class Hit implements Comparable<Hit> { - private String docId; + private final String docId; private double rank; private MatchFeatureData.HitValue matchFeatures; public Hit(Hit h) { @@ -40,7 +43,7 @@ public class SearchResult { } } public static class HitWithSortBlob extends Hit { - private byte [] sortBlob; + private final byte [] sortBlob; public HitWithSortBlob(Hit h, byte [] sb) { super(h); sortBlob = sb; @@ -57,12 +60,12 @@ public class SearchResult { return sortBlob.length - b.sortBlob.length; } } - private int totalHits; - private Hit[] hits; - private TreeMap<Integer, byte []> aggregatorList; - private TreeMap<Integer, byte []> groupingList; - private static int EXTENSION_FLAGS_PRESENT = -1; - private static int MATCH_FEATURES_PRESENT_MASK = 1; + private final int totalHits; + private final Hit[] hits; + private final TreeMap<Integer, byte []> aggregatorList; + private final TreeMap<Integer, byte []> groupingList; + private static final int EXTENSION_FLAGS_PRESENT = -1; + private static final int MATCH_FEATURES_PRESENT_MASK = 1; public SearchResult(Deserializer buf) { BufferSerializer bser = (BufferSerializer) buf; // TODO: dirty cast. must do this differently @@ -76,17 +79,13 @@ public class SearchResult { } hits = new Hit[numHits]; if (numHits != 0) { - int docIdBufferLength = buf.getInt(null); + int docIdBufferLength = buf.getInt(null); // Unused, but need to call getInt() to advance buffer byte[] cArr = bser.getBuf().array(); int start = bser.getBuf().arrayOffset() + bser.position(); for(int i=0; i < numHits; i++) { int end = start; while (cArr[end++] != 0); - try { - hits[i] = new Hit(new String(cArr, start, end-start-1, "utf-8"), 0); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException("UTF-8 apparently not supported"); - } + hits[i] = new Hit(new String(cArr, start, end-start-1, UTF_8), 0); start = end; } bser.position(start - bser.getBuf().arrayOffset()); @@ -104,8 +103,9 @@ public class SearchResult { hits[i] = new HitWithSortBlob(hits[i], buf.getBytes(null, size[i])); } + // Unused, but need to call getInt() to advance buffer int numAggregators = buf.getInt(null); - aggregatorList = new TreeMap<Integer, byte []>(); + aggregatorList = new TreeMap<>(); for (int i = 0; i < numAggregators; i++) { int aggrId = buf.getInt(null); int aggrLength = buf.getInt(null); @@ -113,7 +113,7 @@ public class SearchResult { } int numGroupings = buf.getInt(null); - groupingList = new TreeMap<Integer, byte []>(); + groupingList = new TreeMap<>(); for (int i = 0; i < numGroupings; i++) { int aggrId = buf.getInt(null); int aggrLength = buf.getInt(null); @@ -159,18 +159,8 @@ public class SearchResult { return featureType == 0; } - /** - * Constructs a new message from a byte buffer. - * - * @param buffer A byte buffer that contains a serialized message. - */ - public SearchResult(byte[] buffer) { - this(BufferSerializer.wrap(buffer)); - } - final public int getHitCount() { return hits.length; } final public int getTotalHitCount() { return (totalHits != 0) ? totalHits : getHitCount(); } final public Hit getHit(int hitNo) { return hits[hitNo]; } - final public Map<Integer, byte []> getAggregatorList() { return aggregatorList; } final public Map<Integer, byte []> getGroupingList() { return groupingList; } } diff --git a/vdslib/src/main/java/com/yahoo/vdslib/distribution/ConfiguredNode.java b/vdslib/src/main/java/com/yahoo/vdslib/distribution/ConfiguredNode.java index 92cf8b025e9..965dd018c4f 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/distribution/ConfiguredNode.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/distribution/ConfiguredNode.java @@ -7,31 +7,28 @@ package com.yahoo.vdslib.distribution; * * @author bratseth */ -public class ConfiguredNode implements Comparable<ConfiguredNode> { +public record ConfiguredNode(int index, boolean retired) implements Comparable<ConfiguredNode> { - private final int index; - - private final boolean retired; - - public ConfiguredNode(int index, boolean retired) { - this.index = index; - this.retired = retired; - } - - /** Return the index (distribution key) of this node */ - public int index() { return index; } + /** + * Return the index (distribution key) of this node + */ + @Override + public int index() {return index;} - /** Returns whether the node is configured to be retired */ - public boolean retired() { return retired; } + /** + * Returns whether the node is configured to be retired + */ + @Override + public boolean retired() {return retired;} @Override - public int hashCode() { return index; } + public int hashCode() {return index;} @Override public boolean equals(Object other) { if (other == this) return true; - if ( ! (other instanceof ConfiguredNode)) return false; - return ((ConfiguredNode)other).index == this.index; + if (! (other instanceof ConfiguredNode)) return false; + return ((ConfiguredNode) other).index == this.index; } @Override diff --git a/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java b/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java index a83e2a4f89c..bfa7e919514 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java @@ -26,14 +26,7 @@ import java.util.concurrent.atomic.AtomicReference; public class Distribution { - private static class Config { - Config(Group nodeGraph, int redundancy) { - this.nodeGraph = nodeGraph; - this.redundancy = redundancy; - } - - private final Group nodeGraph; - private final int redundancy; + private record Config(Group nodeGraph, int redundancy) { } private ConfigSubscriber configSub; @@ -197,8 +190,8 @@ public class Distribution { } private static class ScoredGroup implements Comparable<ScoredGroup> { - Group group; - double score; + final Group group; + final double score; ScoredGroup(Group g, double score) { this.group = g; this.score = score; } @@ -266,8 +259,8 @@ public class Distribution { } private static class ResultGroup implements Comparable<ResultGroup> { - Group group; - int redundancy; + final Group group; + final int redundancy; ResultGroup(Group group, int redundancy) { this.group = group; @@ -489,14 +482,11 @@ public class Distribution { public Set<ConfiguredNode> getNodes() { final Set<ConfiguredNode> nodes = new HashSet<>(); - GroupVisitor visitor = new GroupVisitor() { - @Override - public boolean visitGroup(Group g) { - if (g.isLeafGroup()) { - nodes.addAll(g.getNodes()); - } - return true; + GroupVisitor visitor = g -> { + if (g.isLeafGroup()) { + nodes.addAll(g.getNodes()); } + return true; }; visitGroups(visitor); return nodes; diff --git a/vdslib/src/main/java/com/yahoo/vdslib/distribution/Group.java b/vdslib/src/main/java/com/yahoo/vdslib/distribution/Group.java index b926ee3be8d..c1c2eef5c8f 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/distribution/Group.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/distribution/Group.java @@ -9,13 +9,13 @@ import java.text.ParseException; */ public class Group implements Comparable<Group> { - private String name; + private final String name; private Group parent = null; - private int index; + private final int index; private int distributionHash; - private Distribution distribution = null; + private final Distribution distribution; private double capacity; - private Map<Integer, Group> subgroups; + private final Map<Integer, Group> subgroups; private List<ConfiguredNode> nodes; public Group(int index, String name) { @@ -63,8 +63,7 @@ public class Group implements Comparable<Group> { @Override public boolean equals(Object o) { if (o == this) return true; - if ( ! (o instanceof Group)) { return false; } - Group other = (Group) o; + if ( ! (o instanceof Group other)) { return false; } if ( ! name.equals(other.name) || index != other.index || (distribution == null ^ other.distribution == null) @@ -210,7 +209,7 @@ public class Group implements Comparable<Group> { for (int i=0; i<distributionSpec.length; ++i) { String token = st.nextToken(); try{ - distributionSpec[i] = (token.equals("*") ? 0 : Integer.valueOf(token)); + distributionSpec[i] = (token.equals("*") ? 0 : Integer.parseInt(token)); } catch (NumberFormatException e) { throw new ParseException("Illegal distribution spec \"" + serialized + "\". Copy counts must be integer values in the range 1-255.", i); } @@ -243,9 +242,9 @@ public class Group implements Comparable<Group> { int asterixCount = distributionSpec.length - firstAsterix; int[][] preCalculations = new int[maxRedundancy + 1][]; for (int i=1; i<=maxRedundancy; ++i) { - List<Integer> spec = new ArrayList<Integer>(); - for (int j=0; j<distributionSpec.length; ++j) { - spec.add(distributionSpec[j]); + List<Integer> spec = new ArrayList<>(); + for (int k : distributionSpec) { + spec.add(k); } int remainingRedundancy = i; for (int j=0; j<firstAsterix; ++j) { @@ -277,8 +276,7 @@ public class Group implements Comparable<Group> { @Override public boolean equals(Object o) { if (o == this) return true; - if ( ! (o instanceof Distribution)) return false; - Distribution other = (Distribution) o; + if ( ! (o instanceof Distribution other)) return false; return (distributionSpec == other.distributionSpec && preCalculatedResults.length == other.preCalculatedResults.length); } diff --git a/vdslib/src/main/java/com/yahoo/vdslib/distribution/GroupVisitor.java b/vdslib/src/main/java/com/yahoo/vdslib/distribution/GroupVisitor.java index df5a6e5a9d1..1108ce7507d 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/distribution/GroupVisitor.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/distribution/GroupVisitor.java @@ -3,6 +3,6 @@ package com.yahoo.vdslib.distribution; public interface GroupVisitor { - public boolean visitGroup(Group g); + boolean visitGroup(Group g); } diff --git a/vdslib/src/main/java/com/yahoo/vdslib/state/ClusterState.java b/vdslib/src/main/java/com/yahoo/vdslib/state/ClusterState.java index 4bf305e65e0..30a209b6754 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/state/ClusterState.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/state/ClusterState.java @@ -24,7 +24,7 @@ public class ClusterState implements Cloneable { /** * Maintains a bitset where all non-down nodes have a bit set. All nodes that differ from defaultUp - * and defaultDown are store explicit in a hash map. + * and defaultDown are stored explicitly in a hash map. */ private static class Nodes { private int logicalNodeCount; diff --git a/vdslib/src/main/java/com/yahoo/vdslib/state/Diff.java b/vdslib/src/main/java/com/yahoo/vdslib/state/Diff.java index f4eb9ff8dde..8a4bedddaeb 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/state/Diff.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/state/Diff.java @@ -8,8 +8,9 @@ import java.util.List; * TODO: document this */ public class Diff { + public static class Entry { - String id; + final String id; // Values set for entries that contain diff themselves String preContent; String postContent; @@ -32,22 +33,22 @@ public class Diff { public Entry bold() { bold = true; return this; } public Entry splitLine() { splitLine = true; return this; } } - private List<Entry> diff = new LinkedList<Entry>(); + private final List<Entry> diff = new LinkedList<>(); public void add(Entry e) { diff.add(e); } public boolean differs() { return (!diff.isEmpty()); } - class PrintProperties { + static class PrintProperties { boolean insertLineBreaks = false; - boolean ommitGroupForSingleEntries = true; + final boolean ommitGroupForSingleEntries = true; String lineBreak = "\n"; - String entrySeparator = ", "; - String idValueSeparator = ": "; + final String entrySeparator = ", "; + final String idValueSeparator = ": "; String keyValueSeparator = " => "; - String singleGroupSeparator = ""; - String groupStart = "["; - String groupStop = "]"; + final String singleGroupSeparator = ""; + final String groupStart = "["; + final String groupStop = "]"; String indent = " "; String boldStart = ""; String boldStop = ""; diff --git a/vdslib/src/test/java/com/yahoo/vdslib/BucketDistributionTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/BucketDistributionTestCase.java index 7257bf0cc7f..59b5a7ae55a 100644 --- a/vdslib/src/test/java/com/yahoo/vdslib/BucketDistributionTestCase.java +++ b/vdslib/src/test/java/com/yahoo/vdslib/BucketDistributionTestCase.java @@ -25,7 +25,7 @@ public class BucketDistributionTestCase { BucketDistribution bd = new BucketDistribution(NUM_COLUMNS, numBucketBits); for (int i = 0; i < bd.getNumBuckets(); ++i) { if (i % 32 == 0) { - System.out.println(""); + System.out.println(); System.out.print(" "); } System.out.print(bd.getColumn(new BucketId(16, i))); @@ -37,7 +37,7 @@ public class BucketDistributionTestCase { if (numBucketBits < MAX_BUCKETBITS) { System.out.print(","); } - System.out.println(""); + System.out.println(); } System.out.println(" };"); } diff --git a/vdslib/src/test/java/com/yahoo/vdslib/SearchResultTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/SearchResultTestCase.java index 3f3e8fd0f8b..b675798b374 100644 --- a/vdslib/src/test/java/com/yahoo/vdslib/SearchResultTestCase.java +++ b/vdslib/src/test/java/com/yahoo/vdslib/SearchResultTestCase.java @@ -17,12 +17,12 @@ public class SearchResultTestCase { SearchResult.Hit b = new SearchResult.Hit("b", 0.1); SearchResult.Hit c = new SearchResult.Hit("c", 1.0); SearchResult.Hit bb = new SearchResult.Hit("b2", 0.1); - assertTrue(a.compareTo(a) == 0); + assertEquals(0, a.compareTo(a)); assertTrue(a.compareTo(b) > 0); assertTrue(a.compareTo(c) > 0); assertTrue(b.compareTo(a) < 0); - assertTrue(b.compareTo(bb) == 0); - assertTrue(bb.compareTo(b) == 0); + assertEquals(0, b.compareTo(bb)); + assertEquals(0, bb.compareTo(b)); assertTrue(b.compareTo(c) > 0); assertTrue(c.compareTo(a) < 0); assertTrue(c.compareTo(b) < 0); @@ -47,7 +47,7 @@ public class SearchResultTestCase { SearchResult.Hit h5 = new SearchResult.HitWithSortBlob(a, b5); SearchResult.Hit h6 = new SearchResult.HitWithSortBlob(a, b6); - assertTrue(h1.compareTo(h1) == 0); + assertEquals(0, h1.compareTo(h1)); assertTrue(h1.compareTo(h2) < 0); assertTrue(h1.compareTo(h3) < 0); assertTrue(h1.compareTo(h4) < 0); @@ -55,7 +55,7 @@ public class SearchResultTestCase { assertTrue(h1.compareTo(h6) < 0); assertTrue(h2.compareTo(h1) > 0); - assertTrue(h2.compareTo(h2) == 0); + assertEquals(0, h2.compareTo(h2)); assertTrue(h2.compareTo(h3) < 0); assertTrue(h2.compareTo(h4) < 0); assertTrue(h2.compareTo(h5) < 0); @@ -63,7 +63,7 @@ public class SearchResultTestCase { assertTrue(h3.compareTo(h1) > 0); assertTrue(h3.compareTo(h2) > 0); - assertTrue(h3.compareTo(h3) == 0); + assertEquals(0, h3.compareTo(h3)); assertTrue(h3.compareTo(h4) < 0); assertTrue(h3.compareTo(h5) < 0); assertTrue(h3.compareTo(h6) < 0); @@ -71,7 +71,7 @@ public class SearchResultTestCase { assertTrue(h4.compareTo(h1) > 0); assertTrue(h4.compareTo(h2) > 0); assertTrue(h4.compareTo(h3) > 0); - assertTrue(h4.compareTo(h4) == 0); + assertEquals(0, h4.compareTo(h4)); assertTrue(h4.compareTo(h5) < 0); assertTrue(h4.compareTo(h6) < 0); @@ -79,7 +79,7 @@ public class SearchResultTestCase { assertTrue(h5.compareTo(h2) > 0); assertTrue(h5.compareTo(h3) > 0); assertTrue(h5.compareTo(h4) > 0); - assertTrue(h5.compareTo(h5) == 0); + assertEquals(0, h5.compareTo(h5)); assertTrue(h5.compareTo(h6) < 0); assertTrue(h6.compareTo(h1) > 0); @@ -87,6 +87,6 @@ public class SearchResultTestCase { assertTrue(h6.compareTo(h3) > 0); assertTrue(h6.compareTo(h4) > 0); assertTrue(h6.compareTo(h5) > 0); - assertTrue(h6.compareTo(h6) == 0); + assertEquals(0, h6.compareTo(h6)); } } diff --git a/vdslib/src/test/java/com/yahoo/vdslib/distribution/CrossPlatformTestFactory.java b/vdslib/src/test/java/com/yahoo/vdslib/distribution/CrossPlatformTestFactory.java index 70a11ff530f..90128c7c04b 100644 --- a/vdslib/src/test/java/com/yahoo/vdslib/distribution/CrossPlatformTestFactory.java +++ b/vdslib/src/test/java/com/yahoo/vdslib/distribution/CrossPlatformTestFactory.java @@ -20,36 +20,29 @@ public abstract class CrossPlatformTestFactory { public String getName() { return name; } - public boolean loadTestResults() throws Exception { + public void loadTestResults() throws Exception { File reference = new File(directory, name + ".reference.results"); if (!reference.exists()) { - return false; + return; } - BufferedReader br = new BufferedReader(new FileReader(reference)); - StringBuilder sb = new StringBuilder(); - try{ - while(true) { + try (BufferedReader br = new BufferedReader(new FileReader(reference))) { + StringBuilder sb = new StringBuilder(); + while (true) { String line = br.readLine(); if (line == null) break; sb.append(line); } parse(sb.toString()); - } finally { - br.close(); } - return true; } public void recordTestResults() throws Exception { File results = new File(directory, name + ".java.results"); - FileWriter fw = new FileWriter(results); - try{ + try (FileWriter fw = new FileWriter(results)) { fw.write(serialize()); - } finally { - fw.close(); } } - public abstract String serialize() throws Exception; + public abstract String serialize(); public abstract void parse(String serialized) throws Exception; } diff --git a/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestCase.java index 19c9c79522d..6dfffa23aed 100644 --- a/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestCase.java +++ b/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestCase.java @@ -30,26 +30,27 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class DistributionTestCase { + + private static final int minUsedBits = 16; + private DistributionTestFactory test; /** Build a set of buckets to test that should represent the entire bucket space well. */ - private static List<BucketId> getTestBuckets() { return getTestBuckets(16); } - private static List<BucketId> getTestBuckets(int minUsedBits) { + private static List<BucketId> getTestBuckets() { List<BucketId> buckets = new ArrayList<>(); - assertTrue(minUsedBits <= 16); - // Get a set of buckets from the same split level - for (int i=16; i<=18; ++i) { - for (int j=0; j<20; ++j) { + // Get a set of buckets from the same split level + for (int i = 16; i <= 18; ++ i) { + for (int j = 0; j < 20; ++ j) { buckets.add(new BucketId(i, j)); } } - // Get a few random buckets at every split level. + // Get a few random buckets at every split level. Random randomized = new Random(413); long randValue = randomized.nextLong(); - for (int i=minUsedBits; i<58; ++i) { + for (int i = minUsedBits; i < 58; ++ i) { buckets.add(new BucketId(i, randValue)); } randValue = randomized.nextLong(); - for (int i=minUsedBits; i<58; ++i) { + for (int i = minUsedBits; i < 58; ++ i) { buckets.add(new BucketId(i, randValue)); } return Collections.unmodifiableList(buckets); @@ -230,7 +231,7 @@ public class DistributionTestCase { } @Test - public void testSplitBeyondSplitBitDoesntAffectDistribution() throws Exception { + public void testSplitBeyondSplitBitDoesntAffectDistribution() { Random randomized = new Random(7123161); long val = randomized.nextLong(); test = new DistributionTestFactory("abovesplitbit"); @@ -325,7 +326,7 @@ public class DistributionTestCase { } @Test - public void testHierarchicalDistribution() throws Exception { + public void testHierarchicalDistribution() { test = new DistributionTestFactory("hierarchical-grouping") .setDistribution(buildHierarchicalConfig(6, 3, 1, "1|2|*", 3)); for (BucketId bucket : getTestBuckets()) { diff --git a/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestFactory.java b/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestFactory.java index 78b548e5925..e94e4f04199 100644 --- a/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestFactory.java +++ b/vdslib/src/test/java/com/yahoo/vdslib/distribution/DistributionTestFactory.java @@ -21,7 +21,7 @@ import static org.junit.Assert.assertTrue; // TODO: Use config builder instead of ConfigGetter to create test config. public class DistributionTestFactory extends CrossPlatformTestFactory { - ObjectMapper mapper = new ObjectMapper(); + final ObjectMapper mapper = new ObjectMapper(); private static final String testDirectory = "src/tests/distribution/testdata"; private int redundancy; @@ -32,14 +32,14 @@ public class DistributionTestFactory extends CrossPlatformTestFactory { private String upStates; private int testsRecorded = 0; - private List<Test> results = new ArrayList<>(); + private final List<Test> results = new ArrayList<>(); private int testsVerified = 0; - enum Failure { NONE, TOO_FEW_BITS, NO_DISTRIBUTORS_AVAILABLE }; + enum Failure { NONE, TOO_FEW_BITS, NO_DISTRIBUTORS_AVAILABLE } static public class Test { - private BucketId bucket; - private List<Integer> nodes; + private final BucketId bucket; + private final List<Integer> nodes; private Failure failure; public Test(BucketId bucket) { @@ -50,8 +50,7 @@ public class DistributionTestFactory extends CrossPlatformTestFactory { @Override public boolean equals(Object other) { - if (!(other instanceof Test)) return false; - Test t = (Test) other; + if (!(other instanceof Test t)) return false; return (bucket.equals(t.bucket) && nodes.equals(t.nodes) && failure.equals(t.failure)); @@ -81,19 +80,14 @@ public class DistributionTestFactory extends CrossPlatformTestFactory { return nodes; } - public Test assertFailure(Failure f) { - assertEquals(f, failure); - return this; - } public Test assertNodeCount(int count) { if (count > 0) assertEquals(toString(), Failure.NONE, failure); assertEquals(toString(), count, nodes.size()); return this; } - public Test assertNodeUsed(int node) { + public void assertNodeUsed(int node) { assertEquals(toString(), Failure.NONE, failure); assertTrue(toString(), nodes.contains(node)); - return this; } } @@ -166,9 +160,7 @@ public class DistributionTestFactory extends CrossPlatformTestFactory { int node = d.getIdealDistributorNode(state, bucket, upStates); t.nodes.add(node); } else { - for (int i : d.getIdealStorageNodes(state, bucket, upStates)) { - t.nodes.add(i); - } + t.nodes.addAll(d.getIdealStorageNodes(state, bucket, upStates)); } } catch (Distribution.TooFewBucketBitsInUseException e) { t.failure = Failure.TOO_FEW_BITS; @@ -184,7 +176,7 @@ public class DistributionTestFactory extends CrossPlatformTestFactory { return t; } - public String serialize() throws Exception { + public String serialize() { ObjectNode test = new ObjectNode(mapper.getNodeFactory()) .put("cluster-state", state.toString()) .put("distribution", new StorDistributionConfig(distributionConfig).toString()) diff --git a/vdslib/src/test/java/com/yahoo/vdslib/distribution/GroupTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/distribution/GroupTestCase.java index 353f2bf4ebc..ce9d4dcedff 100644 --- a/vdslib/src/test/java/com/yahoo/vdslib/distribution/GroupTestCase.java +++ b/vdslib/src/test/java/com/yahoo/vdslib/distribution/GroupTestCase.java @@ -31,7 +31,7 @@ public class GroupTestCase { private void assertDistributionFailure(String spec, int redundancy, String expectedError) { try{ - Group.Distribution distribution = new Group.Distribution(spec, redundancy); + new Group.Distribution(spec, redundancy); fail("Failed to fail parsing of spec \"" + spec + "\", redundancy " + redundancy + " with failure: " + expectedError); } catch (Exception e) { assertEquals(expectedError, e.getMessage()); diff --git a/vdslib/src/test/java/com/yahoo/vdslib/state/ClusterStateTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/state/ClusterStateTestCase.java index 77dd37b3ebf..c4ff28b75b1 100644 --- a/vdslib/src/test/java/com/yahoo/vdslib/state/ClusterStateTestCase.java +++ b/vdslib/src/test/java/com/yahoo/vdslib/state/ClusterStateTestCase.java @@ -9,7 +9,9 @@ import java.util.function.BiFunction; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class ClusterStateTestCase{ @@ -55,16 +57,16 @@ public class ClusterStateTestCase{ assertEquals(state, new ClusterState("storage:0")); assertEquals(state, new ClusterState("distributor:0")); - assertFalse(state.equals(new ClusterState("version:1"))); - assertFalse(state.equals(new ClusterState("cluster:d"))); - assertFalse(state.equals(new ClusterState("bits:20"))); - assertFalse(state.equals(new ClusterState("storage:1"))); - assertFalse(state.equals(new ClusterState("distributor:1"))); + assertNotEquals(state, new ClusterState("version:1")); + assertNotEquals(state, new ClusterState("cluster:d")); + assertNotEquals(state, new ClusterState("bits:20")); + assertNotEquals(state, new ClusterState("storage:1")); + assertNotEquals(state, new ClusterState("distributor:1")); { ClusterState state1 = new ClusterState("distributor:3 .1.s:d .2.s:m storage:3 .1.s:i .2.s:r"); ClusterState state2 = new ClusterState("distributor:3 .1.s:d .2.s:m storage:3 .1.s:i .2.s:m"); - assertFalse(state1.equals(state2)); + assertNotEquals(state1, state2); assertFalse(state1.similarTo(state2)); assertFalse(state1.similarToIgnoringInitProgress(state2)); } @@ -72,7 +74,7 @@ public class ClusterStateTestCase{ { ClusterState state1 = new ClusterState("cluster:d"); ClusterState state2 = new ClusterState("cluster:d version:1 bits:20 distributor:1 storage:1 .0.s:d"); - assertFalse(state1.equals(state2)); + assertNotEquals(state1, state2); assertTrue(state1.similarTo(state2)); assertTrue(state1.similarToIgnoringInitProgress(state2)); } @@ -80,12 +82,12 @@ public class ClusterStateTestCase{ { ClusterState state1 = new ClusterState("distributor:3 .1.s:d .2.s:m storage:3 .1.s:i .2.s:r"); ClusterState state2 = new ClusterState("distributor:3 storage:3"); - assertFalse(state1.equals(state2)); + assertNotEquals(state1, state2); assertFalse(state1.similarTo(state2)); assertFalse(state1.similarToIgnoringInitProgress(state2)); } - assertFalse(state.equals("class not instance of ClusterState")); + assertNotEquals("class not instance of ClusterState", state); assertFalse(state.similarTo("class not instance of ClusterState")); assertEquals(state, state); @@ -200,36 +202,39 @@ public class ClusterStateTestCase{ ClusterState state3 = new ClusterState("distributor:9 storage:2"); assertEquals("storage: [4: Down => Up, 5: Down => Up], distributor: [7: Up => Down, 8: Up => Down]", state1.getTextualDifference(state2)); - assertEquals("storage: [<br>\n" + - " 4: <b>Down</b> => <b>Up</b>, <br>\n" + - " 5: <b>Down</b> => <b>Up</b><br>\n" + - "], distributor: [<br>\n" + - " 7: <b>Up</b> => <b>Down</b>, <br>\n" + - " 8: <b>Up</b> => <b>Down</b><br>\n" + - "]", state1.getHtmlDifference(state2)); + assertEquals(""" + storage: [<br> + 4: <b>Down</b> => <b>Up</b>, <br> + 5: <b>Down</b> => <b>Up</b><br> + ], distributor: [<br> + 7: <b>Up</b> => <b>Down</b>, <br> + 8: <b>Up</b> => <b>Down</b><br> + ]""", state1.getHtmlDifference(state2)); assertEquals("storage: [2: Up => Down, 3: Up => Down, 4: Up => Down, 5: Up => Down], distributor: [7: Down => Up, 8: Down => Up]", state2.getTextualDifference(state3)); - assertEquals("storage: [<br>\n" + - " 2: <b>Up</b> => <b>Down</b>, <br>\n" + - " 3: <b>Up</b> => <b>Down</b>, <br>\n" + - " 4: <b>Up</b> => <b>Down</b>, <br>\n" + - " 5: <b>Up</b> => <b>Down</b><br>\n" + - "], distributor: [<br>\n" + - " 7: <b>Down</b> => <b>Up</b>, <br>\n" + - " 8: <b>Down</b> => <b>Up</b><br>\n" + - "]", state2.getHtmlDifference(state3)); + assertEquals(""" + storage: [<br> + 2: <b>Up</b> => <b>Down</b>, <br> + 3: <b>Up</b> => <b>Down</b>, <br> + 4: <b>Up</b> => <b>Down</b>, <br> + 5: <b>Up</b> => <b>Down</b><br> + ], distributor: [<br> + 7: <b>Down</b> => <b>Up</b>, <br> + 8: <b>Down</b> => <b>Up</b><br> + ]""", state2.getHtmlDifference(state3)); state1.setVersion(123); state1.setNodeState(new Node(NodeType.STORAGE, 2), new NodeState(NodeType.STORAGE, State.INITIALIZING).setInitProgress(0.2f).setDescription("Booting")); state2.setDistributionBits(21); assertEquals("version: 123 => 0, bits: 16 => 21, storage: [2: [Initializing => Up, description: Booting => ], 4: Down => Up, 5: Down => Up], distributor: [7: Up => Down, 8: Up => Down]", state1.getTextualDifference(state2)); - assertEquals("version: 123 => 0, bits: 16 => 21, storage: [<br>\n" + - " 2: [<b>Initializing</b> => <b>Up</b>, description: Booting => ], <br>\n" + - " 4: <b>Down</b> => <b>Up</b>, <br>\n" + - " 5: <b>Down</b> => <b>Up</b><br>\n" + - "], distributor: [<br>\n" + - " 7: <b>Up</b> => <b>Down</b>, <br>\n" + - " 8: <b>Up</b> => <b>Down</b><br>\n" + - "]", state1.getHtmlDifference(state2)); + assertEquals(""" + version: 123 => 0, bits: 16 => 21, storage: [<br> + 2: [<b>Initializing</b> => <b>Up</b>, description: Booting => ], <br> + 4: <b>Down</b> => <b>Up</b>, <br> + 5: <b>Down</b> => <b>Up</b><br> + ], distributor: [<br> + 7: <b>Up</b> => <b>Down</b>, <br> + 8: <b>Up</b> => <b>Down</b><br> + ]""", state1.getHtmlDifference(state2)); } @Test @@ -254,40 +259,40 @@ public class ClusterStateTestCase{ try { new ClusterState("badtokenwithoutcolon"); - assertTrue("Should fail", false); - } catch (Exception e) {} + fail("Should fail"); + } catch (Exception ignored) {} try { new ClusterState(".0.s:d"); - assertTrue("Should fail", false); - } catch (Exception e) {} + fail("Should fail"); + } catch (Exception ignored) {} try { new ClusterState("cluster:badvalue"); - assertTrue("Should fail", false); - } catch (Exception e) {} + fail("Should fail"); + } catch (Exception ignored) {} try { new ClusterState("cluster:m"); - assertTrue("Should fail", false); - } catch (Exception e) {} + fail("Should fail"); + } catch (Exception ignored) {} try { new ClusterState("version:badvalue"); - assertTrue("Should fail", false); - } catch (Exception e) {} + fail("Should fail"); + } catch (Exception ignored) {} try { new ClusterState("distributor:badvalue"); - assertTrue("Should fail", false); - } catch (Exception e) {} + fail("Should fail"); + } catch (Exception ignored) {} try { new ClusterState("storage:badvalue"); - assertTrue("Should fail", false); - } catch (Exception e) {} + fail("Should fail"); + } catch (Exception ignored) {} try { new ClusterState("distributor:2 .3.s:d"); - assertTrue("Should fail", false); - } catch (Exception e) {} + fail("Should fail"); + } catch (Exception ignored) {} try { new ClusterState("storage:2 .3.s:d"); - assertTrue("Should fail", false); - } catch (Exception e) {} + fail("Should fail"); + } catch (Exception ignored) {} } @Test diff --git a/vdslib/src/test/java/com/yahoo/vdslib/state/NodeStateTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/state/NodeStateTestCase.java index 1ce3655b394..3eff07e80b9 100644 --- a/vdslib/src/test/java/com/yahoo/vdslib/state/NodeStateTestCase.java +++ b/vdslib/src/test/java/com/yahoo/vdslib/state/NodeStateTestCase.java @@ -4,11 +4,12 @@ package com.yahoo.vdslib.state; import org.junit.Test; import java.text.ParseException; -import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class NodeStateTestCase { @@ -56,36 +57,36 @@ public class NodeStateTestCase { assertEquals(ns, NodeState.deserialize(NodeType.STORAGE, ": s:m sbadkey:u bbadkey:2 cbadkey:2.0 rbadkey:2 ibadkey:0.5 tbadkey:2 mbadkey:message dbadkey:2 unknownkey:somevalue")); try { NodeState.deserialize(NodeType.STORAGE, "s:m badtokenwithoutcolon"); - assertTrue("Should fail", false); - } catch (Exception e) {} + fail("Should fail"); + } catch (Exception ignored) {} try { NodeState.deserialize(NodeType.STORAGE, "s:m c:badvalue"); - assertTrue("Should fail", false); - } catch (Exception e) {} + fail("Should fail"); + } catch (Exception ignored) {} try { NodeState.deserialize(NodeType.STORAGE, "s:m i:badvalue"); - assertTrue("Should fail", false); - } catch (Exception e) {} + fail("Should fail"); + } catch (Exception ignored) {} try { NodeState.deserialize(NodeType.STORAGE, "s:m t:badvalue"); - assertTrue("Should fail", false); - } catch (Exception e) {} + fail("Should fail"); + } catch (Exception ignored) {} try { NodeState.deserialize(NodeType.STORAGE, "s:m t:-1"); - assertTrue("Should fail", false); - } catch (Exception e) {} + fail("Should fail"); + } catch (Exception ignored) {} try { NodeState.deserialize(NodeType.STORAGE, "s:m d:badvalue"); - assertTrue("Should fail", false); - } catch (Exception e) {} + fail("Should fail"); + } catch (Exception ignored) {} try { NodeState.deserialize(NodeType.STORAGE, "s:m d.badkey:badvalue"); - assertTrue("Should fail", false); - } catch (Exception e) {} + fail("Should fail"); + } catch (Exception ignored) {} try { NodeState.deserialize(NodeType.STORAGE, "s:m d.1:badindex"); - assertTrue("Should fail", false); - } catch (Exception e) {} + fail("Should fail"); + } catch (Exception ignored) {} ns = new NodeState(NodeType.STORAGE, State.UP).setDescription("Foo bar"); assertEquals("", ns.serialize(2, false)); @@ -127,11 +128,11 @@ public class NodeStateTestCase { assertTrue(ns1.similarToIgnoringInitProgress(ns4)); assertTrue(ns2.similarToIgnoringInitProgress(ns4)); - assertFalse(ns1.equals(ns2)); - assertFalse(ns2.equals(ns3)); - assertFalse(ns3.equals(ns4)); + assertNotEquals(ns1, ns2); + assertNotEquals(ns2, ns3); + assertNotEquals(ns3, ns4); - assertFalse(ns1.equals("class not instance of NodeState")); + assertNotEquals("class not instance of NodeState", ns1); assertFalse(ns1.similarTo("class not instance of NodeState")); } { @@ -139,7 +140,7 @@ public class NodeStateTestCase { NodeState ns2 = new NodeState(NodeType.STORAGE, State.UP).setMinUsedBits(18); assertTrue(ns1.similarTo(ns2)); assertTrue(ns1.similarToIgnoringInitProgress(ns2)); - assertFalse(ns1.equals(ns2)); + assertNotEquals(ns1, ns2); } } @@ -163,12 +164,12 @@ public class NodeStateTestCase { public void testValidInClusterState() { try{ new NodeState(NodeType.DISTRIBUTOR, State.UNKNOWN).verifyValidInSystemState(NodeType.DISTRIBUTOR); - assertTrue("Should not be valid", false); - } catch (Exception e) {} + fail("Should not be valid"); + } catch (Exception ignored) {} try{ new NodeState(NodeType.DISTRIBUTOR, State.UP).setCapacity(3).verifyValidInSystemState(NodeType.DISTRIBUTOR); - assertTrue("Should not be valid", false); - } catch (Exception e) {} + fail("Should not be valid"); + } catch (Exception ignored) {} } } diff --git a/vdslib/src/test/java/com/yahoo/vdslib/state/NodeTest.java b/vdslib/src/test/java/com/yahoo/vdslib/state/NodeTest.java index 971b4782ab0..e3fc0faecd5 100644 --- a/vdslib/src/test/java/com/yahoo/vdslib/state/NodeTest.java +++ b/vdslib/src/test/java/com/yahoo/vdslib/state/NodeTest.java @@ -5,7 +5,9 @@ import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class NodeTest { @@ -29,23 +31,23 @@ public class NodeTest { assertEquals(n3, n3); assertEquals(n4, n4); - assertFalse(n1.equals(n2)); - assertFalse(n1.equals(n3)); - assertFalse(n1.equals(n4)); + assertNotEquals(n1, n2); + assertNotEquals(n1, n3); + assertNotEquals(n1, n4); - assertFalse(n2.equals(n1)); - assertFalse(n2.equals(n3)); - assertFalse(n2.equals(n4)); + assertNotEquals(n2, n1); + assertNotEquals(n2, n3); + assertNotEquals(n2, n4); - assertFalse(n3.equals(n1)); - assertFalse(n3.equals(n2)); - assertFalse(n3.equals(n4)); + assertNotEquals(n3, n1); + assertNotEquals(n3, n2); + assertNotEquals(n3, n4); - assertFalse(n4.equals(n1)); - assertFalse(n4.equals(n2)); - assertFalse(n4.equals(n3)); + assertNotEquals(n4, n1); + assertNotEquals(n4, n2); + assertNotEquals(n4, n3); - assertFalse(n1.equals("class not instance of Node")); + assertNotEquals("class not instance of Node", n1); } @Test @@ -62,19 +64,19 @@ public class NodeTest { try { new Node("nodewithoutdot"); - assertTrue("Method expected to throw IllegalArgumentException", false); + fail("Method expected to throw IllegalArgumentException"); } catch (IllegalArgumentException e) { assertEquals("Not a legal node string 'nodewithoutdot'.", e.getMessage()); } try { new Node("fleetcontroller.0"); - assertTrue("Method expected to throw IllegalArgumentException", false); + fail("Method expected to throw IllegalArgumentException"); } catch (IllegalArgumentException e) { assertEquals("Unknown node type 'fleetcontroller'. Legal values are 'storage' and 'distributor'.", e.getMessage()); } try { new Node("storage.badindex"); - assertTrue("Method expected to throw NumberFormatException", false); + fail("Method expected to throw NumberFormatException"); } catch (NumberFormatException e) { assertEquals("For input string: \"badindex\"", e.getMessage()); } diff --git a/vespa-documentgen-plugin/etc/complex/book.sd b/vespa-documentgen-plugin/etc/complex/book.sd index addc556fc9b..8f071ab63d1 100644 --- a/vespa-documentgen-plugin/etc/complex/book.sd +++ b/vespa-documentgen-plugin/etc/complex/book.sd @@ -47,18 +47,15 @@ search book { field author type string { bolding: on - # index-to: default, author indexing: summary | index } field isbn type string { - # index-to: default, isbn indexing: summary | index } field year type int { indexing: summary | index } field description type string { - # index-to: default, description indexing: summary | index summary: dynamic } diff --git a/vespa-documentgen-plugin/etc/complex/common.sd b/vespa-documentgen-plugin/etc/complex/common.sd index d07f8ca9281..3ab6e18c267 100644 --- a/vespa-documentgen-plugin/etc/complex/common.sd +++ b/vespa-documentgen-plugin/etc/complex/common.sd @@ -6,7 +6,6 @@ search common { } field title type string { bolding: on - # index-to: default, title indexing: index|summary summary-to: smallsum } diff --git a/vespa-documentgen-plugin/etc/complex/music2.sd b/vespa-documentgen-plugin/etc/complex/music2.sd index 327fbeec04b..17736ab4e79 100644 --- a/vespa-documentgen-plugin/etc/complex/music2.sd +++ b/vespa-documentgen-plugin/etc/complex/music2.sd @@ -3,19 +3,16 @@ search music2 { document music2 inherits common { field artist type string { bolding: on - # index-to: default, artist indexing: index|summary } field disp_song type string { indexing: summary } field song type string { - # index-to: default, song indexing: index } field isbn type string { bolding: on - # index-to: default, isbn indexing: index|summary } field year type int { diff --git a/vespa-documentgen-plugin/etc/localapp/book.sd b/vespa-documentgen-plugin/etc/localapp/book.sd index a6f1ed9286f..8d7dd1e054e 100644 --- a/vespa-documentgen-plugin/etc/localapp/book.sd +++ b/vespa-documentgen-plugin/etc/localapp/book.sd @@ -46,18 +46,15 @@ search book { field author type string { bolding: on - # index-to: default, author indexing: summary | index } field isbn type string { - # index-to: default, isbn indexing: summary | index } field year type int { indexing: summary | index } field description type string { - # index-to: default, description indexing: summary | index summary: dynamic } diff --git a/vespa-documentgen-plugin/etc/localapp/common.sd b/vespa-documentgen-plugin/etc/localapp/common.sd index ab55e08af0f..a7828432c6b 100644 --- a/vespa-documentgen-plugin/etc/localapp/common.sd +++ b/vespa-documentgen-plugin/etc/localapp/common.sd @@ -6,7 +6,6 @@ search common { } field title type string { bolding: on - # index-to: default, title indexing: index|summary summary-to: smallsum } diff --git a/vespa-documentgen-plugin/etc/localapp/music.sd b/vespa-documentgen-plugin/etc/localapp/music.sd index cdcfdea6b75..0ce004857c2 100644 --- a/vespa-documentgen-plugin/etc/localapp/music.sd +++ b/vespa-documentgen-plugin/etc/localapp/music.sd @@ -3,19 +3,16 @@ search music { document music inherits common { field artist type string { bolding: on - # index-to: default, artist indexing: index|summary } field disp_song type string { indexing: summary } field song type string { - # index-to: default, song indexing: index } field isbn type string { bolding: on - # index-to: default, isbn indexing: index|summary } field year type int { diff --git a/vespa-documentgen-plugin/etc/localapp/video.sd b/vespa-documentgen-plugin/etc/localapp/video.sd index 2f1f1f84512..03c5c82ed64 100644 --- a/vespa-documentgen-plugin/etc/localapp/video.sd +++ b/vespa-documentgen-plugin/etc/localapp/video.sd @@ -3,26 +3,21 @@ search video { document video inherits common { field director type string { bolding: on - # index-to: default, director indexing: index|summary } field disp_actor type string { bolding: on - # index-to: default, disp_actor indexing: index|summary } field actor type string { bolding: on - # index-to: default, actor indexing: index|summary } field fmt type string { - # index-to: default, fmt indexing: index|summary } field isbn type string { bolding: on - # index-to: default, isbn indexing: index|summary } field year type int { diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java index 7b7d8712308..197b7721eca 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/FeedClientBuilderImpl.java @@ -42,7 +42,7 @@ public class FeedClientBuilderImpl implements FeedClientBuilder { SSLContext sslContext; HostnameVerifier hostnameVerifier; int connectionsPerEndpoint = 8; - int maxStreamsPerConnection = 32; + int maxStreamsPerConnection = 128; FeedClient.RetryStrategy retryStrategy = defaultRetryStrategy; FeedClient.CircuitBreaker circuitBreaker = new GracePeriodCircuitBreaker(Duration.ofSeconds(10)); Path certificateFile; @@ -201,7 +201,6 @@ public class FeedClientBuilderImpl implements FeedClientBuilder { @Override public FeedClientBuilderImpl setProxy(URI uri) { - log.warning("Proxy configuration ignored - not supported yet"); this.proxy = uri; return this; } diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java index c2181821de6..f228717eba5 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpFeedClient.java @@ -321,7 +321,7 @@ class HttpFeedClient implements FeedClient { .map(Boolean::parseBoolean) .orElse(Optional.ofNullable(System.getProperty(name)) .map(Boolean::parseBoolean) - .orElse(false)); + .orElse(true)); } } diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/JettyCluster.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/JettyCluster.java index 7fac977ca51..30dc1ab0d07 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/JettyCluster.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/JettyCluster.java @@ -5,12 +5,15 @@ package ai.vespa.feed.client.impl; import ai.vespa.feed.client.FeedClientBuilder.Compression; import ai.vespa.feed.client.HttpResponse; import org.eclipse.jetty.client.HttpClient; +import org.eclipse.jetty.client.HttpProxy; import org.eclipse.jetty.client.MultiplexConnectionPool; +import org.eclipse.jetty.client.Origin; +import org.eclipse.jetty.client.api.Authentication; import org.eclipse.jetty.client.api.Request; import org.eclipse.jetty.client.api.Response; import org.eclipse.jetty.client.api.Result; -import org.eclipse.jetty.client.util.AbstractRequestContent; import org.eclipse.jetty.client.util.BufferingResponseListener; +import org.eclipse.jetty.client.util.BytesRequestContent; import org.eclipse.jetty.http.HttpField; import org.eclipse.jetty.http.HttpHeader; import org.eclipse.jetty.http.HttpMethod; @@ -18,7 +21,6 @@ import org.eclipse.jetty.http.HttpVersion; import org.eclipse.jetty.http2.client.HTTP2Client; import org.eclipse.jetty.http2.client.http.HttpClientTransportOverHTTP2; import org.eclipse.jetty.io.ClientConnector; -import org.eclipse.jetty.util.Callback; import org.eclipse.jetty.util.HttpCookieStore; import org.eclipse.jetty.util.Pool; import org.eclipse.jetty.util.Promise; @@ -33,12 +35,14 @@ import java.io.UncheckedIOException; import java.net.Inet4Address; import java.net.InetSocketAddress; import java.net.URI; -import java.nio.ByteBuffer; import java.time.Duration; +import java.util.Collections; import java.util.List; -import java.util.Optional; +import java.util.Map; +import java.util.TreeMap; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.zip.GZIPOutputStream; @@ -70,29 +74,44 @@ class JettyCluster implements Cluster { @Override public void dispatch(HttpRequest req, CompletableFuture<HttpResponse> vessel) { client.getExecutor().execute(() -> { + Endpoint endpoint = findLeastBusyEndpoint(endpoints); try { - Endpoint endpoint = findLeastBusyEndpoint(endpoints); + endpoint.inflight.incrementAndGet(); long reqTimeoutMillis = req.timeout() != null ? req.timeout().toMillis() * 11 / 10 + 1000 : IDLE_TIMEOUT.toMillis(); Request jettyReq = client.newRequest(URI.create(endpoint.uri + req.path())) .version(HttpVersion.HTTP_2) .method(HttpMethod.fromString(req.method())) - .headers(hs -> req.headers().forEach((k, v) -> hs.add(k, v.get()))) + .headers(hs -> req.headers().forEach((k, v) -> { + if (!isProxyHeader(k)) hs.add(k, v.get()); + })) .idleTimeout(IDLE_TIMEOUT.toMillis(), MILLISECONDS) .timeout(reqTimeoutMillis, MILLISECONDS); if (req.body() != null) { - FeedContent content = new FeedContent(compression, req.body()); - content.contentEncoding().ifPresent(ce -> jettyReq.headers(hs -> hs.add(ce))); - jettyReq.body(content); + boolean shouldCompress = compression == gzip || compression == auto && req.body().length > 512; + byte[] bytes; + if (shouldCompress) { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(1 << 10); + try (GZIPOutputStream zip = new GZIPOutputStream(buffer)) { + zip.write(req.body()); + } catch (IOException e) { throw new UncheckedIOException(e); } + bytes = buffer.toByteArray(); + jettyReq.headers(hs -> hs.add(HttpHeader.CONTENT_ENCODING, "gzip")); + } else { + bytes = req.body(); + } + jettyReq.body(new BytesRequestContent(APPLICATION_JSON.asString(), bytes)); } jettyReq.send(new BufferingResponseListener() { @Override public void onComplete(Result result) { + endpoint.inflight.decrementAndGet(); if (result.isFailed()) vessel.completeExceptionally(result.getFailure()); else vessel.complete(new JettyResponse(result.getResponse(), getContent())); } }); } catch (Exception e) { + endpoint.inflight.decrementAndGet(); vessel.completeExceptionally(e); } }); @@ -114,13 +133,15 @@ class JettyCluster implements Cluster { clientSslCtxFactory.setEndpointIdentificationAlgorithm(null); } ClientConnector connector = new ClientConnector(); - int threads = Math.max(Math.min(Runtime.getRuntime().availableProcessors(), 20), 8); + int threads = Math.max(Math.min(Runtime.getRuntime().availableProcessors(), 32), 8); connector.setExecutor(new QueuedThreadPool(threads)); connector.setSslContextFactory(clientSslCtxFactory); + connector.setIdleTimeout(IDLE_TIMEOUT); + connector.setConnectTimeout(Duration.ofSeconds(10)); HTTP2Client h2Client = new HTTP2Client(connector); h2Client.setMaxConcurrentPushedStreams(b.maxStreamsPerConnection); // Set the HTTP/2 flow control windows very large to cause TCP congestion instead of HTTP/2 flow control congestion. - int initialWindow = 128 * 1024 * 1024; + int initialWindow = Integer.MAX_VALUE; h2Client.setInitialSessionRecvWindow(initialWindow); h2Client.setInitialStreamRecvWindow(initialWindow); HttpClientTransportOverHTTP2 transport = new HttpClientTransportOverHTTP2(h2Client); @@ -134,20 +155,46 @@ class JettyCluster implements Cluster { httpClient.setFollowRedirects(false); httpClient.setUserAgentField( new HttpField(HttpHeader.USER_AGENT, String.format("vespa-feed-client/%s (Jetty)", Vespa.VERSION))); - httpClient.setConnectTimeout(Duration.ofSeconds(10).toMillis()); // Stop client from trying different IP address when TLS handshake fails httpClient.setSocketAddressResolver(new Ipv4PreferringResolver(httpClient, Duration.ofSeconds(10))); httpClient.setCookieStore(new HttpCookieStore.Empty()); - httpClient.setIdleTimeout(IDLE_TIMEOUT.toMillis()); - try { - httpClient.start(); - } catch (Exception e) { - throw new IOException(e); - } + if (b.proxy != null) addProxyConfiguration(b, httpClient); + try { httpClient.start(); } catch (Exception e) { throw new IOException(e); } return httpClient; } + private static void addProxyConfiguration(FeedClientBuilderImpl b, HttpClient httpClient) throws IOException { + Origin.Address address = new Origin.Address(b.proxy.getHost(), b.proxy.getPort()); + if (b.proxy.getScheme().equals("https")) { + SslContextFactory.Client proxySslCtxFactory = new SslContextFactory.Client(); + if (b.hostnameVerifier != null) proxySslCtxFactory.setHostnameVerifier(b.hostnameVerifier); + // Disable built-in hostname verification in the JDK's TLS implementation + proxySslCtxFactory.setEndpointIdentificationAlgorithm(null); + try { proxySslCtxFactory.start(); } catch (Exception e) { throw new IOException(e); } + httpClient.getProxyConfiguration().addProxy( + new HttpProxy(address, proxySslCtxFactory, new Origin.Protocol(Collections.singletonList("h2"), false))); + } else { + httpClient.getProxyConfiguration().addProxy( + new HttpProxy(address, false, new Origin.Protocol(Collections.singletonList("h2c"), false))); + } + Map<String, Supplier<String>> proxyHeaders = new TreeMap<>(); + b.requestHeaders.forEach((k, v) -> { if (isProxyHeader(k)) proxyHeaders.put(k, v); }); + if (!proxyHeaders.isEmpty()) { + for (URI endpoint : b.endpoints) { + httpClient.getAuthenticationStore().addAuthenticationResult(new Authentication.Result() { + @Override public URI getURI() { return URI.create(endpointUri(endpoint)); } + @Override public void apply(Request r) { + r.headers(hs -> proxyHeaders.forEach((k, v) -> hs.add(k, v.get()))); + } + }); + + } + } + } + + private static boolean isProxyHeader(String h) { return h.equalsIgnoreCase(HttpHeader.PROXY_AUTHORIZATION.asString()); } + private static Endpoint findLeastBusyEndpoint(List<Endpoint> endpoints) { Endpoint leastBusy = endpoints.get(0); int minInflight = leastBusy.inflight.get(); @@ -166,6 +213,10 @@ class JettyCluster implements Cluster { return u.getPort() == -1 ? u.getScheme().equals("http") ? 80 : 443 : u.getPort(); } + private static String endpointUri(URI uri) { + return String.format("%s://%s:%s", uri.getScheme(), uri.getHost(), portOf(uri)); + } + private static class JettyResponse implements HttpResponse { final Response response; final byte[] content; @@ -180,50 +231,7 @@ class JettyCluster implements Cluster { private static class Endpoint { final AtomicInteger inflight = new AtomicInteger(); final String uri; - Endpoint(URI uri) { this.uri = String.format("%s://%s:%s", uri.getScheme(), uri.getHost(), portOf(uri)); } - } - - private static class FeedContent extends AbstractRequestContent { - final Compression compression; - final byte[] body; - - FeedContent(Compression compression, byte[] body) { - super(APPLICATION_JSON.asString()); - this.compression = compression; - this.body = body; - } - - @Override public boolean isReproducible() { return true; } - @Override public long getLength() { return shouldCompress() ? -1 : body.length; } - Optional<HttpField> contentEncoding() { - return shouldCompress() ? Optional.of(new HttpField(HttpHeader.CONTENT_ENCODING, "gzip")) : Optional.empty(); - } - - @Override - public Subscription newSubscription(Consumer consumer, boolean emitInitialContent) { - return new SubscriptionImpl(consumer, emitInitialContent); - } - - boolean shouldCompress() { return compression == gzip || compression == auto && body.length > 512; } - - class SubscriptionImpl extends AbstractSubscription { - SubscriptionImpl(Consumer consumer, boolean emitInitialContent) { super(consumer, emitInitialContent); } - - @Override - protected boolean produceContent(Producer producer) { - byte[] bytes; - if (shouldCompress()) { - ByteArrayOutputStream buffer = new ByteArrayOutputStream(1 << 10); - try (GZIPOutputStream zip = new GZIPOutputStream(buffer)) { - zip.write(body); - } catch (IOException e) { throw new UncheckedIOException(e); } - bytes = buffer.toByteArray(); - } else { - bytes = body; - } - return producer.produce(ByteBuffer.wrap(bytes), true, Callback.NOOP); - } - } + Endpoint(URI uri) { this.uri = endpointUri(uri); } } private static class Ipv4PreferringResolver extends AbstractLifeCycle implements SocketAddressResolver { diff --git a/vespaclient-core/src/main/java/com/yahoo/clientmetrics/MessageTypeMetricSet.java b/vespaclient-core/src/main/java/com/yahoo/clientmetrics/MessageTypeMetricSet.java index 46ad5ebfab6..8b798d4b76e 100644 --- a/vespaclient-core/src/main/java/com/yahoo/clientmetrics/MessageTypeMetricSet.java +++ b/vespaclient-core/src/main/java/com/yahoo/clientmetrics/MessageTypeMetricSet.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.clientmetrics; +import com.yahoo.concurrent.Timer; import com.yahoo.documentapi.messagebus.protocol.DocumentIgnoredReply; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.messagebus.Reply; @@ -12,21 +13,24 @@ import java.util.Map; import java.util.stream.Stream; /** -* @author thomasg +* @author Thomas Gundersen */ public class MessageTypeMetricSet { + public long latency_total; public long latency_min = Long.MAX_VALUE; public long latency_max = Long.MIN_VALUE; public long count = 0; public long ignored = 0; public long errorCount = 0; + public final Timer timer; private final Map<String, Long> errorCounts = new HashMap<>(); private final String msgName; - public MessageTypeMetricSet(String msgName) { + MessageTypeMetricSet(String msgName, Timer timer) { this.msgName = msgName; + this.timer = timer; } public String getMessageName() { @@ -55,7 +59,7 @@ public class MessageTypeMetricSet { private void updateSuccessMetrics(Reply r) { if (!(r instanceof DocumentIgnoredReply)) { if (r.getMessage().getTimeReceived() != 0) { - long latency = (SystemTimer.INSTANCE.milliTime() - r.getMessage().getTimeReceived()); + long latency = (timer.milliTime() - r.getMessage().getTimeReceived()); latency_max = Math.max(latency_max, latency); latency_min = Math.min(latency_min, latency); latency_total += latency; diff --git a/vespaclient-core/src/main/java/com/yahoo/clientmetrics/RouteMetricSet.java b/vespaclient-core/src/main/java/com/yahoo/clientmetrics/RouteMetricSet.java index ebf6246b034..61fd5dfdca3 100644 --- a/vespaclient-core/src/main/java/com/yahoo/clientmetrics/RouteMetricSet.java +++ b/vespaclient-core/src/main/java/com/yahoo/clientmetrics/RouteMetricSet.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.clientmetrics; +import com.yahoo.concurrent.SystemTimer; +import com.yahoo.concurrent.Timer; import com.yahoo.messagebus.Reply; import java.util.HashMap; @@ -12,6 +14,7 @@ import java.util.Map; public class RouteMetricSet { private final String route; + private final Timer timer; private final ProgressCallback callback; private final Map<Integer, MessageTypeMetricSet> typeMap = new HashMap<>(); @@ -20,18 +23,23 @@ public class RouteMetricSet { void done(RouteMetricSet route); } - public RouteMetricSet(String route, ProgressCallback callback) { + public RouteMetricSet(String route, Timer timer, ProgressCallback callback) { this.route = route; + this.timer = timer; this.callback = callback; } + public RouteMetricSet(String route, ProgressCallback callback) { + this(route, SystemTimer.INSTANCE, callback); + } + public Map<Integer, MessageTypeMetricSet> getMetrics() { return typeMap; } public void addReply(Reply r) { MessageTypeMetricSet type = typeMap.get(r.getMessage().getType()); if (type == null) { String msgName = r.getMessage().getClass().getSimpleName().replace("Message", ""); - type = new MessageTypeMetricSet(msgName); + type = new MessageTypeMetricSet(msgName, timer); typeMap.put(r.getMessage().getType(), type); } diff --git a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/BenchmarkProgressPrinterTest.java b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/BenchmarkProgressPrinterTest.java index 6eba29fe9cb..d5244e97118 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespafeeder/BenchmarkProgressPrinterTest.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespafeeder/BenchmarkProgressPrinterTest.java @@ -20,11 +20,11 @@ public class BenchmarkProgressPrinterTest { ByteArrayOutputStream output = new ByteArrayOutputStream(); ManualTimer timer = new ManualTimer(); BenchmarkProgressPrinter printer = new BenchmarkProgressPrinter(timer, new PrintStream(output)); - RouteMetricSet metrics = new RouteMetricSet("foobar", printer); + RouteMetricSet metrics = new RouteMetricSet("foobar", timer, printer); { EmptyReply reply = new EmptyReply(); - reply.setMessage(PutDocumentMessage.createEmpty().setTimeReceived(1)); + reply.setMessage(PutDocumentMessage.createEmpty().setTimeReceived(-1)); metrics.addReply(reply); } @@ -32,13 +32,13 @@ public class BenchmarkProgressPrinterTest { { EmptyReply reply = new EmptyReply(); - reply.setMessage(PutDocumentMessage.createEmpty().setTimeReceived(2)); + reply.setMessage(PutDocumentMessage.createEmpty().setTimeReceived(-1)); metrics.addReply(reply); } { EmptyReply reply = new EmptyReply(); - reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(3)); + reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(-1)); metrics.addReply(reply); } @@ -46,7 +46,7 @@ public class BenchmarkProgressPrinterTest { { EmptyReply reply = new EmptyReply(); - reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(4)); + reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(-1)); reply.addError(new com.yahoo.messagebus.Error(32, "foo")); metrics.addReply(reply); } @@ -55,7 +55,7 @@ public class BenchmarkProgressPrinterTest { { EmptyReply reply = new EmptyReply(); - reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(5)); + reply.setMessage(UpdateDocumentMessage.createEmpty().setTimeReceived(-1)); reply.addError(new com.yahoo.messagebus.Error(64, "bar")); metrics.addReply(reply); } diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/SystemTimer.java b/vespajlib/src/main/java/com/yahoo/concurrent/SystemTimer.java index 8111d52a10f..c2fca806a85 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/SystemTimer.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/SystemTimer.java @@ -42,21 +42,18 @@ public enum SystemTimer implements Timer { SystemTimer() { long napTime = adjustTimeoutByDetectedHz(Duration.ofMillis(1)).toMillis(); - millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()); - Thread thread = new Thread() { - - @Override - public void run() { - while (true) { - millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()); - try { - Thread.sleep(napTime); - } catch (InterruptedException e) { - break; - } + long creationNanos = System.nanoTime(); + millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - creationNanos); + Thread thread = new Thread(() -> { + while (true) { + millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - creationNanos); + try { + Thread.sleep(napTime); + } catch (InterruptedException e) { + break; } } - }; + }); thread.setDaemon(true); thread.setName("vespa-system-timer"); thread.start(); diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/Timer.java b/vespajlib/src/main/java/com/yahoo/concurrent/Timer.java index c41c762c989..9328039aae6 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/Timer.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/Timer.java @@ -20,16 +20,8 @@ public interface Timer { * @return The current value of the timer, in milliseconds. */ long milliTime(); - Timer monotonic = () -> TimeUnit.NANOSECONDS.toMillis(System.nanoTime()); - static Timer wrap(Clock original) { - return new Timer() { - private final Clock clock = original; - - @Override - public long milliTime() { - return clock.millis(); - } - }; } - + long creationNanos = System.nanoTime(); // Avoid monotonic timer overflow for the first 146 years of JVM uptime. + Timer monotonic = () -> TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - creationNanos); + static Timer wrap(Clock original) { return original::millis; } default Instant instant() { return Instant.ofEpochMilli(milliTime()); } } diff --git a/vespajlib/src/main/java/com/yahoo/yolean/Exceptions.java b/vespajlib/src/main/java/com/yahoo/yolean/Exceptions.java index 4f3f048eb0c..c8564a9dac5 100644 --- a/vespajlib/src/main/java/com/yahoo/yolean/Exceptions.java +++ b/vespajlib/src/main/java/com/yahoo/yolean/Exceptions.java @@ -27,6 +27,7 @@ public class Exceptions { for (; t != null; t = t.getCause()) { message = getMessage(t); if (message == null) continue; + if (message.isEmpty()) continue; if (message.equals(lastMessage)) continue; if (b.length() > 0) { b.append(": "); diff --git a/vespalib/src/tests/util/rcuvector/rcuvector_test.cpp b/vespalib/src/tests/util/rcuvector/rcuvector_test.cpp index b842d009ce8..2c5af5c02ec 100644 --- a/vespalib/src/tests/util/rcuvector/rcuvector_test.cpp +++ b/vespalib/src/tests/util/rcuvector/rcuvector_test.cpp @@ -10,6 +10,7 @@ #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <random> +#include <thread> using namespace vespalib; @@ -20,6 +21,18 @@ using vespalib::makeLambdaTask; using MyMemoryAllocator = vespalib::alloc::test::MemoryAllocatorObserver; using AllocStats = MyMemoryAllocator::Stats; +namespace { + +void consider_yield(uint32_t i) +{ + if ((i % 1111) == 0) { + // Need to yield sometimes to avoid livelock when running unit test with valgrind + std::this_thread::yield(); + } +} + +} + bool assertUsage(const MemoryUsage & exp, const MemoryUsage & act) { @@ -452,12 +465,15 @@ StressFixture::read_work() std::mt19937 gen(rd()); std::uniform_int_distribution<uint32_t> distrib(0, read_area - 1); std::vector<int> old(read_area); + uint32_t i = 0; while (!stop_read.load(std::memory_order_relaxed)) { uint32_t idx = distrib(gen); auto guard = generation_handler.takeGuard(); int value = arr.acquire_elem_ref(idx).load_acquire(); EXPECT_LE(old[idx], value); old[idx] = value; + consider_yield(i); + ++i; } } @@ -478,6 +494,7 @@ StressFixture::write_work(uint32_t cnt) uint32_t idx = distrib(gen); arr[idx].store_release(arr[idx].load_relaxed() + 1); commit(); + consider_yield(i); } } |