diff options
147 files changed, 566 insertions, 513 deletions
diff --git a/.gitignore b/.gitignore index adc898a7266..c5ebe0aef49 100644 --- a/.gitignore +++ b/.gitignore @@ -34,12 +34,14 @@ CTestTestfile.cmake cmake_install.cmake Makefile Testing +/.cmake/ /.mvn/ /.ninja_deps /.ninja_log /build.ninja /rules.ninja *_test_app +/cmake-build-debug/ /mvnw /mvnw.cmd /mvnwDebug diff --git a/client/go/internal/vespa/document/dispatcher_test.go b/client/go/internal/vespa/document/dispatcher_test.go index 7ef2043f9c5..252bd94dff9 100644 --- a/client/go/internal/vespa/document/dispatcher_test.go +++ b/client/go/internal/vespa/document/dispatcher_test.go @@ -61,8 +61,8 @@ func TestDispatcher(t *testing.T) { breaker := NewCircuitBreaker(time.Second, 0) dispatcher := NewDispatcher(feeder, throttler, breaker, io.Discard, false) docs := []Document{ - {Id: mustParseId("id:ns:type::doc1"), Operation: OperationPut, Body: []byte(`{"fields":{"foo": "123"}}`)}, - {Id: mustParseId("id:ns:type::doc2"), Operation: OperationPut, Body: []byte(`{"fields":{"bar": "456"}}`)}, + {Id: mustParseId("id:ns:type::doc1"), Operation: OperationPut, Fields: []byte(`{"foo": "123"}`)}, + {Id: mustParseId("id:ns:type::doc2"), Operation: OperationPut, Fields: []byte(`{"bar": "456"}`)}, } for _, d := range docs { dispatcher.Enqueue(d) @@ -192,7 +192,7 @@ func BenchmarkDocumentDispatching(b *testing.B) { throttler := newThrottler(8, clock.now) breaker := NewCircuitBreaker(time.Second, 0) dispatcher := NewDispatcher(feeder, throttler, breaker, io.Discard, false) - doc := Document{Id: mustParseId("id:ns:type::doc1"), Operation: OperationPut, Body: []byte(`{"fields":{"foo": "123"}}`)} + doc := Document{Id: mustParseId("id:ns:type::doc1"), Operation: OperationPut, Fields: []byte(`{"foo": "123"}`)} b.ResetTimer() // ignore setup time for n := 0; n < b.N; n++ { diff --git a/client/go/internal/vespa/document/document.go b/client/go/internal/vespa/document/document.go index 8bc897d49b3..101761e01d0 100644 --- a/client/go/internal/vespa/document/document.go +++ b/client/go/internal/vespa/document/document.go @@ -93,10 +93,10 @@ func ParseId(serialized string) (Id, error) { // Document represents a Vespa document operation. type Document struct { Id Id - Operation Operation Condition string + Fields []byte + Operation Operation Create bool - Body []byte } type jsonDocument struct { @@ -105,8 +105,8 @@ type jsonDocument struct { UpdateId string `json:"update"` RemoveId string `json:"remove"` Condition string `json:"condition"` - Create bool `json:"create"` Fields json.RawMessage `json:"fields"` + Create bool `json:"create"` } // Decoder decodes documents from a JSON structure which is either an array of objects, or objects separated by newline. @@ -235,20 +235,12 @@ func parseDocument(d *jsonDocument) (Document, error) { if err != nil { return Document{}, err } - var body []byte - if d.Fields != nil { - jsonObject := `{"fields":` - body = make([]byte, 0, len(jsonObject)+len(d.Fields)+1) - body = append(body, []byte(jsonObject)...) - body = append(body, d.Fields...) - body = append(body, byte('}')) - } return Document{ Id: docId, Operation: op, Condition: d.Condition, Create: d.Create, - Body: body, + Fields: d.Fields, }, nil } diff --git a/client/go/internal/vespa/document/document_test.go b/client/go/internal/vespa/document/document_test.go index bdf18586753..495f3963322 100644 --- a/client/go/internal/vespa/document/document_test.go +++ b/client/go/internal/vespa/document/document_test.go @@ -134,8 +134,8 @@ func testDocumentDecoder(t *testing.T, jsonLike string) { t.Helper() r := NewDecoder(strings.NewReader(jsonLike)) want := []Document{ - {Id: mustParseId("id:ns:type::doc1"), Operation: OperationPut, Body: []byte(`{"fields":{"foo": "123"}}`)}, - {Id: mustParseId("id:ns:type::doc2"), Operation: OperationPut, Body: []byte(`{"fields":{"bar": "456"}}`)}, + {Id: mustParseId("id:ns:type::doc1"), Operation: OperationPut, Fields: []byte(`{"foo": "123"}`)}, + {Id: mustParseId("id:ns:type::doc2"), Operation: OperationPut, Fields: []byte(`{"bar": "456"}`)}, {Id: mustParseId("id:ns:type::doc1"), Operation: OperationRemove}, } got := []Document{} diff --git a/client/go/internal/vespa/document/feeder.go b/client/go/internal/vespa/document/feeder.go index 761f76398c4..6bcd4afe916 100644 --- a/client/go/internal/vespa/document/feeder.go +++ b/client/go/internal/vespa/document/feeder.go @@ -21,13 +21,13 @@ const ( // Result represents the result of a feeding operation. type Result struct { + Err error Id Id - Status Status - HTTPStatus int Message string Trace string - Err error Stats Stats + Status Status + HTTPStatus int } func (r Result) Success() bool { @@ -36,9 +36,9 @@ func (r Result) Success() bool { // Stats represents feeding operation statistics. type Stats struct { + ResponsesByCode map[int]int64 Requests int64 Responses int64 - ResponsesByCode map[int]int64 Errors int64 Inflight int64 TotalLatency time.Duration diff --git a/client/go/internal/vespa/document/http.go b/client/go/internal/vespa/document/http.go index 3581a791dbe..e58f0f0fd3d 100644 --- a/client/go/internal/vespa/document/http.go +++ b/client/go/internal/vespa/document/http.go @@ -27,6 +27,11 @@ const ( CompressionGzip ) +var ( + fieldsPrefix = []byte(`{"fields":`) + fieldsSuffix = []byte("}") +) + // Client represents a HTTP client for the /document/v1/ API. type Client struct { options ClientOptions @@ -179,13 +184,19 @@ func (c *Client) buffer() *bytes.Buffer { } func (c *Client) createRequest(method, url string, body []byte) (*http.Request, error) { - var r io.Reader + // include the outer object expected by /document/v1/ without copying the body + r := io.MultiReader( + bytes.NewReader(fieldsPrefix), + bytes.NewReader(body), + bytes.NewReader(fieldsSuffix), + ) + contentLength := int64(len(fieldsPrefix) + len(body) + len(fieldsSuffix)) useGzip := c.options.Compression == CompressionGzip || (c.options.Compression == CompressionAuto && len(body) > 512) if useGzip { var buf bytes.Buffer buf.Grow(1024) w := c.gzipWriter(&buf) - if _, err := w.Write(body); err != nil { + if _, err := io.Copy(w, r); err != nil { return nil, err } if err := w.Close(); err != nil { @@ -193,8 +204,7 @@ func (c *Client) createRequest(method, url string, body []byte) (*http.Request, } c.gzippers.Put(w) r = &buf - } else { - r = bytes.NewReader(body) + contentLength = int64(buf.Len()) } req, err := http.NewRequest(method, url, r) if err != nil { @@ -204,6 +214,7 @@ func (c *Client) createRequest(method, url string, body []byte) (*http.Request, req.Header.Set("Content-Encoding", "gzip") } req.Header.Set("Content-Type", "application/json; charset=utf-8") + req.ContentLength = contentLength return req, nil } @@ -219,7 +230,7 @@ func (c *Client) Send(document Document) Result { start := c.now() result := Result{Id: document.Id, Stats: Stats{Requests: 1}} method, url := c.methodAndURL(document) - req, err := c.createRequest(method, url, document.Body) + req, err := c.createRequest(method, url, document.Fields) if err != nil { return resultWithErr(result, err) } @@ -229,7 +240,7 @@ func (c *Client) Send(document Document) Result { } defer resp.Body.Close() elapsed := c.now().Sub(start) - return c.resultWithResponse(resp, result, document, elapsed) + return c.resultWithResponse(resp, req.ContentLength, result, elapsed) } func resultWithErr(result Result, err error) Result { @@ -239,7 +250,7 @@ func resultWithErr(result Result, err error) Result { return result } -func (c *Client) resultWithResponse(resp *http.Response, result Result, document Document, elapsed time.Duration) Result { +func (c *Client) resultWithResponse(resp *http.Response, sentBytes int64, result Result, elapsed time.Duration) Result { result.HTTPStatus = resp.StatusCode result.Stats.Responses++ result.Stats.ResponsesByCode = map[int]int64{resp.StatusCode: 1} @@ -271,7 +282,7 @@ func (c *Client) resultWithResponse(resp *http.Response, result Result, document } result.Message = body.Message result.Trace = string(body.Trace) - result.Stats.BytesSent = int64(len(document.Body)) + result.Stats.BytesSent = sentBytes result.Stats.BytesRecv = int64(written) if !result.Success() { result.Stats.Errors++ diff --git a/client/go/internal/vespa/document/http_test.go b/client/go/internal/vespa/document/http_test.go index 489460b3ed7..928a457568e 100644 --- a/client/go/internal/vespa/document/http_test.go +++ b/client/go/internal/vespa/document/http_test.go @@ -59,9 +59,9 @@ func assertLeastBusy(t *testing.T, id int, client *Client) { func TestClientSend(t *testing.T) { docs := []Document{ - {Create: true, Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Body: []byte(`{"fields":{"foo": "123"}}`)}, - {Create: true, Id: mustParseId("id:ns:type::doc2"), Operation: OperationUpdate, Body: []byte(`{"fields":{"foo": "456"}}`)}, - {Create: true, Id: mustParseId("id:ns:type::doc3"), Operation: OperationUpdate, Body: []byte(`{"fields":{"baz": "789"}}`)}, + {Create: true, Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Fields: []byte(`{"foo": "123"}`)}, + {Create: true, Id: mustParseId("id:ns:type::doc2"), Operation: OperationUpdate, Fields: []byte(`{"foo": "456"}`)}, + {Create: true, Id: mustParseId("id:ns:type::doc3"), Operation: OperationUpdate, Fields: []byte(`{"baz": "789"}`)}, } httpClient := mock.HTTPClient{} client, _ := NewClient(ClientOptions{ @@ -116,9 +116,15 @@ func TestClientSend(t *testing.T) { if err != nil { t.Fatalf("got unexpected error %q", err) } - wantBody := doc.Body - if !bytes.Equal(body, wantBody) { - t.Errorf("got r.Body = %q, want %q", string(body), string(wantBody)) + var wantBody bytes.Buffer + wantBody.WriteString(`{"fields":`) + wantBody.Write(doc.Fields) + wantBody.WriteString("}") + if !bytes.Equal(body, wantBody.Bytes()) { + t.Errorf("got r.Body = %q, want %q", string(body), wantBody.String()) + } + if r.ContentLength != int64(len(body)) { + t.Errorf("got r.ContentLength=%d, want %d", r.ContentLength, len(body)) } } want := Stats{ @@ -148,9 +154,9 @@ func TestClientSendCompressed(t *testing.T) { Timeout: time.Duration(5 * time.Second), }, []util.HTTPClient{&httpClient}) - bigBody := fmt.Sprintf(`{"fields":{"foo": "%s"}}`, strings.Repeat("s", 512+1)) - bigDoc := Document{Create: true, Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Body: []byte(bigBody)} - smallDoc := Document{Create: true, Id: mustParseId("id:ns:type::doc2"), Operation: OperationUpdate, Body: []byte(`{"fields":{"foo": "s"}}`)} + bigBody := fmt.Sprintf(`{"foo": "%s"}`, strings.Repeat("s", 512+1)) + bigDoc := Document{Create: true, Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Fields: []byte(bigBody)} + smallDoc := Document{Create: true, Id: mustParseId("id:ns:type::doc2"), Operation: OperationUpdate, Fields: []byte(`{"foo": "s"}`)} client.options.Compression = CompressionNone _ = client.Send(bigDoc) @@ -184,6 +190,9 @@ func assertCompressedRequest(t *testing.T, want bool, request *http.Request) { if err != nil { t.Fatal(err) } + if request.ContentLength != int64(len(body)) { + t.Errorf("got ContentLength=%d, want %d", request.ContentLength, len(body)) + } compressed := bytes.HasPrefix(body, []byte{0x1f, 0x8b}) if compressed != want { t.Errorf("got compressed=%t, want %t", compressed, want) @@ -289,12 +298,12 @@ func benchmarkClientSend(b *testing.B, compression Compression, document Documen } func BenchmarkClientSend(b *testing.B) { - doc := Document{Create: true, Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Body: []byte(`{"fields":{"foo": "my document"}}`)} + doc := Document{Create: true, Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Fields: []byte(`{"foo": "my document"}`)} benchmarkClientSend(b, CompressionNone, doc) } func BenchmarkClientSendCompressed(b *testing.B) { - body := fmt.Sprintf(`{"fields":{"foo": "%s"}}`, strings.Repeat("my document", 100)) - doc := Document{Create: true, Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Body: []byte(body)} + body := fmt.Sprintf(`{"foo": "%s"}`, strings.Repeat("my document", 100)) + doc := Document{Create: true, Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Fields: []byte(body)} benchmarkClientSend(b, CompressionGzip, doc) } diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index a8b836d5846..7f7624b1b2f 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -134,9 +134,10 @@ <include>com.yahoo.vespa:jdisc_core:*:provided</include> <include>com.yahoo.vespa:jrt:*:provided</include> <include>com.yahoo.vespa:linguistics:*:provided</include> - <include>com.yahoo.vespa:opennlp-linguistics:*:provided</include> <include>com.yahoo.vespa:messagebus:*:provided</include> + <include>com.yahoo.vespa:metrics:*:provided</include> <include>com.yahoo.vespa:model-evaluation:*:provided</include> + <include>com.yahoo.vespa:opennlp-linguistics:*:provided</include> <include>com.yahoo.vespa:predicate-search-core:*:provided</include> <include>com.yahoo.vespa:provided-dependencies:*:provided</include> <include>com.yahoo.vespa:searchcore:*:provided</include> @@ -162,7 +163,6 @@ <include>com.yahoo.vespa:indexinglanguage:*:test</include> <include>com.yahoo.vespa:logd:*:test</include> <include>com.yahoo.vespa:metrics-proxy:*:test</include> - <include>com.yahoo.vespa:metrics:*:test</include> <include>com.yahoo.vespa:model-integration:*:test</include> <include>com.yahoo.vespa:searchsummary:*:test</include> <include>com.yahoo.vespa:standalone-container:*:test</include> diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterEvent.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterEvent.java index 1853a024e11..f7b5b5ee523 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterEvent.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterEvent.java @@ -28,10 +28,6 @@ public class ClusterEvent implements Event{ return description; } - public Type getType() { - return type; - } - @Override public String toString() { return "Cluster event type " + type + " @" + timeMs + ": " + description; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundle.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundle.java index e55f3d4592b..7062f67830b 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundle.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundle.java @@ -3,7 +3,14 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.vdslib.state.ClusterState; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; import java.util.function.Function; import java.util.stream.Collectors; @@ -194,12 +201,6 @@ public class ClusterStateBundle { public static ClusterStateBundle of(AnnotatedClusterState baselineState, Map<String, AnnotatedClusterState> derivedBucketSpaceStates, - boolean deferredActivation) { - return new ClusterStateBundle(baselineState, derivedBucketSpaceStates, null, deferredActivation); - } - - public static ClusterStateBundle of(AnnotatedClusterState baselineState, - Map<String, AnnotatedClusterState> derivedBucketSpaceStates, FeedBlock feedBlock, boolean deferredActivation) { return new ClusterStateBundle(baselineState, derivedBucketSpaceStates, feedBlock, deferredActivation); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/DistributorNodeInfo.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/DistributorNodeInfo.java index fb9a4cf4604..24e3d0de977 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/DistributorNodeInfo.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/DistributorNodeInfo.java @@ -4,8 +4,6 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.vdslib.distribution.Distribution; import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeType; -import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo; -import com.yahoo.vespa.clustercontroller.core.hostinfo.StorageNodeStatsBridge; /** * Class encapsulating what the Cluster Controller knows about a distributor node. Most of the information is diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventLog.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventLog.java index 0058d12f640..e64e684ed70 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventLog.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventLog.java @@ -15,7 +15,7 @@ import java.util.logging.Logger; public class EventLog implements EventLogInterface { - public static Logger log = Logger.getLogger(EventLog.class.getName()); + public static final Logger log = Logger.getLogger(EventLog.class.getName()); private final Timer timer; private final LinkedList<Event> eventLog = new LinkedList<>(); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventLogInterface.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventLogInterface.java index 382025bb67b..e4538162f64 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventLogInterface.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventLogInterface.java @@ -3,8 +3,6 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.vdslib.state.Node; -import java.util.List; - public interface EventLogInterface { void add(Event e); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index ff0d8b87e92..7a4934fe175 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -257,7 +257,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta public int getRpcPort() { return rpcServer.getPort(); } - public void shutdown() throws InterruptedException, java.io.IOException { + public void shutdown() throws InterruptedException { if (runner != null && isRunning()) { context.log(logger, Level.INFO, "Joining event thread."); running.set(false); @@ -393,14 +393,8 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta } private void storeClusterStateMetaDataToZooKeeper(ClusterStateBundle stateBundle) { - try { - database.saveLatestSystemStateVersion(databaseContext, stateBundle.getVersion()); - database.saveLatestClusterStateBundle(databaseContext, stateBundle); - } catch (InterruptedException e) { - // Rethrow as RuntimeException to propagate exception up to main thread method. - // Don't want to hide failures to write cluster state version. - throw new RuntimeException("ZooKeeper write interrupted", e); - } + database.saveLatestSystemStateVersion(databaseContext, stateBundle.getVersion()); + database.saveLatestClusterStateBundle(databaseContext, stateBundle); } /** @@ -408,7 +402,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta * The keys in the given map are indices of fleet controllers. * The values are what fleetcontroller that fleetcontroller wants to * become master. - * + * <p> * If more than half the fleetcontrollers want a node to be master and * that node also wants itself as master, that node is the single master. * If this condition is not met, there is currently no master. @@ -450,7 +444,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta } /** Called when all distributors have acked newest cluster state version. */ - public void handleAllDistributorsInSync(DatabaseHandler database, DatabaseHandler.DatabaseContext dbContext) throws InterruptedException { + public void handleAllDistributorsInSync(DatabaseHandler database, DatabaseHandler.DatabaseContext dbContext) { Set<ConfiguredNode> nodes = new HashSet<>(cluster.clusterInfo().getConfiguredNodes().values()); // TODO wouldn't it be better to always get bundle information from the state broadcaster? var currentBundle = stateVersionTracker.getVersionedClusterStateBundle(); @@ -512,12 +506,6 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta rpcServer.setSlobrokConnectionSpecs(options.slobrokConnectionSpecs(), options.rpcPort()); } - try { - statusPageServer.setPort(options.httpPort()); - } catch (Exception e) { - context.log(logger, Level.WARNING, "Failed to initialize status server socket. This may be natural if cluster has altered the services running on this node: " + e.getMessage()); - } - long currentTime = timer.getCurrentTimeInMillis(); nextStateSendTime = Math.min(currentTime + options.minTimeBetweenNewSystemStates(), nextStateSendTime); } @@ -632,8 +620,6 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta private boolean updateMasterElectionState() { try { return masterElectionHandler.watchMasterElection(database, databaseContext); - } catch (InterruptedException e) { - throw new RuntimeException(e); } catch (Exception e) { context.log(logger, Level.WARNING, "Failed to watch master election: " + e); } @@ -1116,7 +1102,7 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta synchronized (monitor) { monitor.notifyAll(); } } - public DatabaseHandler.DatabaseContext databaseContext = new DatabaseHandler.DatabaseContext() { + public final DatabaseHandler.DatabaseContext databaseContext = new DatabaseHandler.DatabaseContext() { @Override public ContentCluster getCluster() { return cluster; } @Override @@ -1207,10 +1193,6 @@ public class FleetController implements NodeListener, SlobrokListener, SystemSta public ContentCluster getCluster() { return cluster; } - public EventLog getEventLog() { - return eventLog; - } - public StatusHandler.ContainerStatusPageServer statusPageServer() { return statusPageServer; } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java index 21ccfb2750e..1541e1a4218 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetControllerOptions.java @@ -469,10 +469,6 @@ public class FleetControllerOptions { return this; } - public String[] slobrokConnectionSpecs() { - return slobrokConnectionSpecs; - } - public Builder setSlobrokConnectionSpecs(String[] slobrokConnectionSpecs) { Objects.requireNonNull(slobrokConnectionSpecs, "slobrokConnectionSpecs cannot be null"); this.slobrokConnectionSpecs = slobrokConnectionSpecs; @@ -664,10 +660,6 @@ public class FleetControllerOptions { return this; } - public double minMergeCompletionRatio() { - return minMergeCompletionRatio; - } - public Builder setMinMergeCompletionRatio(double minMergeCompletionRatio) { this.minMergeCompletionRatio = minMergeCompletionRatio; return this; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java index dfc328346bb..b041e6b14f8 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java @@ -120,8 +120,7 @@ public class MasterElectionHandler implements MasterInterface { public boolean isAmongNthFirst(int first) { return (nextInLineCount < first); } - public boolean watchMasterElection(DatabaseHandler database, - DatabaseHandler.DatabaseContext dbContext) throws InterruptedException { + public boolean watchMasterElection(DatabaseHandler database, DatabaseHandler.DatabaseContext dbContext) { if (totalCount == 1 && !usingZooKeeper) { return false; // Allow single configured node to become master implicitly if no ZK configured } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java index d7aac1c26fa..069139b8c9e 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java @@ -241,8 +241,6 @@ abstract public class NodeInfo implements Comparable<NodeInfo> { return node.getIndex(); } - public ContentCluster getCluster() { return cluster; } - /** Returns true if the node is registered in slobrok */ public boolean isInSlobrok() { return lastSeenInSlobrok == null; } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java index 0efba56284b..72bd5148f11 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java @@ -14,11 +14,10 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.logging.Logger; -import java.util.stream.Collectors; public class SystemStateBroadcaster { - private static Logger log = Logger.getLogger(SystemStateBroadcaster.class.getName()); + private static final Logger log = Logger.getLogger(SystemStateBroadcaster.class.getName()); private final FleetControllerContext context; private final Timer timer; @@ -261,7 +260,7 @@ public class SystemStateBroadcaster { lastStateVersionBundleAcked = clusterStateBundle.getVersion(); } - private void markCurrentClusterStateAsConverged(DatabaseHandler database, DatabaseHandler.DatabaseContext dbContext, FleetController fleetController) throws InterruptedException { + private void markCurrentClusterStateAsConverged(DatabaseHandler database, DatabaseHandler.DatabaseContext dbContext, FleetController fleetController) { context.log(log, Level.FINE, "All distributors have newest clusterstate, updating start timestamps in zookeeper and clearing them from cluster state"); lastClusterStateVersionConverged = clusterStateBundle.getVersion(); lastClusterStateBundleConverged = clusterStateBundle; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java index 408c10e81b1..efb97a4a69e 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java @@ -91,8 +91,11 @@ public class DatabaseHandler { private boolean lostZooKeeperConnectionEvent = false; private Map<Integer, Integer> masterDataEvent = null; - public DatabaseHandler(FleetControllerContext fleetControllerContext, DatabaseFactory databaseFactory, Timer timer, String zooKeeperAddress, Object monitor) throws InterruptedException - { + public DatabaseHandler(FleetControllerContext fleetControllerContext, + DatabaseFactory databaseFactory, + Timer timer, + String zooKeeperAddress, + Object monitor) { this.fleetControllerContext = fleetControllerContext; this.databaseFactory = databaseFactory; this.timer = timer; @@ -356,7 +359,7 @@ public class DatabaseHandler { } } - public void saveLatestSystemStateVersion(DatabaseContext databaseContext, int version) throws InterruptedException { + public void saveLatestSystemStateVersion(DatabaseContext databaseContext, int version) { fleetControllerContext.log(logger, Level.FINE, () -> "Checking if latest system state version has been updated and need to be stored."); // Schedule a write if one of the following is true: // - There is already a pending vote to be written, that may have been written already without our knowledge diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java index ea745a56066..042d091fdbb 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java @@ -245,7 +245,7 @@ public class ZooKeeperDatabase extends Database { if (!toStore.equals(nodeState)) { log.warning("Attempted to store wanted state with more than just a main state. Extra data stripped. Original data '" + nodeState.serialize(true)); } - sb.append(node.toString()).append(':').append(toStore.serialize(true)).append('\n'); + sb.append(node).append(':').append(toStore.serialize(true)).append('\n'); } } byte[] val = sb.toString().getBytes(utf8); @@ -299,7 +299,7 @@ public class ZooKeeperDatabase extends Database { Long timestamp = timestamps.get(n); sb.append(n.toString()).append(':').append(timestamp).append('\n'); } - byte val[] = sb.toString().getBytes(utf8); + byte[] val = sb.toString().getBytes(utf8); try{ context.log(log, Level.FINE, () -> "Storing start timestamps at '" + paths.startTimestamps() + "'"); session.setData(paths.startTimestamps(), val, -1); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/ContentNode.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/ContentNode.java index 945745b7ff5..45f67c09ac4 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/ContentNode.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/ContentNode.java @@ -13,7 +13,7 @@ import java.util.Optional; */ public class ContentNode { @JsonProperty("resource-usage") - private Map<String, ResourceUsage> resourceUsage = new HashMap<>(); + private final Map<String, ResourceUsage> resourceUsage = new HashMap<>(); public Map<String, ResourceUsage> getResourceUsage() { return Collections.unmodifiableMap(resourceUsage); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfo.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfo.java index 1a11e32c9b8..9314eb61bf2 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfo.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/HostInfo.java @@ -16,7 +16,7 @@ import java.util.logging.Logger; */ public class HostInfo { - private static Logger log = Logger.getLogger(HostInfo.class.getName()); + private static final Logger log = Logger.getLogger(HostInfo.class.getName()); private static final ObjectMapper mapper = new ObjectMapper(); private String rawCreationString = "NOT SET"; static { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNodeStatsBridge.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNodeStatsBridge.java index 73f504b6004..aaaf44c3f3f 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNodeStatsBridge.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/hostinfo/StorageNodeStatsBridge.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.clustercontroller.core.hostinfo; import com.yahoo.vespa.clustercontroller.core.*; import java.util.HashMap; -import java.util.List; import java.util.Map; /** diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/VerboseReport.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/VerboseReport.java deleted file mode 100644 index c18afb811fa..00000000000 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/VerboseReport.java +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.clustercontroller.core.restapiv2.requests; - -public enum VerboseReport { - STATISTICS -} diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCActivateClusterStateVersionWaiter.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCActivateClusterStateVersionWaiter.java index fe441a0eb91..1e7591a9aaa 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCActivateClusterStateVersionWaiter.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCActivateClusterStateVersionWaiter.java @@ -7,7 +7,6 @@ import com.yahoo.jrt.RequestWaiter; import com.yahoo.vespa.clustercontroller.core.ActivateClusterStateVersionRequest; import com.yahoo.vespa.clustercontroller.core.Communicator; import com.yahoo.vespa.clustercontroller.core.NodeInfo; -import com.yahoo.vespa.clustercontroller.core.Timer; /** * Binds together the reply received for a particular cluster state activation RPC and diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java index 58bbd03044f..fc2a8eda7e9 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java @@ -135,7 +135,7 @@ public class RPCCommunicator implements Communicator { @Override public void setSystemState(ClusterStateBundle stateBundle, NodeInfo node, Waiter<SetClusterStateRequest> externalWaiter) { - RPCSetClusterStateWaiter waiter = new RPCSetClusterStateWaiter(externalWaiter, timer); + RPCSetClusterStateWaiter waiter = new RPCSetClusterStateWaiter(externalWaiter); ClusterState baselineState = stateBundle.getBaselineClusterState(); Target connection = getConnection(node); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCSetClusterStateWaiter.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCSetClusterStateWaiter.java index 41fc7f67fbb..eaeec8abae7 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCSetClusterStateWaiter.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCSetClusterStateWaiter.java @@ -7,7 +7,6 @@ import com.yahoo.jrt.RequestWaiter; import com.yahoo.vespa.clustercontroller.core.Communicator; import com.yahoo.vespa.clustercontroller.core.NodeInfo; import com.yahoo.vespa.clustercontroller.core.SetClusterStateRequest; -import com.yahoo.vespa.clustercontroller.core.Timer; /** * Waiter class for set cluster state RPC commands. @@ -15,11 +14,9 @@ import com.yahoo.vespa.clustercontroller.core.Timer; public class RPCSetClusterStateWaiter implements RequestWaiter { SetClusterStateRequest request; - Timer timer; Communicator.Waiter<SetClusterStateRequest> waiter; - public RPCSetClusterStateWaiter(Communicator.Waiter<SetClusterStateRequest> waiter, Timer timer) { - this.timer = timer; + public RPCSetClusterStateWaiter(Communicator.Waiter<SetClusterStateRequest> waiter) { this.waiter = waiter; } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/RunDataExtractor.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/RunDataExtractor.java deleted file mode 100644 index 599931ba13c..00000000000 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/RunDataExtractor.java +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.clustercontroller.core.status; - -import com.yahoo.vespa.clustercontroller.core.ContentCluster; -import com.yahoo.vespa.clustercontroller.core.FleetControllerOptions; - -/** - * @author Haakon Humberset - */ -public interface RunDataExtractor { - - FleetControllerOptions getOptions(); - ContentCluster getCluster(); - -} diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/StatusHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/StatusHandler.java index 302832e4542..a8a858b5d31 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/StatusHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/StatusHandler.java @@ -6,7 +6,6 @@ import com.yahoo.vespa.clustercontroller.core.status.statuspage.StatusPageServer import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequest; import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequestHandler; import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpResult; -import java.io.IOException; import java.io.StringWriter; import java.nio.charset.StandardCharsets; import java.util.Map; @@ -35,8 +34,7 @@ public class StatusHandler implements HttpRequestHandler { private final Object answerMonitor = new Object(); public int getPort() { return 0; } - public void shutdown() throws InterruptedException, IOException {} - public void setPort(int port) {} + public void shutdown() {} public StatusPageServer.HttpRequest getCurrentHttpRequest() { synchronized (answerMonitor) { StatusPageServer.HttpRequest r = request; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/HtmlTable.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/HtmlTable.java index af059809815..b2c12cbbdcf 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/HtmlTable.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/HtmlTable.java @@ -10,7 +10,8 @@ public class HtmlTable { int border = 1; int cellSpacing = 0; - enum Orientation { LEFT, CENTER, RIGHT }; + enum Orientation { LEFT, CENTER, RIGHT } + public static class CellProperties { Integer backgroundColor; Integer colSpan; // Colspan 0 indicate rest of table @@ -59,7 +60,7 @@ public class HtmlTable { Row addProperties(CellProperties p) { rowProperties.add(p); return this; } } - private ArrayList<Row> cells = new ArrayList<Row>(); + private final ArrayList<Row> cells = new ArrayList<Row>(); public HtmlTable() { } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageResponse.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageResponse.java index 87aa1fc497c..0701403732a 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageResponse.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageResponse.java @@ -10,10 +10,9 @@ import java.io.OutputStreamWriter; public class StatusPageResponse { - private ByteArrayOutputStream output = new ByteArrayOutputStream(); + private final ByteArrayOutputStream output = new ByteArrayOutputStream(); private String contentType; private ResponseCode responseCode = ResponseCode.OK; - private boolean clientCachingEnabled = false; public enum ResponseCode { OK(200, "OK"), @@ -61,14 +60,6 @@ public class StatusPageResponse { this.responseCode = responseCode; } - public boolean isClientCachingEnabled() { - return clientCachingEnabled; - } - - public void setClientCachingEnabled(boolean clientCachingEnabled) { - this.clientCachingEnabled = clientCachingEnabled; - } - public void writeHtmlHeader(StringBuilder content, String title) { String escaped_title = HtmlEscapers.htmlEscaper().escape(title); content.append("<html>\n") diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java index 1bd011993a0..ec5760181e5 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java @@ -24,7 +24,7 @@ public class StatusPageServer { private final String request; private String pathPrefix = ""; private final Map<String, String> params = new HashMap<String, String>(); - private String path; + private final String path; static Pattern pathPattern; static { @@ -112,7 +112,7 @@ public class StatusPageServer { } } - private List<PatternRouting> patterns = new ArrayList<>(); + private final List<PatternRouting> patterns = new ArrayList<>(); public void addHandler(Pattern pattern, RequestHandler handler) { patterns.add(new PatternRouting(pattern, handler)); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRenderer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRenderer.java index 547647e82e6..b350467c284 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRenderer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/VdsClusterHtmlRenderer.java @@ -58,7 +58,7 @@ public class VdsClusterHtmlRenderer { public void addTable(final StringBuilder destination, final long stableStateTimePeriode) { destination.append(contentBuilder); - destination.append(table.toString()) + destination.append(table) .append("<p>") .append("<p>"); addFooter(destination, stableStateTimePeriode); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFixture.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFixture.java index eccdb4d7832..6855f771258 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFixture.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterFixture.java @@ -19,14 +19,12 @@ import static org.mockito.Mockito.mock; public class ClusterFixture { public final ContentCluster cluster; - public final Distribution distribution; public final FakeTimer timer; final StateChangeHandler nodeStateChangeHandler; private final ClusterStateGenerator.Params params = new ClusterStateGenerator.Params(); - public ClusterFixture(ContentCluster cluster, Distribution distribution) { + public ClusterFixture(ContentCluster cluster) { this.cluster = cluster; - this.distribution = distribution; this.timer = new FakeTimer(); var context = new FleetControllerContextImpl(new FleetControllerId(cluster.getName(), 0)); this.nodeStateChangeHandler = new StateChangeHandler(context, timer, mock(EventLogInterface.class)); @@ -184,7 +182,7 @@ public class ClusterFixture { Distribution distribution = DistributionBuilder.forFlatCluster(nodeCount); ContentCluster cluster = new ContentCluster("foo", nodes, distribution); - return new ClusterFixture(cluster, distribution); + return new ClusterFixture(cluster); } static ClusterFixture forHierarchicCluster(DistributionBuilder.GroupBuilder root) { @@ -192,7 +190,7 @@ public class ClusterFixture { Distribution distribution = DistributionBuilder.forHierarchicCluster(root); ContentCluster cluster = new ContentCluster("foo", nodes, distribution); - return new ClusterFixture(cluster, distribution); + return new ClusterFixture(cluster); } ClusterStateGenerator.Params generatorParams() { diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundleTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundleTest.java index 17c461a487b..930040b0143 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundleTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ClusterStateBundleTest.java @@ -244,7 +244,6 @@ public class ClusterStateBundleTest { @Test void cloning_preserves_feed_block_state() { var bundle = createTestBundleWithFeedBlock("foo"); - ; var derived = bundle.cloneWithMapper(Function.identity()); assertEquals(bundle, derived); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendererTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendererTest.java index 31432d561d0..8d5c2f685f8 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendererTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ContentClusterHtmlRendererTest.java @@ -10,8 +10,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.io.IOException; -import java.io.StringWriter; -import java.io.Writer; import java.util.Collections; import java.util.Map; import java.util.TreeMap; diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseHandlerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseHandlerTest.java index c07a29b3c1e..71ba160a314 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseHandlerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DatabaseHandlerTest.java @@ -45,7 +45,7 @@ public class DatabaseHandlerTest { final Object monitor = new Object(); final ClusterStateBundle dummyBundle; - Fixture() throws Exception { + Fixture() { dummyBundle = ClusterStateBundleUtil.makeBundle("distributor:2 storage:2", StateMapping.of("default", "distributor:2 storage:2 .0.s:d"), StateMapping.of("upsidedown", "distributor:2 .0.s:d storage:2")); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyCommunicator.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyCommunicator.java index b82d4a135ae..682e36254c9 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyCommunicator.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyCommunicator.java @@ -25,18 +25,15 @@ public class DummyCommunicator implements Communicator, NodeLookup { } static class DummyGetNodeStateRequest extends GetNodeStateRequest { - Waiter<GetNodeStateRequest> waiter; + final Waiter<GetNodeStateRequest> waiter; DummyGetNodeStateRequest(NodeInfo nodeInfo, Waiter<GetNodeStateRequest> waiter) { super(nodeInfo); - this.waiter = waiter; } @Override - public void abort() { - - } + public void abort() {} } public static class DummySetClusterStateRequest extends SetClusterStateRequest { diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java index f2494ea0301..df7357f31da 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java @@ -61,14 +61,15 @@ public class DummyVdsNode { private boolean registeredInSlobrok = false; static class Req { - Request request; - long timeToReply; + final Request request; + final long timeToReply; Req(Request r, long timeToReply) { request = r; this.timeToReply = timeToReply; } } + static class BackOff implements BackOffPolicy { public void reset() {} public double get() { return 0.01; } @@ -118,7 +119,7 @@ public class DummyVdsNode { }; public DummyVdsNode(Timer timer, String[] slobrokConnectionSpecs, String clusterName, - NodeType nodeType, int index) throws Exception { + NodeType nodeType, int index) { this.timer = timer; this.slobrokConnectionSpecs = slobrokConnectionSpecs; this.clusterName = clusterName; @@ -249,12 +250,6 @@ public class DummyVdsNode { setNodeState(new NodeState(type, state)); } - public NodeState getNodeState() { - synchronized(timer) { - return nodeState; - } - } - List<ClusterState> getSystemStatesReceived() { synchronized(timer) { return clusterStateBundles.stream() diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerContextImplTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerContextImplTest.java index 96ce050f33b..b922011b4af 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerContextImplTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerContextImplTest.java @@ -29,7 +29,7 @@ public class FleetControllerContextImplTest { } private static class MockLogger extends Logger { - public List<LogRecord> records = new ArrayList<>(); + public final List<LogRecord> records = new ArrayList<>(); public MockLogger() { super(MockLogger.class.getName(), null); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java index f43c5354be5..6abd5dae23f 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java @@ -24,7 +24,6 @@ import com.yahoo.vespa.clustercontroller.core.testutils.WaitTask; import com.yahoo.vespa.clustercontroller.core.testutils.Waiter; import com.yahoo.vespa.clustercontroller.utils.util.NoMetricReporter; import org.junit.jupiter.api.AfterEach; -import java.io.IOException; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; @@ -56,7 +55,7 @@ public abstract class FleetControllerTest implements Waiter { protected Slobrok slobrok; protected FleetControllerOptions options; ZooKeeperTestServer zooKeeperServer; - protected List<FleetController> fleetControllers = new ArrayList<>(); + protected final List<FleetController> fleetControllers = new ArrayList<>(); protected List<DummyVdsNode> nodes = new ArrayList<>(); private String testName; @@ -154,7 +153,7 @@ public abstract class FleetControllerTest implements Waiter { fleetControllers.forEach(f -> { try { f.shutdown(); - } catch (InterruptedException | IOException e) { + } catch (InterruptedException e) { throw new RuntimeException(e); } }); @@ -257,7 +256,7 @@ public abstract class FleetControllerTest implements Waiter { fleetControllers.forEach(f -> { try { f.shutdown(); - } catch (InterruptedException | IOException e) { + } catch (InterruptedException e) { throw new RuntimeException(e); } }); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownTest.java index f0c8503fed9..0f9fad2c76c 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownTest.java @@ -295,7 +295,7 @@ public class GroupAutoTakedownTest { } @Test - void previously_cleared_start_timestamps_are_not_reintroduced_on_up_edge() throws Exception { + void previously_cleared_start_timestamps_are_not_reintroduced_on_up_edge() { ClusterFixture fixture = createFixtureForAllUpHierarchicCluster( DistributionBuilder.withGroups(3).eachWithNodeCount(2), 0.51); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcasterTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcasterTest.java index 6bc9f434ee6..1aa3222921d 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcasterTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcasterTest.java @@ -319,7 +319,6 @@ public class SystemStateBroadcasterTest { // Type erasure of Waiter in mocked argument capture @Test - @SuppressWarnings("unchecked") void activation_not_sent_if_deferred_activation_is_disabled_in_state_bundle() { var f = StateActivationFixture.withTwoPhaseDisabled(); var cf = f.cf; diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ZooKeeperDatabaseTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ZooKeeperDatabaseTest.java index 79c03a67698..52bfd0e2dff 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ZooKeeperDatabaseTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ZooKeeperDatabaseTest.java @@ -71,7 +71,7 @@ public class ZooKeeperDatabaseTest { } @Test - void storing_cluster_state_bundle_with_mismatching_expected_znode_version_throws_exception() throws Exception { + void storing_cluster_state_bundle_with_mismatching_expected_znode_version_throws_exception() { Throwable exception = assertThrows(CasWriteFailed.class, () -> { try (Fixture f = new Fixture()) { f.createDatabase(); @@ -82,7 +82,7 @@ public class ZooKeeperDatabaseTest { } @Test - void storing_cluster_state_version_with_mismatching_expected_znode_version_throws_exception() throws Exception { + void storing_cluster_state_version_with_mismatching_expected_znode_version_throws_exception() { Throwable exception = assertThrows(CasWriteFailed.class, () -> { try (Fixture f = new Fixture()) { f.createDatabase(); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/HasMetricContext.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/HasMetricContext.java index c00076d0faa..92123752465 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/HasMetricContext.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/HasMetricContext.java @@ -29,7 +29,7 @@ public class HasMetricContext extends BaseMatcher<MetricReporter.Context> { @Override public void describeTo(Description description) { - description.appendText(String.format("Context with dimensions %s", dimensions.toString())); + description.appendText(String.format("Context with dimensions %s", dimensions)); } @Override diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/mocks/TestEventLog.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/mocks/TestEventLog.java index b4b468ba70f..e076c4af1e7 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/mocks/TestEventLog.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/mocks/TestEventLog.java @@ -10,28 +10,23 @@ import java.util.logging.Level; public class TestEventLog implements EventLogInterface { private StringBuilder events = new StringBuilder(); - private int eventCount = 0; - public void clear() { events = new StringBuilder(); eventCount = 0; } + public void clear() { events = new StringBuilder(); } public String toString() { return events.toString(); } - public int getEventCount() { return eventCount; } @Override public void add(Event e) { events.append("add(" + e.getDescription() + ")\n"); - ++eventCount; } @Override public void add(Event e, boolean logInfo) { events.append("add(" + e + ", log ? " + logInfo + ")\n"); - ++eventCount; } @Override public void addNodeOnlyEvent(NodeEvent e, Level level) { events.append("add(" + e + ", " + level + ")\n"); - ++eventCount; } @Override diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/StateRestApiTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/StateRestApiTest.java index b09069dbad7..f7403b45707 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/StateRestApiTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/restapiv2/StateRestApiTest.java @@ -49,7 +49,7 @@ public abstract class StateRestApiTest { public String[] getUnitPath() { return path; } } - protected void setUp(boolean dontInitializeNode2) throws Exception { + protected void setUp(boolean dontInitializeNode2) { Distribution distribution = new Distribution(getSimpleGroupConfig(2, 10)); jsonWriter.setDefaultPathPrefix("/cluster/v2"); { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java index 8bb6be16e4a..685ca377e70 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/AutoscalingMetrics.java @@ -1,9 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; -import com.yahoo.metrics.ContainerMetrics; -import com.yahoo.metrics.SearchNodeMetrics; -import com.yahoo.metrics.StorageMetrics; -import com.yahoo.metrics.HostedNodeAdminMetrics; + +import ai.vespa.metrics.ContainerMetrics; +import ai.vespa.metrics.HostedNodeAdminMetrics; +import ai.vespa.metrics.SearchNodeMetrics; +import ai.vespa.metrics.StorageMetrics; + import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java index 1977a584713..8652acbd546 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultMetrics.java @@ -2,22 +2,22 @@ package com.yahoo.vespa.model.admin.monitoring; -import com.yahoo.metrics.ContainerMetrics; -import com.yahoo.metrics.SearchNodeMetrics; -import com.yahoo.metrics.Suffix; -import com.yahoo.metrics.VespaMetrics; +import ai.vespa.metrics.ContainerMetrics; +import ai.vespa.metrics.SearchNodeMetrics; +import ai.vespa.metrics.Suffix; +import ai.vespa.metrics.VespaMetrics; import java.util.Collections; import java.util.EnumSet; import java.util.LinkedHashSet; import java.util.Set; -import static com.yahoo.metrics.Suffix.average; -import static com.yahoo.metrics.Suffix.count; -import static com.yahoo.metrics.Suffix.max; -import static com.yahoo.metrics.Suffix.ninety_five_percentile; -import static com.yahoo.metrics.Suffix.ninety_nine_percentile; -import static com.yahoo.metrics.Suffix.sum; +import static ai.vespa.metrics.Suffix.average; +import static ai.vespa.metrics.Suffix.count; +import static ai.vespa.metrics.Suffix.max; +import static ai.vespa.metrics.Suffix.ninety_five_percentile; +import static ai.vespa.metrics.Suffix.ninety_nine_percentile; +import static ai.vespa.metrics.Suffix.sum; import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultVespaMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultVespaMetrics.java index d4c3f908eeb..ac03fcf56cb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultVespaMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/DefaultVespaMetrics.java @@ -1,9 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; -import com.yahoo.metrics.ContainerMetrics; -import com.yahoo.metrics.SearchNodeMetrics; - +import ai.vespa.metrics.ContainerMetrics; +import ai.vespa.metrics.SearchNodeMetrics; import com.google.common.collect.ImmutableSet; import java.util.Set; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/NetworkMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/NetworkMetrics.java index 839dcad64ee..21e7d35fc7c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/NetworkMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/NetworkMetrics.java @@ -1,8 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; -import com.yahoo.metrics.HostedNodeAdminMetrics; - +import ai.vespa.metrics.HostedNodeAdminMetrics; import com.google.common.collect.ImmutableSet; import java.util.Set; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java index eee6be9af93..64f2a72d1b4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java @@ -1,8 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; -import com.yahoo.metrics.HostedNodeAdminMetrics; - +import ai.vespa.metrics.HostedNodeAdminMetrics; import com.google.common.collect.ImmutableSet; import java.util.Set; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index f933325b8ae..362bc7b0964 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -1,34 +1,34 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; -import com.yahoo.metrics.ClusterControllerMetrics; -import com.yahoo.metrics.ConfigServerMetrics; -import com.yahoo.metrics.ContainerMetrics; -import com.yahoo.metrics.DistributorMetrics; -import com.yahoo.metrics.LogdMetrics; -import com.yahoo.metrics.NodeAdminMetrics; -import com.yahoo.metrics.RoutingLayerMetrics; -import com.yahoo.metrics.SearchNodeMetrics; -import com.yahoo.metrics.SentinelMetrics; -import com.yahoo.metrics.SlobrokMetrics; -import com.yahoo.metrics.StorageMetrics; -import com.yahoo.metrics.Suffix; -import com.yahoo.metrics.VespaMetrics; +import ai.vespa.metrics.ClusterControllerMetrics; +import ai.vespa.metrics.ConfigServerMetrics; +import ai.vespa.metrics.ContainerMetrics; +import ai.vespa.metrics.DistributorMetrics; +import ai.vespa.metrics.LogdMetrics; +import ai.vespa.metrics.NodeAdminMetrics; +import ai.vespa.metrics.RoutingLayerMetrics; +import ai.vespa.metrics.SearchNodeMetrics; +import ai.vespa.metrics.SentinelMetrics; +import ai.vespa.metrics.SlobrokMetrics; +import ai.vespa.metrics.StorageMetrics; +import ai.vespa.metrics.Suffix; +import ai.vespa.metrics.VespaMetrics; import java.util.Collections; import java.util.EnumSet; import java.util.LinkedHashSet; import java.util.Set; -import static com.yahoo.metrics.Suffix.average; -import static com.yahoo.metrics.Suffix.count; -import static com.yahoo.metrics.Suffix.last; -import static com.yahoo.metrics.Suffix.max; -import static com.yahoo.metrics.Suffix.min; -import static com.yahoo.metrics.Suffix.ninety_five_percentile; -import static com.yahoo.metrics.Suffix.ninety_nine_percentile; -import static com.yahoo.metrics.Suffix.rate; -import static com.yahoo.metrics.Suffix.sum; +import static ai.vespa.metrics.Suffix.average; +import static ai.vespa.metrics.Suffix.count; +import static ai.vespa.metrics.Suffix.last; +import static ai.vespa.metrics.Suffix.max; +import static ai.vespa.metrics.Suffix.min; +import static ai.vespa.metrics.Suffix.ninety_five_percentile; +import static ai.vespa.metrics.Suffix.ninety_nine_percentile; +import static ai.vespa.metrics.Suffix.rate; +import static ai.vespa.metrics.Suffix.sum; import static com.yahoo.vespa.model.admin.monitoring.DefaultVespaMetrics.defaultVespaMetricSet; import static java.util.Collections.singleton; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index f901bf3c826..9e21fd2d23a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -9,6 +9,7 @@ import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.search.config.QrStartConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.model.container.component.SimpleComponent; import java.time.Duration; import java.util.Optional; @@ -20,6 +21,7 @@ import java.util.Optional; */ public final class ApplicationContainer extends Container implements QrStartConfig.Producer, + OnnxModelsConfig.Producer, ZookeeperServerConfig.Producer { private final boolean isHostedVespa; @@ -42,12 +44,15 @@ public final class ApplicationContainer extends Container implements @Override public void getConfig(QrStartConfig.Builder builder) { - if (getHostResource() != null) { - NodeResources nodeResources = getHostResource().realResources(); - if ( ! nodeResources.isUnspecified()) { - builder.jvm.availableProcessors(Math.max(2, (int)Math.ceil(nodeResources.vcpu()))); - } - } + realResources().ifPresent(r -> builder.jvm.availableProcessors(Math.max(2, (int) Math.ceil(r.vcpu())))); + } + + @Override + public void getConfig(OnnxModelsConfig.Builder builder) { + realResources().ifPresent(r -> { + int count = r.gpuResources().count(); + if (count >= 0) builder.gpu.count(count); + }); } @Override @@ -84,4 +89,14 @@ public final class ApplicationContainer extends Container implements @Override public Optional<String> getPreShutdownCommand() { return Optional.of(prepareStopCommand(Duration.ofMinutes(6))); } + private Optional<NodeResources> realResources() { + if (getHostResource() != null) { + NodeResources nodeResources = getHostResource().realResources(); + if ( ! nodeResources.isUnspecified()) { + return Optional.of(nodeResources); + } + } + return Optional.empty(); + } + } diff --git a/config-model/src/test/derived/globalphase_onnx_inside/onnx-models.cfg b/config-model/src/test/derived/globalphase_onnx_inside/onnx-models.cfg index d63e85e2f19..99f65336794 100644 --- a/config-model/src/test/derived/globalphase_onnx_inside/onnx-models.cfg +++ b/config-model/src/test/derived/globalphase_onnx_inside/onnx-models.cfg @@ -1,3 +1,4 @@ +gpu.count -1 model[].name "direct" model[].fileref "files/ax_plus_b.onnx" model[].input[].name "vector_B" diff --git a/config-model/src/test/derived/globalphase_token_functions/onnx-models.cfg b/config-model/src/test/derived/globalphase_token_functions/onnx-models.cfg index 6283159c324..cea4c065014 100644 --- a/config-model/src/test/derived/globalphase_token_functions/onnx-models.cfg +++ b/config-model/src/test/derived/globalphase_token_functions/onnx-models.cfg @@ -1,3 +1,4 @@ +gpu.count -1 model[].name "my_ranking_model" model[].fileref "files/ranking_model.onnx" model[].input[].name "input_ids" diff --git a/config-model/src/test/derived/vector_constant/onnx-models.cfg b/config-model/src/test/derived/vector_constant/onnx-models.cfg index 4c52b72b519..1dcaf0e1bd6 100644 --- a/config-model/src/test/derived/vector_constant/onnx-models.cfg +++ b/config-model/src/test/derived/vector_constant/onnx-models.cfg @@ -1,3 +1,4 @@ +gpu.count -1 model[].name "inside" model[].fileref "ax_plus_b.onnx" model[].input[].name "vector_B" diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java index 7920bbed763..938f0908abc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterDeploymentMetricsRetriever.java @@ -1,10 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.metrics; +import ai.vespa.metrics.ClusterControllerMetrics; import ai.vespa.util.http.hc5.VespaHttpClientBuilder; import com.yahoo.concurrent.DaemonThreadFactory; -import com.yahoo.metrics.ClusterControllerMetrics; -import com.yahoo.metrics.ContainerMetrics; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; diff --git a/container-core/pom.xml b/container-core/pom.xml index 2b1d2253534..5fed4f02912 100644 --- a/container-core/pom.xml +++ b/container-core/pom.xml @@ -85,6 +85,11 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>metrics</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>opennlp-linguistics</artifactId> <version>${project.version}</version> <exclusions> diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java index b9a6d8d9462..f0db9c35f12 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java +++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler; +import ai.vespa.metrics.ContainerMetrics; import com.yahoo.component.annotation.Inject; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.core.VipStatusConfig; import com.yahoo.container.jdisc.state.StateMonitor; import com.yahoo.jdisc.Metric; -import com.yahoo.metrics.ContainerMetrics; import java.util.Map; import java.util.stream.Collectors; diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java index ed2d044c38c..e3b3b37414e 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java @@ -1,6 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler; +import ai.vespa.metrics.ContainerMetrics; +import com.yahoo.component.annotation.Inject; +import com.yahoo.container.core.VipStatusConfig; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.jdisc.Metric; +import com.yahoo.text.Utf8; +import com.yahoo.vespa.defaults.Defaults; + import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -10,19 +20,6 @@ import java.util.concurrent.Executor; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.logging.Logger; - -import com.yahoo.component.annotation.Inject; -import com.yahoo.container.core.VipStatusConfig; -import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; -import com.yahoo.jdisc.Metric; -import java.util.logging.Level; - -import com.yahoo.metrics.ContainerMetrics; -import com.yahoo.text.Utf8; -import com.yahoo.vespa.defaults.Defaults; /** * Transmit status to VIP from file or memory. Bind this to diff --git a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java index 15b29ad0e6a..24bc7ebc2d0 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java +++ b/container-core/src/main/java/com/yahoo/container/handler/threadpool/ThreadPoolMetric.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler.threadpool; +import ai.vespa.metrics.ContainerMetrics; import com.yahoo.jdisc.Metric; -import com.yahoo.metrics.ContainerMetrics; import java.util.Map; diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java index 2d51ffd11e6..17a2c99a760 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc; +import ai.vespa.metrics.ContainerMetrics; import com.yahoo.component.annotation.Inject; import com.yahoo.container.logging.AccessLogEntry; import com.yahoo.jdisc.Metric; @@ -11,7 +12,6 @@ import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.handler.UnsafeContentInputStream; import com.yahoo.jdisc.http.server.jetty.AccessLoggingRequestHandler; -import com.yahoo.metrics.ContainerMetrics; import com.yahoo.yolean.Exceptions; import java.io.IOException; diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java index a4eaab2e8f8..327640cb7ed 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.server.jetty; -import com.yahoo.metrics.ContainerMetrics; +import ai.vespa.metrics.ContainerMetrics; /** * Name and dimensions for jdisc/container metrics diff --git a/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolImplTest.java b/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolImplTest.java index 606f8052670..4e4456da57e 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolImplTest.java +++ b/container-core/src/test/java/com/yahoo/container/handler/threadpool/ContainerThreadPoolImplTest.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler.threadpool; +import ai.vespa.metrics.ContainerMetrics; import com.yahoo.collections.Tuple2; import com.yahoo.concurrent.Receiver; import com.yahoo.container.protect.ProcessTerminator; import com.yahoo.container.test.MetricMock; import com.yahoo.jdisc.Metric; -import com.yahoo.metrics.ContainerMetrics; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index f3f1374762d..570f868c669 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -63,11 +63,11 @@ import java.util.concurrent.Phaser; import java.util.logging.Level; import java.util.logging.Logger; +import static ai.vespa.metrics.ContainerMetrics.APPLICATION_GENERATION; +import static ai.vespa.metrics.ContainerMetrics.JDISC_APPLICATION_COMPONENT_GRAPH_CREATION_TIME_MILLIS; +import static ai.vespa.metrics.ContainerMetrics.JDISC_APPLICATION_COMPONENT_GRAPH_RECONFIGURATIONS; +import static ai.vespa.metrics.ContainerMetrics.JDISC_APPLICATION_FAILED_COMPONENT_GRAPHS; import static com.yahoo.collections.CollectionUtil.first; -import static com.yahoo.metrics.ContainerMetrics.APPLICATION_GENERATION; -import static com.yahoo.metrics.ContainerMetrics.JDISC_APPLICATION_COMPONENT_GRAPH_CREATION_TIME_MILLIS; -import static com.yahoo.metrics.ContainerMetrics.JDISC_APPLICATION_COMPONENT_GRAPH_RECONFIGURATIONS; -import static com.yahoo.metrics.ContainerMetrics.JDISC_APPLICATION_FAILED_COMPONENT_GRAPHS; /** * @author Tony Vaagenes diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/GarbageCollectionMetrics.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/GarbageCollectionMetrics.java index 06f61d18d10..43fd7960526 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/GarbageCollectionMetrics.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/GarbageCollectionMetrics.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.metric; +import ai.vespa.metrics.ContainerMetrics; import com.yahoo.jdisc.Metric; -import com.yahoo.metrics.ContainerMetrics; import java.lang.management.GarbageCollectorMXBean; import java.lang.management.ManagementFactory; diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/JrtMetrics.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/JrtMetrics.java index 24bb862cad5..c1b95cce654 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/JrtMetrics.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/JrtMetrics.java @@ -2,9 +2,10 @@ package com.yahoo.container.jdisc.metric; // import com.yahoo.jdisc.Container; + +import ai.vespa.metrics.ContainerMetrics; import com.yahoo.jdisc.Metric; import com.yahoo.jrt.TransportMetrics; -import com.yahoo.metrics.ContainerMetrics; import static com.yahoo.jrt.TransportMetrics.Snapshot; diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java index 43925367c1d..3e6ee3a35a2 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/metric/MetricUpdater.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.jdisc.metric; -import com.yahoo.component.annotation.Inject; +import ai.vespa.metrics.ContainerMetrics; import com.yahoo.component.AbstractComponent; +import com.yahoo.component.annotation.Inject; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.statistics.ContainerWatchdogMetrics; -import com.yahoo.metrics.ContainerMetrics; import com.yahoo.nativec.NativeHeap; import com.yahoo.security.tls.TlsMetrics; diff --git a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java index de24196327e..04857e982a9 100644 --- a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java @@ -1,13 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.statistics; +import ai.vespa.metrics.ContainerMetrics; import com.yahoo.component.chain.dependencies.Before; import com.yahoo.concurrent.CopyOnWriteHashMap; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.jdisc.Metric; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.metrics.simple.MetricSettings; -import com.yahoo.metrics.ContainerMetrics; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 3b4d15f788f..6a0015f9d3a 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -2,6 +2,7 @@ package com.yahoo.search.handler; import ai.vespa.cloud.ZoneInfo; +import ai.vespa.metrics.ContainerMetrics; import com.yahoo.collections.Tuple2; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.Vtag; @@ -21,7 +22,6 @@ import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Request; import com.yahoo.language.process.Embedder; import com.yahoo.language.provider.DefaultEmbedderProvider; -import com.yahoo.metrics.ContainerMetrics; import com.yahoo.net.HostName; import com.yahoo.net.UriTools; import com.yahoo.prelude.query.parser.ParseException; diff --git a/docproc/src/main/java/com/yahoo/docproc/Call.java b/docproc/src/main/java/com/yahoo/docproc/Call.java index 8fafa0fd059..3840de63e13 100644 --- a/docproc/src/main/java/com/yahoo/docproc/Call.java +++ b/docproc/src/main/java/com/yahoo/docproc/Call.java @@ -1,7 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.docproc; +import ai.vespa.metrics.ContainerMetrics; import com.yahoo.component.ComponentId; +import com.yahoo.concurrent.SystemTimer; import com.yahoo.docproc.impl.DocumentOperationWrapper; import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.docproc.proxy.ProxyDocument; @@ -11,8 +13,6 @@ import com.yahoo.document.DocumentOperation; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentUpdate; import com.yahoo.jdisc.Metric; -import com.yahoo.concurrent.SystemTimer; -import com.yahoo.metrics.ContainerMetrics; import java.util.List; import java.util.Map; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 77bda4b02e8..e2810bbede4 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -408,6 +408,12 @@ public class Flags { "Takes effect when provisioning new AWS hosts", APPLICATION_ID); + public static final UnboundBooleanFlag ENABLE_THE_ONE_THAT_SHOULD_NOT_BE_NAMED = defineFeatureFlag( + "enable-the-one-that-should-not-be-named", false, List.of("hmusum"), "2023-05-08", "2023-06-01", + "Whether to enable the one program that should not be named", + "Takes effect at next host-admin tick", + ZONE_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java index 2eee55e4102..356055f37be 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java @@ -1,13 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.filter.security.athenz; +import ai.vespa.metrics.ContainerMetrics; import com.yahoo.component.annotation.Inject; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.athenz.RequestResourceMapper.ResourceNameAndAction; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; -import com.yahoo.metrics.ContainerMetrics; import com.yahoo.vespa.athenz.api.AthenzAccessToken; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/rule/RuleBasedRequestFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/rule/RuleBasedRequestFilter.java index f1366ceea72..7ff69ad4ec6 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/rule/RuleBasedRequestFilter.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/rule/RuleBasedRequestFilter.java @@ -1,15 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.filter.security.rule; +import ai.vespa.metrics.ContainerMetrics; import com.yahoo.component.annotation.Inject; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; -import com.yahoo.metrics.ContainerMetrics; +import com.yahoo.restapi.Path; import com.yahoo.vespa.config.jdisc.http.filter.RuleBasedFilterConfig; import com.yahoo.vespa.config.jdisc.http.filter.RuleBasedFilterConfig.Rule.Action; -import com.yahoo.restapi.Path; import java.net.URI; import java.util.List; diff --git a/metrics/pom.xml b/metrics/pom.xml index f44a2569ace..e8303e5a01f 100644 --- a/metrics/pom.xml +++ b/metrics/pom.xml @@ -15,6 +15,12 @@ <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>annotations</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>config-lib</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/container-core/src/main/java/com/yahoo/metrics/ClusterControllerMetrics.java b/metrics/src/main/java/ai/vespa/metrics/ClusterControllerMetrics.java index fabfd5504f7..545de543663 100644 --- a/container-core/src/main/java/com/yahoo/metrics/ClusterControllerMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/ClusterControllerMetrics.java @@ -1,4 +1,4 @@ -package com.yahoo.metrics; +package ai.vespa.metrics; /** * @author yngve diff --git a/container-core/src/main/java/com/yahoo/metrics/ConfigServerMetrics.java b/metrics/src/main/java/ai/vespa/metrics/ConfigServerMetrics.java index e501fa5e832..dba75c08aae 100644 --- a/container-core/src/main/java/com/yahoo/metrics/ConfigServerMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/ConfigServerMetrics.java @@ -1,4 +1,4 @@ -package com.yahoo.metrics; +package ai.vespa.metrics; /** * @author yngveaasheim diff --git a/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java b/metrics/src/main/java/ai/vespa/metrics/ContainerMetrics.java index 11241e1558d..84de1878b3f 100644 --- a/container-core/src/main/java/com/yahoo/metrics/ContainerMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/ContainerMetrics.java @@ -1,4 +1,4 @@ -package com.yahoo.metrics; +package ai.vespa.metrics; /** * @author gjoranv diff --git a/container-core/src/main/java/com/yahoo/metrics/DistributorMetrics.java b/metrics/src/main/java/ai/vespa/metrics/DistributorMetrics.java index 82e9aff74a8..b2ebbc4d6aa 100644 --- a/container-core/src/main/java/com/yahoo/metrics/DistributorMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/DistributorMetrics.java @@ -1,6 +1,4 @@ -package com.yahoo.metrics; - -import java.util.List; +package ai.vespa.metrics; /** * @author yngveaasheim diff --git a/container-core/src/main/java/com/yahoo/metrics/HostedNodeAdminMetrics.java b/metrics/src/main/java/ai/vespa/metrics/HostedNodeAdminMetrics.java index 1dc58090c69..927672a43f7 100644 --- a/container-core/src/main/java/com/yahoo/metrics/HostedNodeAdminMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/HostedNodeAdminMetrics.java @@ -1,4 +1,4 @@ -package com.yahoo.metrics; +package ai.vespa.metrics; /** * @author yngveaasheim diff --git a/container-core/src/main/java/com/yahoo/metrics/LogdMetrics.java b/metrics/src/main/java/ai/vespa/metrics/LogdMetrics.java index 3dae4283b9f..79122e3b922 100644 --- a/container-core/src/main/java/com/yahoo/metrics/LogdMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/LogdMetrics.java @@ -1,4 +1,4 @@ -package com.yahoo.metrics; +package ai.vespa.metrics; /** * @author yngveaasheim diff --git a/container-core/src/main/java/com/yahoo/metrics/NodeAdminMetrics.java b/metrics/src/main/java/ai/vespa/metrics/NodeAdminMetrics.java index 004a226f825..74da68dbcb7 100644 --- a/container-core/src/main/java/com/yahoo/metrics/NodeAdminMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/NodeAdminMetrics.java @@ -1,4 +1,4 @@ -package com.yahoo.metrics; +package ai.vespa.metrics; /** * @author yngveaasheim diff --git a/container-core/src/main/java/com/yahoo/metrics/RoutingLayerMetrics.java b/metrics/src/main/java/ai/vespa/metrics/RoutingLayerMetrics.java index 773afae00ba..cf35cdae90e 100644 --- a/container-core/src/main/java/com/yahoo/metrics/RoutingLayerMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/RoutingLayerMetrics.java @@ -1,4 +1,4 @@ -package com.yahoo.metrics; +package ai.vespa.metrics; /** * @author yngveaasheim diff --git a/container-core/src/main/java/com/yahoo/metrics/SearchNodeMetrics.java b/metrics/src/main/java/ai/vespa/metrics/SearchNodeMetrics.java index b36abbdf211..d6018dc0633 100644 --- a/container-core/src/main/java/com/yahoo/metrics/SearchNodeMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/SearchNodeMetrics.java @@ -1,6 +1,4 @@ -package com.yahoo.metrics; - -import java.util.List; +package ai.vespa.metrics; /** * @author gjoranv diff --git a/container-core/src/main/java/com/yahoo/metrics/SentinelMetrics.java b/metrics/src/main/java/ai/vespa/metrics/SentinelMetrics.java index 7711b7e75f4..35ecbae85d8 100644 --- a/container-core/src/main/java/com/yahoo/metrics/SentinelMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/SentinelMetrics.java @@ -1,4 +1,4 @@ -package com.yahoo.metrics; +package ai.vespa.metrics; /** * @author yngve diff --git a/container-core/src/main/java/com/yahoo/metrics/SlobrokMetrics.java b/metrics/src/main/java/ai/vespa/metrics/SlobrokMetrics.java index 8c30bf8e414..1a6735af860 100644 --- a/container-core/src/main/java/com/yahoo/metrics/SlobrokMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/SlobrokMetrics.java @@ -1,4 +1,4 @@ -package com.yahoo.metrics; +package ai.vespa.metrics; /** * @author yngve diff --git a/container-core/src/main/java/com/yahoo/metrics/StorageMetrics.java b/metrics/src/main/java/ai/vespa/metrics/StorageMetrics.java index 05ae5180d3b..7071fe0ae77 100644 --- a/container-core/src/main/java/com/yahoo/metrics/StorageMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/StorageMetrics.java @@ -1,7 +1,4 @@ -package com.yahoo.metrics; - -import java.util.EnumSet; -import java.util.List; +package ai.vespa.metrics; /** * @author yngveaasheim diff --git a/container-core/src/main/java/com/yahoo/metrics/Suffix.java b/metrics/src/main/java/ai/vespa/metrics/Suffix.java index b9ef3a5ce11..ce5e0aaa602 100644 --- a/container-core/src/main/java/com/yahoo/metrics/Suffix.java +++ b/metrics/src/main/java/ai/vespa/metrics/Suffix.java @@ -1,6 +1,4 @@ -package com.yahoo.metrics; - -import java.util.EnumSet; +package ai.vespa.metrics; public enum Suffix { ninety_five_percentile("95percentile"), diff --git a/container-core/src/main/java/com/yahoo/metrics/Unit.java b/metrics/src/main/java/ai/vespa/metrics/Unit.java index 53c231a2e5e..a2123d72246 100644 --- a/container-core/src/main/java/com/yahoo/metrics/Unit.java +++ b/metrics/src/main/java/ai/vespa/metrics/Unit.java @@ -1,4 +1,4 @@ -package com.yahoo.metrics; +package ai.vespa.metrics; /** * @author gjoranv diff --git a/container-core/src/main/java/com/yahoo/metrics/VespaMetrics.java b/metrics/src/main/java/ai/vespa/metrics/VespaMetrics.java index c9d1f058619..3a17d8a3155 100644 --- a/container-core/src/main/java/com/yahoo/metrics/VespaMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/VespaMetrics.java @@ -1,4 +1,4 @@ -package com.yahoo.metrics; +package ai.vespa.metrics; /** * @author gjoranv diff --git a/metrics/src/main/java/ai/vespa/metrics/package-info.java b/metrics/src/main/java/ai/vespa/metrics/package-info.java new file mode 100644 index 00000000000..98986f61dc5 --- /dev/null +++ b/metrics/src/main/java/ai/vespa/metrics/package-info.java @@ -0,0 +1,5 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package ai.vespa.metrics; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java index fd5306f9add..303d2acd79e 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java @@ -41,7 +41,7 @@ public class ModelsEvaluator extends AbstractComponent { RankingExpressionsConfig expressionsConfig, OnnxModelsConfig onnxModelsConfig, FileAcquirer fileAcquirer) { - this(config, constantsConfig, expressionsConfig, onnxModelsConfig, fileAcquirer, new OnnxRuntime()); + this(config, constantsConfig, expressionsConfig, onnxModelsConfig, fileAcquirer, new OnnxRuntime(onnxModelsConfig)); } public ModelsEvaluator(RankProfilesConfigImporter importer, diff --git a/model-integration/pom.xml b/model-integration/pom.xml index c27ed9d2c31..d5d7ae534a4 100644 --- a/model-integration/pom.xml +++ b/model-integration/pom.xml @@ -40,6 +40,12 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> + <artifactId>searchcore</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> <artifactId>searchlib</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java b/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java index 8e5211ccff1..b172ef7beee 100644 --- a/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java +++ b/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java @@ -58,6 +58,7 @@ public class BertBaseEmbedder extends AbstractComponent implements Embedder { OnnxEvaluatorOptions options = new OnnxEvaluatorOptions(); options.setExecutionMode(config.onnxExecutionMode().toString()); options.setThreads(config.onnxInterOpThreads(), config.onnxIntraOpThreads()); + if (config.onnxGpuDevice() >= 0) options.setGpuDevice(config.onnxGpuDevice()); tokenizer = new WordPieceEmbedder.Builder(config.tokenizerVocab().toString()).build(); this.evaluator = onnx.evaluatorOf(config.transformerModel().toString(), options); diff --git a/model-integration/src/main/java/ai/vespa/embedding/huggingface/Encoding.java b/model-integration/src/main/java/ai/vespa/embedding/huggingface/Encoding.java index f1c0244bfb3..274c29a57b2 100644 --- a/model-integration/src/main/java/ai/vespa/embedding/huggingface/Encoding.java +++ b/model-integration/src/main/java/ai/vespa/embedding/huggingface/Encoding.java @@ -14,9 +14,12 @@ public record Encoding( List<Long> specialTokenMask, List<CharSpan> charTokenSpans, List<Encoding> overflowing) { public record CharSpan(int start, int end) { + public static final CharSpan NONE = new CharSpan(-1, -1); static CharSpan from(ai.djl.huggingface.tokenizers.jni.CharSpan s) { + if (s == null) return NONE; return new CharSpan(s.getStart(), s.getEnd()); } + public boolean isNone() { return this.equals(NONE); } } public Encoding { @@ -43,6 +46,7 @@ public record Encoding( } private static List<Long> toList(long[] array) { + if (array == null) return List.of(); var list = new ArrayList<Long>(array.length); for (long e : array) list.add(e); return list; diff --git a/model-integration/src/main/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedder.java b/model-integration/src/main/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedder.java index 21dd326689c..cc13254385b 100644 --- a/model-integration/src/main/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedder.java +++ b/model-integration/src/main/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedder.java @@ -40,7 +40,7 @@ public class HuggingFaceEmbedder extends AbstractComponent implements Embedder { tokenizer = new HuggingFaceTokenizer(Paths.get(config.tokenizerPath().toString())); var onnxOpts = new OnnxEvaluatorOptions(); if (config.transformerGpuDevice() >= 0) - onnxOpts.setGpuDevice(config.transformerGpuDevice(), config.transformerGpuRequired()); + onnxOpts.setGpuDevice(config.transformerGpuDevice()); onnxOpts.setExecutionMode(config.transformerExecutionMode().toString()); onnxOpts.setThreads(config.transformerInterOpThreads(), config.transformerIntraOpThreads()); evaluator = onnx.evaluatorOf(config.transformerModel().toString(), onnxOpts); diff --git a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorOptions.java b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorOptions.java index 4a35f4275fa..6048be8aca9 100644 --- a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorOptions.java +++ b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorOptions.java @@ -17,7 +17,7 @@ import static ai.onnxruntime.OrtSession.SessionOptions.ExecutionMode.SEQUENTIAL; */ public class OnnxEvaluatorOptions { - private final OrtSession.SessionOptions.OptLevel optimizationLevel; + private OrtSession.SessionOptions.OptLevel optimizationLevel; private OrtSession.SessionOptions.ExecutionMode executionMode; private int interOpThreads; private int intraOpThreads; @@ -86,6 +86,8 @@ public class OnnxEvaluatorOptions { this.gpuDeviceRequired = required; } + public void setGpuDevice(int deviceNumber) { gpuDeviceNumber = deviceNumber; } + public boolean requestingGpu() { return gpuDeviceNumber > -1; } @@ -94,6 +96,19 @@ public class OnnxEvaluatorOptions { return gpuDeviceRequired; } + public int gpuDeviceNumber() { return gpuDeviceNumber; } + + public OnnxEvaluatorOptions copy() { + var copy = new OnnxEvaluatorOptions(); + copy.gpuDeviceNumber = gpuDeviceNumber; + copy.gpuDeviceRequired = gpuDeviceRequired; + copy.executionMode = executionMode; + copy.interOpThreads = interOpThreads; + copy.intraOpThreads = intraOpThreads; + copy.optimizationLevel = optimizationLevel; + return copy; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxRuntime.java b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxRuntime.java index ece1db55c1e..ab44a2ae33f 100644 --- a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxRuntime.java +++ b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxRuntime.java @@ -10,6 +10,7 @@ import com.yahoo.component.annotation.Inject; import com.yahoo.jdisc.ResourceReference; import com.yahoo.jdisc.refcount.DebugReferencesWithStack; import com.yahoo.jdisc.refcount.References; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import net.jpountz.xxhash.XXHashFactory; import java.io.IOException; @@ -52,17 +53,24 @@ public class OnnxRuntime extends AbstractComponent { private final Object monitor = new Object(); private final Map<OrtSessionId, SharedOrtSession> sessions = new HashMap<>(); private final OrtSessionFactory factory; + private final int gpusAvailable; - @Inject public OnnxRuntime() { this(defaultFactory); } + // For test use only + public OnnxRuntime() { this(defaultFactory, new OnnxModelsConfig.Builder().build()); } - OnnxRuntime(OrtSessionFactory factory) { this.factory = factory; } + @Inject public OnnxRuntime(OnnxModelsConfig cfg) { this(defaultFactory, cfg); } + + OnnxRuntime(OrtSessionFactory factory, OnnxModelsConfig cfg) { + this.factory = factory; + this.gpusAvailable = cfg.gpu().count(); + } public OnnxEvaluator evaluatorOf(byte[] model) { return new OnnxEvaluator(model, null, this); } public OnnxEvaluator evaluatorOf(byte[] model, OnnxEvaluatorOptions options) { - return new OnnxEvaluator(model, options, this); + return new OnnxEvaluator(model, overrideOptions(options), this); } public OnnxEvaluator evaluatorOf(String modelPath) { @@ -70,7 +78,7 @@ public class OnnxRuntime extends AbstractComponent { } public OnnxEvaluator evaluatorOf(String modelPath, OnnxEvaluatorOptions options) { - return new OnnxEvaluator(modelPath, options, this); + return new OnnxEvaluator(modelPath, overrideOptions(options), this); } public static OrtEnvironment ortEnvironment() { @@ -167,6 +175,16 @@ public class OnnxRuntime extends AbstractComponent { } } + private OnnxEvaluatorOptions overrideOptions(OnnxEvaluatorOptions opts) { + // Set GPU device required if GPU requested and GPUs are available on system + if (gpusAvailable > 0 && opts.requestingGpu() && !opts.gpuDeviceRequired()) { + var copy = opts.copy(); + copy.setGpuDevice(opts.gpuDeviceNumber(), true); + return copy; + } + return opts; + } + int sessionsCached() { synchronized(monitor) { return sessions.size(); } } static class ReferencedOrtSession implements AutoCloseable { diff --git a/model-integration/src/main/resources/configdefinitions/embedding.bert-base-embedder.def b/model-integration/src/main/resources/configdefinitions/embedding.bert-base-embedder.def index ef42d81e1fe..e37a33d3b81 100644 --- a/model-integration/src/main/resources/configdefinitions/embedding.bert-base-embedder.def +++ b/model-integration/src/main/resources/configdefinitions/embedding.bert-base-embedder.def @@ -28,4 +28,4 @@ transformerOutput string default=output_0 onnxExecutionMode enum { parallel, sequential } default=sequential onnxInterOpThreads int default=1 onnxIntraOpThreads int default=-4 # n=number of threads -> n<0: CPUs/(-n), n==0: CPUs, n>0: n - +onnxGpuDevice int default=-1 diff --git a/model-integration/src/main/resources/configdefinitions/embedding.huggingface.hugging-face-embedder.def b/model-integration/src/main/resources/configdefinitions/embedding.huggingface.hugging-face-embedder.def index adc8f653168..584f23046ba 100644 --- a/model-integration/src/main/resources/configdefinitions/embedding.huggingface.hugging-face-embedder.def +++ b/model-integration/src/main/resources/configdefinitions/embedding.huggingface.hugging-face-embedder.def @@ -17,9 +17,6 @@ transformerAttentionMask string default=attention_mask # Output name transformerOutput string default=last_hidden_state -# GPU configuration -transformerGpuDevice int default=-1 -transformerGpuRequired bool default=false # Normalize tensors from tokenizer normalize bool default=false @@ -28,3 +25,4 @@ normalize bool default=false transformerExecutionMode enum { parallel, sequential } default=sequential transformerInterOpThreads int default=1 transformerIntraOpThreads int default=-4 +transformerGpuDevice int default=-1 diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java index fa9cfff0d68..92ec88ed9f4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java @@ -1,13 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; +import ai.vespa.metrics.ContainerMetrics; +import ai.vespa.metrics.HostedNodeAdminMetrics; +import ai.vespa.metrics.SearchNodeMetrics; +import ai.vespa.metrics.StorageMetrics; import com.yahoo.collections.ListMap; import com.yahoo.collections.Pair; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.metrics.ContainerMetrics; -import com.yahoo.metrics.HostedNodeAdminMetrics; -import com.yahoo.metrics.SearchNodeMetrics; -import com.yahoo.metrics.StorageMetrics; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; @@ -24,8 +24,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import static com.yahoo.metrics.ContainerMetrics.APPLICATION_GENERATION; -import static com.yahoo.metrics.ContainerMetrics.IN_SERVICE; +import static ai.vespa.metrics.ContainerMetrics.APPLICATION_GENERATION; +import static ai.vespa.metrics.ContainerMetrics.IN_SERVICE; /** * A response containing metrics for a collection of nodes. diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java index e228d31384c..5284511af47 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java @@ -119,6 +119,10 @@ public class LoadBalancerInstance { cloudAccount); } + public LoadBalancerInstance with(CloudAccount cloudAccount) { + return new LoadBalancerInstance(hostname, ipAddress, dnsZone, ports, networks, reals, settings, serviceIds, cloudAccount); + } + /** Prepends the given service IDs, possibly changing the order of those we have in this. */ public LoadBalancerInstance withServiceIds(List<PrivateServiceId> serviceIds) { List<PrivateServiceId> ids = new ArrayList<>(serviceIds); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index 478d49d67d4..01fc103423c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -5,6 +5,8 @@ import com.yahoo.collections.ListMap; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationTransaction; +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeType; @@ -81,8 +83,10 @@ public class Nodes { Instant start = clock.instant(); int nodesWritten = performOn(list(), (node, mutex) -> { // TODO (valerijf): Remove after 8.162 - if (node.cloudAccount().isUnspecified() && !zone.getCloud().account().isUnspecified()) - node = node.with(zone.getCloud().account()); +// if (node.cloudAccount().isUnspecified() && !zone.getCloud().account().isUnspecified()) +// node = node.with(zone.getCloud().account()); + if (zone.getCloud().name().equals(CloudName.GCP)) + node = node.with(CloudAccount.empty); return write(node, mutex); }).size(); Instant end = clock.instant(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index b3198a72d1b..21d7f1908ab 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; @@ -69,10 +70,18 @@ public class LoadBalancerProvisioner { this.service = service; this.deactivateRouting = PermanentFlags.DEACTIVATE_ROUTING.bindTo(nodeRepository.flagSource()); // Read and write all load balancers to make sure they are stored in the latest version of the serialization format + + CloudAccount zoneAccount = nodeRepository.zone().cloud().account(); for (var id : db.readLoadBalancerIds()) { try (var lock = db.lock(id.application())) { var loadBalancer = db.readLoadBalancer(id); - loadBalancer.ifPresent(lb -> db.writeLoadBalancer(lb, lb.state())); + loadBalancer.ifPresent(lb -> { +// if (!zoneAccount.isUnspecified() && lb.instance().isPresent() && lb.instance().get().cloudAccount().isUnspecified()) +// lb = lb.with(Optional.of(lb.instance().get().with(zoneAccount))); + if (nodeRepository.zone().getCloud().name().equals(CloudName.GCP)) + lb = lb.with(lb.instance().map(i -> i.with(CloudAccount.empty))); + db.writeLoadBalancer(lb, lb.state()); + }); } } } @@ -315,8 +324,8 @@ public class LoadBalancerProvisioner { } /** Returns whether load balancer is provisioned in given account */ - private static boolean inAccount(CloudAccount cloudAccount, LoadBalancer loadBalancer) { - return loadBalancer.instance().isEmpty() || loadBalancer.instance().get().cloudAccount().equals(cloudAccount); + private boolean inAccount(CloudAccount cloudAccount, LoadBalancer loadBalancer) { + return !nodeRepository.zone().cloud().name().equals(CloudName.AWS) || loadBalancer.instance().isEmpty() || loadBalancer.instance().get().cloudAccount().equals(cloudAccount); } /** Find IP addresses reachable by the load balancer service */ @@ -330,7 +339,7 @@ public class LoadBalancerProvisioner { return reachable; } - private static void requireInstance(LoadBalancerId id, LoadBalancer loadBalancer, CloudAccount cloudAccount, ZoneEndpoint zoneEndpoint) { + private void requireInstance(LoadBalancerId id, LoadBalancer loadBalancer, CloudAccount cloudAccount, ZoneEndpoint zoneEndpoint) { if (loadBalancer.instance().isEmpty()) { // Signal that load balancer is not ready yet throw new LoadBalancerServiceException("Could not provision " + id + ". The operation will be retried on next deployment"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index d7b5f30a9bc..36fc2eb25a4 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -5,14 +5,20 @@ import ai.vespa.http.DomainName; import com.google.common.collect.Iterators; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.Cloud; import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; import com.yahoo.config.provision.ZoneEndpoint; import com.yahoo.config.provision.ZoneEndpoint.AccessType; import com.yahoo.config.provision.ZoneEndpoint.AllowedUrn; @@ -60,7 +66,8 @@ public class LoadBalancerProvisionerTest { private final NodeResources nodeResources = new NodeResources(2, 4, 10, 0.3); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); - private final ProvisioningTester tester = new ProvisioningTester.Builder().flagSource(flagSource).build(); + private final ProvisioningTester tester = new ProvisioningTester.Builder().flagSource(flagSource) + .zone(new Zone(Cloud.builder().name(CloudName.AWS).account(CloudAccount.from("001122334455")).build(), SystemName.main, Environment.prod, RegionName.defaultName())).build(); @Test public void provision_load_balancer() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index d0ff11fde0c..991c6c69c35 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -447,7 +447,8 @@ public class ProvisioningTester { nameResolver.addRecord(nodeHostname, ipv4Addr); } } - Node.Builder builder = Node.create(hostname, IP.Config.of(hostIps, ipAddressPool), hostname, flavor, type); + Node.Builder builder = Node.create(hostname, IP.Config.of(hostIps, ipAddressPool), hostname, flavor, type) + .cloudAccount(nodeRepository.zone().cloud().account()); reservedTo.ifPresent(builder::reservedTo); nodes.add(builder.build()); } diff --git a/searchcore/src/tests/proton/flushengine/flushengine_test.cpp b/searchcore/src/tests/proton/flushengine/flushengine_test.cpp index a7efe2952eb..3fdc5a8ce9f 100644 --- a/searchcore/src/tests/proton/flushengine/flushengine_test.cpp +++ b/searchcore/src/tests/proton/flushengine/flushengine_test.cpp @@ -450,6 +450,17 @@ struct Fixture } }; +TEST("require that leaf defaults are sane") { + test::DummyFlushTarget leaf("dummy"); + EXPECT_FALSE(leaf.needUrgentFlush()); + EXPECT_EQUAL(0.0, leaf.get_replay_operation_cost()); + EXPECT_TRUE(IFlushTarget::Priority::NORMAL == leaf.getPriority()); + EXPECT_TRUE(50 == static_cast<int>(IFlushTarget::Priority::NORMAL)); + EXPECT_TRUE(100 == static_cast<int>(IFlushTarget::Priority::HIGH)); + EXPECT_TRUE(IFlushTarget::Priority::NORMAL < IFlushTarget::Priority::HIGH); + EXPECT_TRUE(IFlushTarget::Priority::HIGH > IFlushTarget::Priority::NORMAL); +} + TEST_F("require that strategy controls flush target", Fixture(1, IINTERVAL)) { vespalib::Gate fooG, barG; diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index 880bf8aa3e0..2f6ebcd967f 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -344,6 +344,10 @@ TEST_F(IndexManagerTest, require_that_large_memory_footprint_triggers_urgent_flu EXPECT_TRUE(IndexFlushTarget(_index_manager->getMaintainer(), FlushStats(17_Gi)).needUrgentFlush()); } +TEST_F(IndexManagerTest, require_that_flush_priority_is_high) { + EXPECT_EQ(IFlushTarget::Priority::HIGH, IndexFlushTarget(_index_manager->getMaintainer()).getPriority()); +} + TEST_F(IndexManagerTest, require_that_multiple_flushes_gives_multiple_indexes) { size_t flush_count = 10; diff --git a/searchcore/src/vespa/searchcore/config/onnx-models.def b/searchcore/src/vespa/searchcore/config/onnx-models.def index b8f5d319075..67a83e2afb7 100644 --- a/searchcore/src/vespa/searchcore/config/onnx-models.def +++ b/searchcore/src/vespa/searchcore/config/onnx-models.def @@ -1,6 +1,9 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. namespace=vespa.config.search.core +# Number of GPUs available for ONNX evaluation, or -1 if unknown. +gpu.count int default=-1 + model[].name string model[].fileref file model[].input[].name string diff --git a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp index 8101b29d98c..401cc42de52 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp @@ -155,7 +155,7 @@ FlushableAttribute::FlushableAttribute(AttributeVectorSP attr, vespalib::ISequencedTaskExecutor & attributeFieldWriter, const HwInfo &hwInfo) - : IFlushTarget(make_string("attribute.flush.%s", attr->getName().c_str()), Type::SYNC, Component::ATTRIBUTE), + : LeafFlushTarget(make_string("attribute.flush.%s", attr->getName().c_str()), Type::SYNC, Component::ATTRIBUTE), _attr(attr), _cleanUpAfterFlush(true), _lastStats(), diff --git a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h index 2ae63bef4db..56dd0e0dfec 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.h @@ -19,7 +19,7 @@ class TransientResourceUsage; /** * Implementation of IFlushTarget interface for attribute vectors. */ -class FlushableAttribute : public searchcorespi::IFlushTarget +class FlushableAttribute : public searchcorespi::LeafFlushTarget { private: /** @@ -29,15 +29,15 @@ private: using AttributeVectorSP = std::shared_ptr<search::AttributeVector>; using FlushStats = searchcorespi::FlushStats; - AttributeVectorSP _attr; - bool _cleanUpAfterFlush; - FlushStats _lastStats; - const search::TuneFileAttributes _tuneFileAttributes; + AttributeVectorSP _attr; + bool _cleanUpAfterFlush; + FlushStats _lastStats; + const search::TuneFileAttributes _tuneFileAttributes; const search::common::FileHeaderContext &_fileHeaderContext; - vespalib::ISequencedTaskExecutor &_attributeFieldWriter; - HwInfo _hwInfo; - std::shared_ptr<AttributeDirectory> _attrDir; - double _replay_operation_cost; + vespalib::ISequencedTaskExecutor &_attributeFieldWriter; + HwInfo _hwInfo; + std::shared_ptr<AttributeDirectory> _attrDir; + double _replay_operation_cost; Task::UP internalInitFlush(SerialNum currentSerial); diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.cpp index 06bf8d0a8a6..57435c91b5f 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.cpp @@ -66,7 +66,7 @@ private: } SummaryGCTarget::SummaryGCTarget(const vespalib::string & name, vespalib::Executor & summaryService, IDocumentStore & docStore) - : IFlushTarget(name, Type::GC, Component::DOCUMENT_STORE), + : LeafFlushTarget(name, Type::GC, Component::DOCUMENT_STORE), _summaryService(summaryService), _docStore(docStore), _lastStats() diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.h b/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.h index 083f763d8e6..56e17d76210 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.h @@ -12,7 +12,7 @@ namespace proton { /** * This class implements the IFlushTarget interface to proxy a summary manager. */ -class SummaryGCTarget : public searchcorespi::IFlushTarget { +class SummaryGCTarget : public searchcorespi::LeafFlushTarget { public: using FlushStats = searchcorespi::FlushStats; using IDocumentStore = search::IDocumentStore; diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.cpp index 45fc23175bf..5cb60c907f1 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.cpp @@ -43,7 +43,7 @@ public: SummaryFlushTarget::SummaryFlushTarget(IDocumentStore & docStore, vespalib::Executor & summaryService) - : IFlushTarget("summary.flush", Type::SYNC, Component::DOCUMENT_STORE), + : LeafFlushTarget("summary.flush", Type::SYNC, Component::DOCUMENT_STORE), _docStore(docStore), _summaryService(summaryService), _lastStats() diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.h b/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.h index f864b922af8..1ac7b6c9c0e 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.h @@ -9,7 +9,7 @@ namespace proton { /** * This class implements the IFlushTarget interface to proxy a summary manager. */ -class SummaryFlushTarget : public searchcorespi::IFlushTarget { +class SummaryFlushTarget : public searchcorespi::LeafFlushTarget { private: using FlushStats = searchcorespi::FlushStats; search::IDocumentStore & _docStore; diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp index 543af05db45..d1e39628c83 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp @@ -42,7 +42,7 @@ namespace proton { namespace { -class ShrinkSummaryLidSpaceFlushTarget : public ShrinkLidSpaceFlushTarget +class ShrinkSummaryLidSpaceFlushTarget : public ShrinkLidSpaceFlushTarget { using ICompactableLidSpace = search::common::ICompactableLidSpace; vespalib::Executor & _summaryService; diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp index a712035e9af..db3d8adfe26 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp @@ -150,7 +150,7 @@ DocumentMetaStoreFlushTarget:: DocumentMetaStoreFlushTarget(const DocumentMetaStore::SP dms, ITlsSyncer &tlsSyncer, const vespalib::string & baseDir, const TuneFileAttributes &tuneFileAttributes, const FileHeaderContext &fileHeaderContext, const HwInfo &hwInfo) - : IFlushTarget("documentmetastore.flush", Type::SYNC, Component::ATTRIBUTE), + : LeafFlushTarget("documentmetastore.flush", Type::SYNC, Component::ATTRIBUTE), _dms(dms), _tlsSyncer(tlsSyncer), _baseDir(baseDir), diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h index 777d7de1e17..5a3a671553c 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.h @@ -19,7 +19,7 @@ class TransientResourceUsage; /** * Implementation of IFlushTarget interface for document meta store. **/ -class DocumentMetaStoreFlushTarget : public searchcorespi::IFlushTarget +class DocumentMetaStoreFlushTarget : public searchcorespi::LeafFlushTarget { private: /** @@ -29,16 +29,16 @@ private: using DocumentMetaStoreSP = std::shared_ptr<DocumentMetaStore>; using FlushStats = searchcorespi::FlushStats; - DocumentMetaStoreSP _dms; - ITlsSyncer &_tlsSyncer; - vespalib::string _baseDir; - bool _cleanUpAfterFlush; - FlushStats _lastStats; - const search::TuneFileAttributes _tuneFileAttributes; + DocumentMetaStoreSP _dms; + ITlsSyncer &_tlsSyncer; + vespalib::string _baseDir; + bool _cleanUpAfterFlush; + FlushStats _lastStats; + const search::TuneFileAttributes _tuneFileAttributes; const search::common::FileHeaderContext &_fileHeaderContext; - HwInfo _hwInfo; - std::shared_ptr<AttributeDiskLayout> _diskLayout; - std::shared_ptr<AttributeDirectory> _dmsDir; + HwInfo _hwInfo; + std::shared_ptr<AttributeDiskLayout> _diskLayout; + std::shared_ptr<AttributeDirectory> _dmsDir; public: using SP = std::shared_ptr<DocumentMetaStoreFlushTarget>; diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/cachedflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/cachedflushtarget.cpp index 3947ac1d07d..9f212006234 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/cachedflushtarget.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/cachedflushtarget.cpp @@ -11,18 +11,10 @@ CachedFlushTarget::CachedFlushTarget(const IFlushTarget::SP &target) _lastFlushTime(target->getLastFlushTime()), _memoryGain(target->getApproxMemoryGain()), _diskGain(target->getApproxDiskGain()), + _approxBytesToWriteToDisk(target->getApproxBytesToWriteToDisk()), + _replay_operation_cost(target->get_replay_operation_cost()), _needUrgentFlush(target->needUrgentFlush()), - _approxBytesToWriteToDisk(target->getApproxBytesToWriteToDisk()) -{ - // empty -} - - -uint64_t -CachedFlushTarget::getApproxBytesToWriteToDisk() const -{ - return _approxBytesToWriteToDisk; -} - + _priority(target->getPriority()) +{ } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/cachedflushtarget.h b/searchcore/src/vespa/searchcore/proton/flushengine/cachedflushtarget.h index 1bd51e9ab11..a43c7c16217 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/cachedflushtarget.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/cachedflushtarget.h @@ -19,8 +19,10 @@ private: Time _lastFlushTime; MemoryGain _memoryGain; DiskGain _diskGain; - bool _needUrgentFlush; uint64_t _approxBytesToWriteToDisk; + double _replay_operation_cost; + bool _needUrgentFlush; + Priority _priority; public: /** @@ -47,13 +49,15 @@ public: SerialNum getFlushedSerialNum() const override { return _flushedSerialNum; } Time getLastFlushTime() const override { return _lastFlushTime; } bool needUrgentFlush() const override { return _needUrgentFlush; } + Priority getPriority() const override { return _priority; } + double get_replay_operation_cost() const override { return _replay_operation_cost; } Task::UP initFlush(SerialNum currentSerial, std::shared_ptr<search::IFlushToken> flush_token) override { return _target->initFlush(currentSerial, std::move(flush_token)); } FlushStats getLastFlushStats() const override { return _target->getLastFlushStats(); } - uint64_t getApproxBytesToWriteToDisk() const override; + uint64_t getApproxBytesToWriteToDisk() const override { return _approxBytesToWriteToDisk; } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp index 3e577bf6cbe..51713f26307 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp @@ -128,8 +128,7 @@ FlushEngine::close() if (_thread.joinable()) { _thread.join(); } - _executor.shutdown(); - _executor.sync(); + _executor.shutdown().sync(); return *this; } @@ -158,36 +157,60 @@ bool FlushEngine::wait(vespalib::duration minimumWaitTimeIfReady, bool ignorePendingPrune) { std::unique_lock<std::mutex> guard(_lock); - if ( (minimumWaitTimeIfReady != vespalib::duration::zero()) && canFlushMore(guard) && _pendingPrune.empty()) { + if (canFlushMore(guard) && _pendingPrune.empty()) { _cond.wait_for(guard, minimumWaitTimeIfReady); } while ( ! canFlushMore(guard) && ( ignorePendingPrune || _pendingPrune.empty())) { _cond.wait_for(guard, 1s); // broadcast when flush done } - return !_closed; + return !_closed.load(std::memory_order_relaxed); +} + +void +FlushEngine::wait_for_slot_or_pending_prune(IFlushTarget::Priority priority) +{ + (void) priority; + std::unique_lock<std::mutex> guard(_lock); + while ( ! canFlushMore(guard) && _pendingPrune.empty()) { + _cond.wait_for(guard, 1s); // broadcast when flush done + } +} + +vespalib::string +FlushEngine::checkAndFlush(vespalib::string prev) { + std::pair<FlushContext::List, bool> lst = getSortedTargetList(); + if (lst.second) { + // Everything returned from a priority strategy should be flushed + flushAll(lst.first); + } else if ( ! lst.first.empty()) { + wait_for_slot_or_pending_prune(lst.first[0]->getTarget()->getPriority()); + prev = flushNextTarget(prev, lst.first); + if (!prev.empty()) { + // Sleep 1 ms after a successful flush in order to avoid busy loop in case + // of strategy or target error. + std::this_thread::sleep_for(1ms); + return prev; + } + } + return ""; } void FlushEngine::run() { _has_thread = true; - bool shouldIdle = false; vespalib::string prevFlushName; - while (wait(shouldIdle ? _idleInterval : vespalib::duration::zero(), false)) { - shouldIdle = false; + for (vespalib::duration idleInterval=vespalib::duration::zero(); !_closed.load(std::memory_order_relaxed); idleInterval = _idleInterval) { + LOG(debug, "Making another check for something to flush, last was '%s'", prevFlushName.c_str()); + wait_for_slot_or_pending_prune(IFlushTarget::Priority::HIGH); if (prune()) { - continue; // Prune attempted on one or more handlers - } - prevFlushName = flushNextTarget(prevFlushName); - if ( ! prevFlushName.empty()) { - // Sleep 1 ms after a successful flush in order to avoid busy loop in case - // of strategy or target error. - std::this_thread::sleep_for(1ms); + // Prune attempted on one or more handlers } else { - shouldIdle = true; + prevFlushName = checkAndFlush(prevFlushName); + if (prevFlushName.empty()) { + wait(idleInterval); + } } - LOG(debug, "Making another wait(idle=%s, timeS=%1.3f) last was '%s'", - shouldIdle ? "true" : "false", shouldIdle ? vespalib::to_s(_idleInterval) : 0, prevFlushName.c_str()); } _executor.sync(); prune(); @@ -337,27 +360,21 @@ FlushEngine::flushAll(const FlushContext::List &lst) } } } + _executor.sync(); + prune(); + std::lock_guard<std::mutex> strategyGuard(_strategyLock); + _priorityStrategy.reset(); + _strategyCond.notify_all(); } vespalib::string -FlushEngine::flushNextTarget(const vespalib::string & name) +FlushEngine::flushNextTarget(const vespalib::string & name, const FlushContext::List & contexts) { - std::pair<FlushContext::List,bool> lst = getSortedTargetList(); - if (lst.second) { - // Everything returned from a priority strategy should be flushed - flushAll(lst.first); - _executor.sync(); - prune(); - std::lock_guard<std::mutex> strategyGuard(_strategyLock); - _priorityStrategy.reset(); - _strategyCond.notify_all(); - return ""; - } - if (lst.first.empty()) { + if (contexts.empty()) { LOG(debug, "No target to flush."); return ""; } - FlushContext::SP ctx = initNextFlush(lst.first); + FlushContext::SP ctx = initNextFlush(contexts); if ( ! ctx) { LOG(debug, "All targets refused to flush."); return ""; @@ -365,7 +382,7 @@ FlushEngine::flushNextTarget(const vespalib::string & name) if ( name == ctx->getName()) { LOG(info, "The same target %s out of %ld has been asked to flush again. " "This might indicate flush logic flaw so I will wait 100 ms before doing it.", - name.c_str(), lst.first.size()); + name.c_str(), contexts.size()); std::this_thread::sleep_for(100ms); } _executor.execute(std::make_unique<FlushTask>(initFlush(*ctx), *this, ctx)); @@ -445,7 +462,6 @@ FlushEngine::initFlush(const IFlushHandler::SP &handler, const IFlushTarget::SP { std::lock_guard<std::mutex> guard(_lock); taskId = _taskId++; - vespalib::string name(FlushContext::createName(*handler, *target)); FlushInfo flush(taskId, handler->getName(), target); _flushing[taskId] = flush; } @@ -459,7 +475,7 @@ FlushEngine::setStrategy(IFlushStrategy::SP strategy) { std::lock_guard<std::mutex> setStrategyGuard(_setStrategyLock); std::unique_lock<std::mutex> strategyGuard(_strategyLock); - if (_closed) { + if (_closed.load(std::memory_order_relaxed)) { return; } assert(!_priorityStrategy); diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h index 1d6ed763ff6..0be086729fa 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h @@ -50,7 +50,7 @@ private: }; using FlushMap = std::map<uint32_t, FlushInfo>; using FlushHandlerMap = HandlerMap<IFlushHandler>; - bool _closed; + std::atomic<bool> _closed; const uint32_t _maxConcurrent; const vespalib::duration _idleInterval; uint32_t _taskId; @@ -75,15 +75,20 @@ private: std::pair<FlushContext::List,bool> getSortedTargetList(); std::shared_ptr<search::IFlushToken> get_flush_token(const FlushContext& ctx); FlushContext::SP initNextFlush(const FlushContext::List &lst); - vespalib::string flushNextTarget(const vespalib::string & name); + vespalib::string flushNextTarget(const vespalib::string & name, const FlushContext::List & contexts); void flushAll(const FlushContext::List &lst); bool prune(); uint32_t initFlush(const FlushContext &ctx); uint32_t initFlush(const IFlushHandler::SP &handler, const IFlushTarget::SP &target); void flushDone(const FlushContext &ctx, uint32_t taskId); bool canFlushMore(const std::unique_lock<std::mutex> &guard) const; + void wait_for_slot_or_pending_prune(IFlushTarget::Priority priority); bool wait(vespalib::duration minimumWaitTimeIfReady, bool ignorePendingPrune); + void wait(vespalib::duration minimumWaitTimeIfReady) { + wait(minimumWaitTimeIfReady, false); + } bool isFlushing(const std::lock_guard<std::mutex> &guard, const vespalib::string & name) const; + vespalib::string checkAndFlush(vespalib::string prev); friend class FlushTask; friend class FlushEngineExplorer; diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushtargetproxy.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushtargetproxy.cpp index eb35e2b2eb1..b62a9191625 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushtargetproxy.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushtargetproxy.cpp @@ -22,61 +22,10 @@ FlushTargetProxy::FlushTargetProxy(const IFlushTarget::SP &target, { } - -IFlushTarget::MemoryGain -FlushTargetProxy::getApproxMemoryGain() const -{ - return _target->getApproxMemoryGain(); -} - - -IFlushTarget::DiskGain -FlushTargetProxy::getApproxDiskGain() const -{ - return _target->getApproxDiskGain(); -} - - -IFlushTarget::SerialNum -FlushTargetProxy::getFlushedSerialNum() const -{ - return _target->getFlushedSerialNum(); -} - - -IFlushTarget::Time -FlushTargetProxy::getLastFlushTime() const -{ - return _target->getLastFlushTime(); -} - - -bool -FlushTargetProxy::needUrgentFlush() const -{ - return _target->needUrgentFlush(); -} - - IFlushTarget::Task::UP FlushTargetProxy::initFlush(SerialNum currentSerial, std::shared_ptr<search::IFlushToken> flush_token) { return _target->initFlush(currentSerial, std::move(flush_token)); } - -FlushStats -FlushTargetProxy::getLastFlushStats() const -{ - return _target->getLastFlushStats(); -} - - -uint64_t -FlushTargetProxy::getApproxBytesToWriteToDisk() const -{ - return _target->getApproxBytesToWriteToDisk(); -} - - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushtargetproxy.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushtargetproxy.h index 0967ca1d6c1..e2ac832e023 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushtargetproxy.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushtargetproxy.h @@ -41,14 +41,16 @@ public: */ const IFlushTarget::SP & getFlushTarget() const { return _target; } // Implements IFlushTarget. - MemoryGain getApproxMemoryGain() const override; - DiskGain getApproxDiskGain() const override; - SerialNum getFlushedSerialNum() const override; - Time getLastFlushTime() const override; - bool needUrgentFlush() const override; + MemoryGain getApproxMemoryGain() const override { return _target->getApproxMemoryGain(); } + DiskGain getApproxDiskGain() const override { return _target->getApproxDiskGain(); } + SerialNum getFlushedSerialNum() const override { return _target->getFlushedSerialNum(); } + Time getLastFlushTime() const override { return _target->getLastFlushTime(); } + bool needUrgentFlush() const override { return _target->needUrgentFlush(); } + Priority getPriority() const override { return _target->getPriority(); } + uint64_t getApproxBytesToWriteToDisk() const override { return _target->getApproxBytesToWriteToDisk(); } + searchcorespi::FlushStats getLastFlushStats() const override { return _target->getLastFlushStats(); } + double get_replay_operation_cost() const override { return _target->get_replay_operation_cost(); } Task::UP initFlush(SerialNum currentSerial, std::shared_ptr<search::IFlushToken> flush_token) override; - searchcorespi::FlushStats getLastFlushStats() const override; - uint64_t getApproxBytesToWriteToDisk() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.cpp index b824f2a7c88..d292f8347a7 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.cpp @@ -6,6 +6,7 @@ namespace proton { using searchcorespi::IFlushTarget; +using searchcorespi::LeafFlushTarget; using searchcorespi::FlushStats; using searchcorespi::FlushTask; @@ -46,7 +47,7 @@ ShrinkLidSpaceFlushTarget::ShrinkLidSpaceFlushTarget(const vespalib::string &nam SerialNum flushedSerialNum, Time lastFlushTime, std::shared_ptr<ICompactableLidSpace> target) - : IFlushTarget(name, type, component), + : LeafFlushTarget(name, type, component), _target(std::move(target)), _flushedSerialNum(flushedSerialNum), @@ -80,12 +81,6 @@ ShrinkLidSpaceFlushTarget::getLastFlushTime() const return _lastFlushTime; } -bool -ShrinkLidSpaceFlushTarget::needUrgentFlush() const -{ - return false; -} - IFlushTarget::Task::UP ShrinkLidSpaceFlushTarget::initFlush(SerialNum currentSerial, std::shared_ptr<search::IFlushToken>) { diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h b/searchcore/src/vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h index e3efbd9b273..13f221e40a7 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h @@ -11,7 +11,7 @@ namespace proton { /** * Implements a flush target that shrinks lid space in target. */ -class ShrinkLidSpaceFlushTarget : public searchcorespi::IFlushTarget +class ShrinkLidSpaceFlushTarget : public searchcorespi::LeafFlushTarget { /** * Task representing that shrinking has been performed. @@ -46,7 +46,6 @@ public: DiskGain getApproxDiskGain() const override; SerialNum getFlushedSerialNum() const override; Time getLastFlushTime() const override; - bool needUrgentFlush() const override; Task::UP initFlush(SerialNum currentSerial, std::shared_ptr<search::IFlushToken> flush_token) override; searchcorespi::FlushStats getLastFlushStats() const override; uint64_t getApproxBytesToWriteToDisk() const override; diff --git a/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.cpp b/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.cpp index 5b49c724c6f..852ed73dc74 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.cpp @@ -30,10 +30,4 @@ JobTrackedFlushTarget::initFlush(SerialNum currentSerial, std::shared_ptr<search return FlushTask::UP(); } -uint64_t -JobTrackedFlushTarget::getApproxBytesToWriteToDisk() const -{ - return _target->getApproxBytesToWriteToDisk(); -} - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.h b/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.h index 35d1b0b0b12..c09c1fac055 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/job_tracked_flush_target.h @@ -19,7 +19,7 @@ private: public: JobTrackedFlushTarget(std::shared_ptr<IJobTracker> tracker, std::shared_ptr<searchcorespi::IFlushTarget> target); - ~JobTrackedFlushTarget(); + ~JobTrackedFlushTarget() override; const IJobTracker &getTracker() const { return *_tracker; } const searchcorespi::IFlushTarget &getTarget() const { return *_target; } @@ -40,13 +40,18 @@ public: bool needUrgentFlush() const override { return _target->needUrgentFlush(); } + double get_replay_operation_cost() const override { + return _target->get_replay_operation_cost(); + } + Priority getPriority() const override { return _target->getPriority(); } searchcorespi::FlushTask::UP initFlush(SerialNum currentSerial, std::shared_ptr<search::IFlushToken> flush_token) override; searchcorespi::FlushStats getLastFlushStats() const override { return _target->getLastFlushStats(); } - uint64_t getApproxBytesToWriteToDisk() const override; + uint64_t getApproxBytesToWriteToDisk() const override { + return _target->getApproxBytesToWriteToDisk(); + } }; } // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_flush_target.cpp b/searchcore/src/vespa/searchcore/proton/test/dummy_flush_target.cpp index 8915e3b367c..29b31dd2add 100644 --- a/searchcore/src/vespa/searchcore/proton/test/dummy_flush_target.cpp +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_flush_target.cpp @@ -5,10 +5,10 @@ namespace proton::test { DummyFlushTarget::DummyFlushTarget(const vespalib::string &name) noexcept - : searchcorespi::IFlushTarget(name) + : searchcorespi::LeafFlushTarget(name, Type::OTHER, Component::OTHER) {} DummyFlushTarget::DummyFlushTarget(const vespalib::string &name, const Type &type, const Component &component) noexcept - : searchcorespi::IFlushTarget(name, type, component) + : searchcorespi::LeafFlushTarget(name, type, component) {} DummyFlushTarget::~DummyFlushTarget() = default; diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_flush_target.h b/searchcore/src/vespa/searchcore/proton/test/dummy_flush_target.h index a9206233c9d..6b261f7bc4e 100644 --- a/searchcore/src/vespa/searchcore/proton/test/dummy_flush_target.h +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_flush_target.h @@ -5,7 +5,7 @@ namespace proton::test { -struct DummyFlushTarget : public searchcorespi::IFlushTarget +struct DummyFlushTarget : public searchcorespi::LeafFlushTarget { DummyFlushTarget(const vespalib::string &name) noexcept; DummyFlushTarget(const vespalib::string &name, const Type &type, const Component &component) noexcept; @@ -14,7 +14,6 @@ struct DummyFlushTarget : public searchcorespi::IFlushTarget DiskGain getApproxDiskGain() const override { return DiskGain(0, 0); } SerialNum getFlushedSerialNum() const override { return 0; } Time getLastFlushTime() const override { return Time(); } - bool needUrgentFlush() const override { return false; } searchcorespi::FlushTask::UP initFlush(SerialNum, std::shared_ptr<search::IFlushToken>) override { return searchcorespi::FlushTask::UP(); } diff --git a/searchcore/src/vespa/searchcorespi/flush/iflushtarget.cpp b/searchcore/src/vespa/searchcorespi/flush/iflushtarget.cpp index d821e06a2a3..b31113e1abc 100644 --- a/searchcore/src/vespa/searchcorespi/flush/iflushtarget.cpp +++ b/searchcore/src/vespa/searchcorespi/flush/iflushtarget.cpp @@ -16,4 +16,8 @@ IFlushTarget::IFlushTarget(const vespalib::string &name, const Type &type, const IFlushTarget::~IFlushTarget() = default; +LeafFlushTarget::LeafFlushTarget(const vespalib::string &name, const Type &type, const Component &component) noexcept + : IFlushTarget(name, type, component) +{} + } diff --git a/searchcore/src/vespa/searchcorespi/flush/iflushtarget.h b/searchcore/src/vespa/searchcorespi/flush/iflushtarget.h index e3ddf98ce9f..9e960757115 100644 --- a/searchcore/src/vespa/searchcorespi/flush/iflushtarget.h +++ b/searchcore/src/vespa/searchcorespi/flush/iflushtarget.h @@ -38,6 +38,11 @@ public: OTHER }; + enum class Priority { + NORMAL = 50, + HIGH = 100 + }; + private: vespalib::string _name; Type _type; @@ -133,7 +138,7 @@ public: /** * Return cost of replaying a feed operation relative to cost of reading a feed operation from tls. */ - virtual double get_replay_operation_cost() const { return 0.0; } + virtual double get_replay_operation_cost() const = 0; /** * Returns the last serial number for the transaction applied to @@ -156,7 +161,10 @@ public: * * @return true if an urgent flush is needed */ - virtual bool needUrgentFlush() const { return false; } + virtual bool needUrgentFlush() const = 0; + + /// Returns a priority for this target + virtual Priority getPriority() const = 0; /** * Initiates the flushing of temporary memory. This method must perform @@ -175,7 +183,14 @@ public: * @return The stats for the last flush. */ virtual FlushStats getLastFlushStats() const = 0; +}; +class LeafFlushTarget : public IFlushTarget { +public: + LeafFlushTarget(const vespalib::string &name, const Type &type, const Component &component) noexcept; + bool needUrgentFlush() const override { return false; } + Priority getPriority() const override { return Priority::NORMAL; } + double get_replay_operation_cost() const override { return 0.0; } }; } // namespace searchcorespi diff --git a/searchcore/src/vespa/searchcorespi/index/indexflushtarget.cpp b/searchcore/src/vespa/searchcorespi/index/indexflushtarget.cpp index 53fb21bf1ed..b5a5e2c2843 100644 --- a/searchcore/src/vespa/searchcorespi/index/indexflushtarget.cpp +++ b/searchcore/src/vespa/searchcorespi/index/indexflushtarget.cpp @@ -10,7 +10,7 @@ LOG_SETUP(".searchcorespi.index.indexflushtarget"); namespace searchcorespi::index { IndexFlushTarget::IndexFlushTarget(IndexMaintainer &indexMaintainer, IndexMaintainer::FlushStats flushStats) - : IFlushTarget("memoryindex.flush", Type::FLUSH, Component::INDEX), + : LeafFlushTarget("memoryindex.flush", Type::FLUSH, Component::INDEX), _indexMaintainer(indexMaintainer), _flushStats(flushStats), _numFrozenMemoryIndexes(indexMaintainer.getNumFrozenMemoryIndexes()), diff --git a/searchcore/src/vespa/searchcorespi/index/indexflushtarget.h b/searchcore/src/vespa/searchcorespi/index/indexflushtarget.h index 2b9ecc9574b..9f524bc341d 100644 --- a/searchcore/src/vespa/searchcorespi/index/indexflushtarget.h +++ b/searchcore/src/vespa/searchcorespi/index/indexflushtarget.h @@ -9,7 +9,7 @@ namespace searchcorespi::index { /** * Flush target for flushing a memory index in an IndexMaintainer. **/ -class IndexFlushTarget : public IFlushTarget { +class IndexFlushTarget : public LeafFlushTarget { private: IndexMaintainer &_indexMaintainer; const IndexMaintainer::FlushStats _flushStats; @@ -29,6 +29,7 @@ public: Time getLastFlushTime() const override; bool needUrgentFlush() const override; + Priority getPriority() const override { return Priority::HIGH; } Task::UP initFlush(SerialNum currentSerial, std::shared_ptr<search::IFlushToken> flush_token) override; FlushStats getLastFlushStats() const override { return _lastStats; } diff --git a/searchcore/src/vespa/searchcorespi/index/indexfusiontarget.cpp b/searchcore/src/vespa/searchcorespi/index/indexfusiontarget.cpp index 6755976939b..562d49a4348 100644 --- a/searchcore/src/vespa/searchcorespi/index/indexfusiontarget.cpp +++ b/searchcore/src/vespa/searchcorespi/index/indexfusiontarget.cpp @@ -38,7 +38,7 @@ public: } IndexFusionTarget::IndexFusionTarget(IndexMaintainer &indexMaintainer) - : IFlushTarget("memoryindex.fusion", Type::GC, Component::INDEX), + : LeafFlushTarget("memoryindex.fusion", Type::GC, Component::INDEX), _indexMaintainer(indexMaintainer), _fusionStats(indexMaintainer.getFusionStats()), _lastStats() diff --git a/searchcore/src/vespa/searchcorespi/index/indexfusiontarget.h b/searchcore/src/vespa/searchcorespi/index/indexfusiontarget.h index 7a9f44e6612..2be7bcc33a9 100644 --- a/searchcore/src/vespa/searchcorespi/index/indexfusiontarget.h +++ b/searchcore/src/vespa/searchcorespi/index/indexfusiontarget.h @@ -9,7 +9,7 @@ namespace searchcorespi::index { /** * Flush target for doing fusion on disk indexes in an IndexMaintainer. **/ -class IndexFusionTarget : public IFlushTarget { +class IndexFusionTarget : public LeafFlushTarget { private: IndexMaintainer &_indexMaintainer; IndexMaintainer::FusionStats _fusionStats; diff --git a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp index 9d0b7259912..363193da110 100644 --- a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp +++ b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp @@ -528,8 +528,10 @@ double computeTransformedMipsChecked(TypedCells a, TypedCells b, bool check_inse double closeness_r = d_r->to_rawscore(result); EXPECT_DOUBLE_EQ(closeness_n, closeness_f); EXPECT_DOUBLE_EQ(closeness_n, closeness_r); - EXPECT_GT(closeness_n, 0.0); - EXPECT_LE(closeness_n, 1.0); + EXPECT_DOUBLE_EQ(closeness_n, -result); + EXPECT_DOUBLE_EQ(result, d_n->to_distance(closeness_n)); + EXPECT_DOUBLE_EQ(result, d_f->to_distance(closeness_f)); + EXPECT_DOUBLE_EQ(result, d_r->to_distance(closeness_r)); if (check_insert) { auto d_i = dbl_dff.for_insertion_vector(a); EXPECT_DOUBLE_EQ(d_i->calc(b), result); @@ -601,11 +603,6 @@ TEST(DistanceFunctionsTest, transformed_mips_growing_norm) EXPECT_DOUBLE_EQ(4.0, f->calc(t(p7))); EXPECT_DOUBLE_EQ(-4.0, f->calc(t(p8))); - // closeness - EXPECT_DOUBLE_EQ(0.25, f->to_rawscore(1.0)); - EXPECT_DOUBLE_EQ(0.50, f->to_rawscore(0.0)); - EXPECT_DOUBLE_EQ(0.75, f->to_rawscore(-1.0)); - // now "insert" a bigger vector f = dff.for_insertion_vector(t(p6)); EXPECT_DOUBLE_EQ(0.0, f->calc(t(p1))); @@ -618,12 +615,6 @@ TEST(DistanceFunctionsTest, transformed_mips_growing_norm) // now max squared norm is 32, so p1 is "closer" to itself f = dff.for_insertion_vector(t(p1)); EXPECT_DOUBLE_EQ(-32.0, f->calc(t(p1))); - // closeness (rawscore) is also different: - EXPECT_DOUBLE_EQ(0.25, f->to_rawscore(32.0)); - EXPECT_DOUBLE_EQ(1/3., f->to_rawscore(16.0)); - EXPECT_DOUBLE_EQ(0.50, f->to_rawscore(0.0)); - EXPECT_DOUBLE_EQ(2/3., f->to_rawscore(-16.0)); - EXPECT_DOUBLE_EQ(0.75, f->to_rawscore(-32.0)); // also closer to other small vectors EXPECT_DOUBLE_EQ(-31.0, f->calc(t(p2))); diff --git a/searchlib/src/vespa/searchlib/features/distancefeature.cpp b/searchlib/src/vespa/searchlib/features/distancefeature.cpp index f601c91a0b2..4f98625d0e1 100644 --- a/searchlib/src/vespa/searchlib/features/distancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/distancefeature.cpp @@ -58,11 +58,11 @@ ConvertRawscoreToDistance::execute(uint32_t docId) const TermFieldMatchData *tfmd = _md->resolveTermField(elem.handle); if (tfmd->getDocId() == docId) { feature_t invdist = tfmd->getRawScore(); - feature_t converted = (1.0 / invdist) - 1.0; + feature_t converted = elem.calc ? elem.calc->function().to_distance(invdist) : ((1.0 / invdist) - 1.0); min_distance = std::min(min_distance, converted); } else if (elem.calc) { feature_t invdist = elem.calc->calc_raw_score(docId); - feature_t converted = (1.0 / invdist) - 1.0; + feature_t converted = elem.calc->function().to_distance(invdist); min_distance = std::min(min_distance, converted); } } diff --git a/searchlib/src/vespa/searchlib/tensor/distance_function.h b/searchlib/src/vespa/searchlib/tensor/distance_function.h index 31a837d17f4..a06c451d5e2 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_function.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_function.h @@ -13,11 +13,23 @@ class DistanceConverter { public: virtual ~DistanceConverter() = default; - // convert threshold (external distance units) to internal units + /** + * Convert threshold (external distance units) to internal units. + */ virtual double convert_threshold(double threshold) const = 0; - // convert internal distance to rawscore (1.0 / (1.0 + d)) + /** + * Convert internal distance to rawscore (also used as closeness). + */ virtual double to_rawscore(double distance) const = 0; + + /** + * Convert rawscore to external distance. + * Override this when the rawscore is NOT defined as (1.0 / (1.0 + external_distance)). + */ + virtual double to_distance(double rawscore) const { + return (1.0 / rawscore) - 1.0; + } }; } diff --git a/searchlib/src/vespa/searchlib/tensor/mips_distance_transform.cpp b/searchlib/src/vespa/searchlib/tensor/mips_distance_transform.cpp index 1e238aaacc7..16f9eeeabc2 100644 --- a/searchlib/src/vespa/searchlib/tensor/mips_distance_transform.cpp +++ b/searchlib/src/vespa/searchlib/tensor/mips_distance_transform.cpp @@ -62,11 +62,10 @@ public: return threshold; } double to_rawscore(double distance) const override { - double dp = -distance; - double t1 = dp / _max_sq_norm; - double t2 = t1 / (1.0 + std::fabs(t1)); - double r = (t2 + 1.0) * 0.5; - return r; + return -distance; + } + double to_distance(double rawscore) const override { + return -rawscore; } double calc_with_limit(const vespalib::eval::TypedCells& rhs, double) const override { return calc(rhs); diff --git a/storage/src/tests/storageserver/documentapiconvertertest.cpp b/storage/src/tests/storageserver/documentapiconvertertest.cpp index 42944c81f13..5829aa83893 100644 --- a/storage/src/tests/storageserver/documentapiconvertertest.cpp +++ b/storage/src/tests/storageserver/documentapiconvertertest.cpp @@ -19,6 +19,7 @@ #include <vespa/storageapi/message/removelocation.h> #include <vespa/storageapi/message/stat.h> #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/documentapi/messagebus/messages/testandsetcondition.h> using document::Bucket; using document::BucketId; @@ -29,6 +30,7 @@ using document::DocumentId; using document::DocumentTypeRepo; using document::readDocumenttypesConfig; using document::test::makeDocumentBucket; +using documentapi::TestAndSetCondition; using namespace ::testing; using namespace std::chrono_literals; @@ -38,6 +40,7 @@ const DocumentId defaultDocId("id:test:text/html::0"); const BucketSpace defaultBucketSpace(5); const vespalib::string defaultSpaceName("myspace"); const Bucket defaultBucket(defaultBucketSpace, BucketId(0)); +const TestAndSetCondition my_condition("my condition"); struct MockBucketResolver : public BucketResolver { virtual Bucket bucketFromId(const DocumentId &documentId) const override { @@ -110,10 +113,13 @@ TEST_F(DocumentApiConverterTest, put) { documentapi::PutDocumentMessage putmsg(doc); putmsg.setTimestamp(1234); + putmsg.setCondition(my_condition); auto cmd = toStorageAPI<api::PutCommand>(putmsg); EXPECT_EQ(defaultBucket, cmd->getBucket()); ASSERT_EQ(cmd->getDocument().get(), doc.get()); + EXPECT_EQ(cmd->getCondition(), my_condition); + EXPECT_FALSE(cmd->get_create_if_non_existent()); std::unique_ptr<mbus::Reply> reply = putmsg.createReply(); ASSERT_TRUE(reply.get()); @@ -123,6 +129,18 @@ TEST_F(DocumentApiConverterTest, put) { auto mbusPut = toDocumentAPI<documentapi::PutDocumentMessage>(*cmd); ASSERT_EQ(mbusPut->getDocumentSP().get(), doc.get()); EXPECT_EQ(mbusPut->getTimestamp(), 1234); + EXPECT_EQ(mbusPut->getCondition(), my_condition); + EXPECT_FALSE(mbusPut->get_create_if_non_existent()); +} + +TEST_F(DocumentApiConverterTest, put_with_create) { + documentapi::PutDocumentMessage putmsg(std::make_shared<Document>(*_repo, _html_type, defaultDocId)); + putmsg.setCondition(my_condition); + putmsg.set_create_if_non_existent(true); + auto cmd = toStorageAPI<api::PutCommand>(putmsg); + EXPECT_TRUE(cmd->get_create_if_non_existent()); + auto mbusPut = toDocumentAPI<documentapi::PutDocumentMessage>(*cmd); + EXPECT_TRUE(mbusPut->get_create_if_non_existent()); } TEST_F(DocumentApiConverterTest, forwarded_put) { @@ -145,12 +163,14 @@ TEST_F(DocumentApiConverterTest, update) { documentapi::UpdateDocumentMessage updateMsg(update); updateMsg.setOldTimestamp(1234); updateMsg.setNewTimestamp(5678); + updateMsg.setCondition(my_condition); auto updateCmd = toStorageAPI<api::UpdateCommand>(updateMsg); EXPECT_EQ(defaultBucket, updateCmd->getBucket()); ASSERT_EQ(update.get(), updateCmd->getUpdate().get()); EXPECT_EQ(api::Timestamp(1234), updateCmd->getOldTimestamp()); EXPECT_EQ(api::Timestamp(5678), updateCmd->getTimestamp()); + EXPECT_EQ(my_condition, updateCmd->getCondition()); auto mbusReply = updateMsg.createReply(); ASSERT_TRUE(mbusReply.get()); @@ -160,13 +180,16 @@ TEST_F(DocumentApiConverterTest, update) { ASSERT_EQ((&mbusUpdate->getDocumentUpdate()), update.get()); EXPECT_EQ(api::Timestamp(1234), mbusUpdate->getOldTimestamp()); EXPECT_EQ(api::Timestamp(5678), mbusUpdate->getNewTimestamp()); + EXPECT_EQ(my_condition, mbusUpdate->getCondition()); } TEST_F(DocumentApiConverterTest, remove) { documentapi::RemoveDocumentMessage removemsg(defaultDocId); + removemsg.setCondition(my_condition); auto cmd = toStorageAPI<api::RemoveCommand>(removemsg); EXPECT_EQ(defaultBucket, cmd->getBucket()); EXPECT_EQ(defaultDocId, cmd->getDocumentId()); + EXPECT_EQ(my_condition, cmd->getCondition()); std::unique_ptr<mbus::Reply> reply = removemsg.createReply(); ASSERT_TRUE(reply.get()); @@ -175,6 +198,7 @@ TEST_F(DocumentApiConverterTest, remove) { auto mbusRemove = toDocumentAPI<documentapi::RemoveDocumentMessage>(*cmd); EXPECT_EQ(defaultDocId, mbusRemove->getDocumentId()); + EXPECT_EQ(my_condition, mbusRemove->getCondition()); } TEST_F(DocumentApiConverterTest, get) { diff --git a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp index e7eb7a752fb..ebf9c1be142 100644 --- a/storage/src/vespa/storage/storageserver/documentapiconverter.cpp +++ b/storage/src/vespa/storage/storageserver/documentapiconverter.cpp @@ -44,6 +44,7 @@ DocumentApiConverter::toStorageAPI(documentapi::DocumentMessage& fromMsg) document::Bucket bucket = bucketResolver()->bucketFromId(from.getDocument().getId()); auto to = std::make_unique<api::PutCommand>(bucket, from.stealDocument(), from.getTimestamp()); to->setCondition(from.getCondition()); + to->set_create_if_non_existent(from.get_create_if_non_existent()); toMsg = std::move(to); break; } @@ -205,6 +206,8 @@ DocumentApiConverter::toDocumentAPI(api::StorageCommand& fromMsg) auto & from(static_cast<api::PutCommand&>(fromMsg)); auto to = std::make_unique<documentapi::PutDocumentMessage>(from.getDocument()); to->setTimestamp(from.getTimestamp()); + to->setCondition(from.getCondition()); + to->set_create_if_non_existent(from.get_create_if_non_existent()); toMsg = std::move(to); break; } @@ -214,13 +217,16 @@ DocumentApiConverter::toDocumentAPI(api::StorageCommand& fromMsg) auto to = std::make_unique<documentapi::UpdateDocumentMessage>(from.getUpdate()); to->setOldTimestamp(from.getOldTimestamp()); to->setNewTimestamp(from.getTimestamp()); + to->setCondition(from.getCondition()); toMsg = std::move(to); break; } case api::MessageType::REMOVE_ID: { auto & from(static_cast<api::RemoveCommand&>(fromMsg)); - toMsg = std::make_unique<documentapi::RemoveDocumentMessage>(from.getDocumentId()); + auto to = std::make_unique<documentapi::RemoveDocumentMessage>(from.getDocumentId()); + to->setCondition(from.getCondition()); + toMsg = std::move(to); break; } case api::MessageType::VISITOR_INFO_ID: diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index e81292b4170..94bc3da7df9 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -461,13 +461,16 @@ void SearchVisitor::init(const Parameters & params) SearchVisitorFactory::SearchVisitorFactory(const config::ConfigUri & configUri) : VisitorFactory(), - _configUri(configUri) + _configUri(configUri), + _env(std::make_shared<SearchEnvironment>(_configUri)) {} +SearchVisitorFactory::~SearchVisitorFactory() = default; + std::shared_ptr<VisitorEnvironment> SearchVisitorFactory::makeVisitorEnvironment(StorageComponent&) { - return std::make_shared<SearchEnvironment>(_configUri); + return _env; } storage::Visitor* diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h index 24b7bd64aec..daeb4013ebd 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h @@ -490,12 +490,14 @@ private: class SearchVisitorFactory : public storage::VisitorFactory { config::ConfigUri _configUri; + std::shared_ptr<storage::VisitorEnvironment> _env; std::shared_ptr<storage::VisitorEnvironment> makeVisitorEnvironment(storage::StorageComponent&) override; storage::Visitor* makeVisitor(storage::StorageComponent&, storage::VisitorEnvironment&env, const vdslib::Parameters& params) override; public: explicit SearchVisitorFactory(const config::ConfigUri & configUri); + ~SearchVisitorFactory() override; }; } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java index 77aaf17419d..2f8ef8cbcc0 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.identityprovider.client; +import ai.vespa.metrics.ContainerMetrics; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; @@ -10,7 +11,6 @@ import com.yahoo.container.core.identity.IdentityConfig; import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider; import com.yahoo.container.jdisc.athenz.AthenzIdentityProviderException; import com.yahoo.jdisc.Metric; -import com.yahoo.metrics.ContainerMetrics; import com.yahoo.security.AutoReloadingX509KeyManager; import com.yahoo.security.KeyStoreBuilder; import com.yahoo.security.KeyUtils; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/LegacyAthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/LegacyAthenzIdentityProviderImpl.java index d699564a4ee..a9223c5477f 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/LegacyAthenzIdentityProviderImpl.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/LegacyAthenzIdentityProviderImpl.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.identityprovider.client; +import ai.vespa.metrics.ContainerMetrics; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; @@ -10,7 +11,6 @@ import com.yahoo.container.core.identity.IdentityConfig; import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider; import com.yahoo.container.jdisc.athenz.AthenzIdentityProviderException; import com.yahoo.jdisc.Metric; -import com.yahoo.metrics.ContainerMetrics; import com.yahoo.security.KeyStoreBuilder; import com.yahoo.security.MutableX509KeyManager; import com.yahoo.security.Pkcs10Csr; |