diff options
107 files changed, 661 insertions, 762 deletions
diff --git a/client/go/internal/vespa/document/dispatcher_test.go b/client/go/internal/vespa/document/dispatcher_test.go index 252bd94dff9..382d21501c3 100644 --- a/client/go/internal/vespa/document/dispatcher_test.go +++ b/client/go/internal/vespa/document/dispatcher_test.go @@ -61,8 +61,8 @@ func TestDispatcher(t *testing.T) { breaker := NewCircuitBreaker(time.Second, 0) dispatcher := NewDispatcher(feeder, throttler, breaker, io.Discard, false) docs := []Document{ - {Id: mustParseId("id:ns:type::doc1"), Operation: OperationPut, Fields: []byte(`{"foo": "123"}`)}, - {Id: mustParseId("id:ns:type::doc2"), Operation: OperationPut, Fields: []byte(`{"bar": "456"}`)}, + {Id: mustParseId("id:ns:type::doc1"), Operation: OperationPut, Body: []byte(`{"fields": {"foo": "123"}}`)}, + {Id: mustParseId("id:ns:type::doc2"), Operation: OperationPut, Body: []byte(`{"fields": {"bar": "456"}}`)}, } for _, d := range docs { dispatcher.Enqueue(d) @@ -192,7 +192,7 @@ func BenchmarkDocumentDispatching(b *testing.B) { throttler := newThrottler(8, clock.now) breaker := NewCircuitBreaker(time.Second, 0) dispatcher := NewDispatcher(feeder, throttler, breaker, io.Discard, false) - doc := Document{Id: mustParseId("id:ns:type::doc1"), Operation: OperationPut, Fields: []byte(`{"foo": "123"}`)} + doc := Document{Id: mustParseId("id:ns:type::doc1"), Operation: OperationPut, Body: []byte(`{"fields": {"foo": "123"}}`)} b.ResetTimer() // ignore setup time for n := 0; n < b.N; n++ { diff --git a/client/go/internal/vespa/document/document.go b/client/go/internal/vespa/document/document.go index a33c4a3c5af..8f884b223d7 100644 --- a/client/go/internal/vespa/document/document.go +++ b/client/go/internal/vespa/document/document.go @@ -20,8 +20,6 @@ import ( "github.com/go-json-experiment/json" ) -var asciiSpace = [256]uint8{'\t': 1, '\n': 1, '\v': 1, '\f': 1, '\r': 1, ' ': 1} - type Operation int const ( @@ -36,6 +34,11 @@ const ( jsonString json.Kind = '"' ) +var ( + fieldsPrefix = []byte(`{"fields":`) + fieldsSuffix = []byte("}") +) + // Id represents a Vespa document ID. type Id struct { id string @@ -110,14 +113,13 @@ func ParseId(serialized string) (Id, error) { type Document struct { Id Id Condition string - Fields []byte + Body []byte Operation Operation Create bool } // Decoder decodes documents from a JSON structure which is either an array of objects, or objects separated by newline. type Decoder struct { - r *bufio.Reader dec *json.Decoder buf bytes.Buffer @@ -145,37 +147,28 @@ func (d Document) String() string { if d.Create { sb.WriteString(", create=true") } - if d.Fields != nil { - sb.WriteString(", fields=") - sb.WriteString(string(d.Fields)) + if d.Body != nil { + sb.WriteString(", body=") + sb.WriteString(string(d.Body)) } return sb.String() } func (d *Decoder) guessMode() error { - for !d.array && !d.jsonl { - b, err := d.r.ReadByte() - if err != nil { - return err - } - // Skip leading whitespace - if b < 0x80 && asciiSpace[b] != 0 { - continue - } - switch json.Kind(b) { - case jsonObjectStart: - d.jsonl = true - case jsonArrayStart: - d.array = true - default: - return fmt.Errorf("unexpected token: %q", string(b)) - } - if err := d.r.UnreadByte(); err != nil { - return err - } - if err := d.readArrayDelim(true); err != nil { + if d.array || d.jsonl { + return nil + } + kind := d.dec.PeekKind() + switch kind { + case jsonArrayStart: + if _, err := d.readNext(jsonArrayStart); err != nil { return err } + d.array = true + case jsonObjectStart: + d.jsonl = true + default: + return fmt.Errorf("expected %s or %s, got %s", jsonArrayStart, jsonObjectStart, kind) } return nil } @@ -191,18 +184,6 @@ func (d *Decoder) readNext(kind json.Kind) (json.Token, error) { return t, nil } -func (d *Decoder) readArrayDelim(open bool) error { - if !d.array { - return nil - } - kind := jsonArrayEnd - if open { - kind = jsonArrayStart - } - _, err := d.readNext(kind) - return err -} - func (d *Decoder) readString() (string, error) { t, err := d.readNext(jsonString) if err != nil { @@ -231,7 +212,7 @@ func (d *Decoder) Decode() (Document, error) { return doc, err } -func (d *Decoder) readField(name string, doc *Document) error { +func (d *Decoder) readField(name string, offset int64, doc *Document) error { readId := false switch name { case "id", "put": @@ -259,9 +240,9 @@ func (d *Decoder) readField(name string, doc *Document) error { if _, err := d.readNext(jsonObjectStart); err != nil { return err } - start := d.dec.InputOffset() - 1 - // Skip data between the most recent ending position of fields and current offset - d.buf.Next(int(start - d.fieldsEnd)) + // Skip data between start of operation and start of fields + fieldsStart := d.dec.InputOffset() - 1 + d.buf.Next(int(fieldsStart - offset)) depth := 1 for depth > 0 { t, err := d.dec.ReadToken() @@ -276,10 +257,11 @@ func (d *Decoder) readField(name string, doc *Document) error { } } d.fieldsEnd = d.dec.InputOffset() - doc.Fields = make([]byte, int(d.fieldsEnd-start)) - if _, err := d.buf.Read(doc.Fields); err != nil { - return err - } + fields := d.buf.Next(int(d.fieldsEnd - fieldsStart)) + doc.Body = make([]byte, 0, len(fieldsPrefix)+len(fields)+len(fieldsSuffix)) + doc.Body = append(doc.Body, fieldsPrefix...) + doc.Body = append(doc.Body, fields...) + doc.Body = append(doc.Body, fieldsSuffix...) } if readId { s, err := d.readString() @@ -296,12 +278,13 @@ func (d *Decoder) readField(name string, doc *Document) error { } func (d *Decoder) decode() (Document, error) { + start := d.dec.InputOffset() if err := d.guessMode(); err != nil { return Document{}, err } - if d.dec.PeekKind() == jsonArrayEnd { + if d.array && d.dec.PeekKind() == jsonArrayEnd { // Reached end of the array holding document operations - if err := d.readArrayDelim(false); err != nil { + if _, err := d.readNext(jsonArrayEnd); err != nil { return Document{}, err } return Document{}, io.EOF @@ -319,13 +302,17 @@ loop: if err != nil { return Document{}, err } - if err := d.readField(t.String(), &doc); err != nil { + if err := d.readField(t.String(), start, &doc); err != nil { return Document{}, err } default: if _, err := d.readNext(jsonObjectEnd); err != nil { return Document{}, err } + // Drop operation from the buffer + start = max(start, d.fieldsEnd) + end := d.dec.InputOffset() + d.buf.Next(int(end - start)) break loop } } @@ -333,9 +320,9 @@ loop: } func NewDecoder(r io.Reader) *Decoder { - sz := 1 << 26 - d := &Decoder{r: bufio.NewReaderSize(r, sz)} - d.dec = json.NewDecoder(io.TeeReader(d.r, &d.buf)) + br := bufio.NewReaderSize(r, 1<<26) + d := &Decoder{} + d.dec = json.NewDecoder(io.TeeReader(br, &d.buf)) return d } diff --git a/client/go/internal/vespa/document/document_test.go b/client/go/internal/vespa/document/document_test.go index 71400314634..fbaa076ab9d 100644 --- a/client/go/internal/vespa/document/document_test.go +++ b/client/go/internal/vespa/document/document_test.go @@ -116,7 +116,8 @@ func feedInput(jsonl bool) string { "fields": { "foo" : "123", "bar": {"a": [1, 2, 3]}} }`, ` -{ + + { "put": "id:ns:type::doc2", "create": false, "condition": "foo", @@ -129,9 +130,13 @@ func feedInput(jsonl bool) string { `, ` { + "fields": {"qux": "789"}, "put": "id:ns:type::doc4", - "create": true, - "fields": {"qux": "789"} + "create": true +}`, + ` +{ + "remove": "id:ns:type::doc5" }`} if jsonl { return strings.Join(operations, "\n") @@ -141,16 +146,17 @@ func feedInput(jsonl bool) string { func testDocumentDecoder(t *testing.T, jsonLike string) { t.Helper() - r := NewDecoder(strings.NewReader(jsonLike)) + dec := NewDecoder(strings.NewReader(jsonLike)) want := []Document{ - {Id: mustParseId("id:ns:type::doc1"), Operation: OperationPut, Fields: []byte(`{ "foo" : "123", "bar": {"a": [1, 2, 3]}}`)}, - {Id: mustParseId("id:ns:type::doc2"), Operation: OperationPut, Condition: "foo", Fields: []byte(`{"bar": "456"}`)}, + {Id: mustParseId("id:ns:type::doc1"), Operation: OperationPut, Body: []byte(`{"fields":{ "foo" : "123", "bar": {"a": [1, 2, 3]}}}`)}, + {Id: mustParseId("id:ns:type::doc2"), Operation: OperationPut, Condition: "foo", Body: []byte(`{"fields":{"bar": "456"}}`)}, {Id: mustParseId("id:ns:type::doc3"), Operation: OperationRemove}, - {Id: mustParseId("id:ns:type::doc4"), Operation: OperationPut, Create: true, Fields: []byte(`{"qux": "789"}`)}, + {Id: mustParseId("id:ns:type::doc4"), Operation: OperationPut, Create: true, Body: []byte(`{"fields":{"qux": "789"}}`)}, + {Id: mustParseId("id:ns:type::doc5"), Operation: OperationRemove}, } got := []Document{} for { - doc, err := r.Decode() + doc, err := dec.Decode() if err == io.EOF { break } @@ -159,6 +165,13 @@ func testDocumentDecoder(t *testing.T, jsonLike string) { } got = append(got, doc) } + wantBufLen := 0 + if dec.array { + wantBufLen = 1 + } + if l := dec.buf.Len(); l != wantBufLen { + t.Errorf("got dec.buf.Len() = %d, want %d", l, wantBufLen) + } if !reflect.DeepEqual(got, want) { t.Errorf("got %+v, want %+v", got, want) } @@ -179,13 +192,13 @@ func TestDocumentDecoderInvalid(t *testing.T) { "fields": {"foo": "invalid } ` - r := NewDecoder(strings.NewReader(jsonLike)) - _, err := r.Decode() // first object is valid + dec := NewDecoder(strings.NewReader(jsonLike)) + _, err := dec.Decode() // first object is valid if err != nil { t.Errorf("unexpected error: %s", err) } - _, err = r.Decode() - wantErr := "invalid json at byte offset 109: json: invalid character '\\n' within string (expecting non-control character)" + _, err = dec.Decode() + wantErr := "invalid json at byte offset 110: json: invalid character '\\n' within string (expecting non-control character)" if err.Error() != wantErr { t.Errorf("want error %q, got %q", wantErr, err.Error()) } diff --git a/client/go/internal/vespa/document/http.go b/client/go/internal/vespa/document/http.go index 8f7ac5bfe63..ce57ac55f03 100644 --- a/client/go/internal/vespa/document/http.go +++ b/client/go/internal/vespa/document/http.go @@ -30,9 +30,6 @@ const ( ) var ( - fieldsPrefix = []byte(`{"fields":`) - fieldsSuffix = []byte("}") - defaultHeaders http.Header = map[string][]string{ "User-Agent": {fmt.Sprintf("Vespa CLI/%s", build.Version)}, "Content-Type": {"application/json; charset=utf-8"}, @@ -132,15 +129,6 @@ func writeQueryParam(sb *bytes.Buffer, start int, escape bool, k, v string) { } } -func writeRequestBody(w io.Writer, body []byte) error { - for _, b := range [][]byte{fieldsPrefix, body, fieldsSuffix} { - if _, err := w.Write(b); err != nil { - return err - } - } - return nil -} - func (c *Client) methodAndURL(d Document, sb *bytes.Buffer) (string, string) { httpMethod := "" switch d.Operation { @@ -229,7 +217,7 @@ func (c *Client) preparePending() { for pd := range c.pending { pd.buf = c.buffer() method, url := c.methodAndURL(pd.document, pd.buf) - pd.request, pd.err = c.createRequest(method, url, pd.document.Fields, pd.buf) + pd.request, pd.err = c.createRequest(method, url, pd.document.Body, pd.buf) pd.prepared <- true } } @@ -259,24 +247,23 @@ func (c *Client) createRequest(method, url string, body []byte, buf *bytes.Buffe if len(body) == 0 { return newRequest(method, url, nil, false) } - bodySize := len(fieldsPrefix) + len(body) + len(fieldsSuffix) - useGzip := c.options.Compression == CompressionGzip || (c.options.Compression == CompressionAuto && bodySize > 512) - buf.Grow(min(1024, bodySize)) + useGzip := c.options.Compression == CompressionGzip || (c.options.Compression == CompressionAuto && len(body) > 512) + var r io.Reader if useGzip { + buf.Grow(min(1024, len(body))) zw := c.gzipWriter(buf) defer c.gzippers.Put(zw) - if err := writeRequestBody(zw, body); err != nil { + if _, err := zw.Write(body); err != nil { return nil, err } if err := zw.Close(); err != nil { return nil, err } + r = buf } else { - if err := writeRequestBody(buf, body); err != nil { - return nil, err - } + r = bytes.NewReader(body) } - return newRequest(method, url, buf, useGzip) + return newRequest(method, url, r, useGzip) } func (c *Client) clientTimeout() time.Duration { @@ -295,7 +282,10 @@ func (c *Client) Send(document Document) Result { if err != nil { return resultWithErr(result, err) } - bodySize := buf.Len() + bodySize := len(document.Body) + if buf.Len() > 0 { + bodySize = buf.Len() + } resp, err := c.leastBusyClient().Do(req, c.clientTimeout()) if err != nil { return resultWithErr(result, err) diff --git a/client/go/internal/vespa/document/http_test.go b/client/go/internal/vespa/document/http_test.go index 7d636aa8d5c..6eda5f04fd6 100644 --- a/client/go/internal/vespa/document/http_test.go +++ b/client/go/internal/vespa/document/http_test.go @@ -3,7 +3,6 @@ package document import ( "bytes" "fmt" - "net/http" "reflect" "strings" "testing" @@ -62,16 +61,16 @@ func TestClientSend(t *testing.T) { method string url string }{ - {Document{Create: true, Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Fields: []byte(`{"foo": "123"}`)}, + {Document{Create: true, Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Body: []byte(`{"fields":{"foo": "123"}}`)}, "PUT", "https://example.com:1337/document/v1/ns/type/docid/doc1?timeout=5000ms&create=true"}, - {Document{Id: mustParseId("id:ns:type::doc2"), Operation: OperationUpdate, Fields: []byte(`{"foo": "456"}`)}, + {Document{Id: mustParseId("id:ns:type::doc2"), Operation: OperationUpdate, Body: []byte(`{"fields":{"foo": "456"}}`)}, "PUT", "https://example.com:1337/document/v1/ns/type/docid/doc2?timeout=5000ms"}, {Document{Id: mustParseId("id:ns:type::doc3"), Operation: OperationRemove}, "DELETE", "https://example.com:1337/document/v1/ns/type/docid/doc3?timeout=5000ms"}, - {Document{Condition: "foo", Id: mustParseId("id:ns:type::doc4"), Operation: OperationUpdate, Fields: []byte(`{"baz": "789"}`)}, + {Document{Condition: "foo", Id: mustParseId("id:ns:type::doc4"), Operation: OperationUpdate, Body: []byte(`{"fields":{"baz": "789"}}`)}, "PUT", "https://example.com:1337/document/v1/ns/type/docid/doc4?timeout=5000ms&condition=foo"}, } @@ -95,7 +94,6 @@ func TestClientSend(t *testing.T) { MaxLatency: time.Second, }, } - var wantBody bytes.Buffer if i < 3 { httpClient.NextResponseString(200, `{"message":"All good!"}`) wantRes.Status = StatusSuccess @@ -112,11 +110,6 @@ func TestClientSend(t *testing.T) { wantRes.Stats.Errors = 1 wantRes.Stats.BytesRecv = 36 } - if tt.method == http.MethodPut { - wantBody.WriteString(`{"fields":`) - wantBody.Write(doc.Fields) - wantBody.WriteString("}") - } res := client.Send(doc) wantRes.Stats.BytesSent = int64(len(httpClient.LastBody)) if !reflect.DeepEqual(res, wantRes) { @@ -133,8 +126,8 @@ func TestClientSend(t *testing.T) { if r.URL.String() != tt.url { t.Errorf("got r.URL = %q, want %q", r.URL, tt.url) } - if !bytes.Equal(httpClient.LastBody, wantBody.Bytes()) { - t.Errorf("got r.Body = %q, want %q", string(httpClient.LastBody), wantBody.String()) + if !bytes.Equal(httpClient.LastBody, doc.Body) { + t.Errorf("got r.Body = %q, want %q", string(httpClient.LastBody), doc.Body) } } want := Stats{ @@ -164,9 +157,9 @@ func TestClientSendCompressed(t *testing.T) { Timeout: time.Duration(5 * time.Second), }, []util.HTTPClient{httpClient}) - bigBody := fmt.Sprintf(`{"foo": "%s"}`, strings.Repeat("s", 512+1)) - bigDoc := Document{Create: true, Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Fields: []byte(bigBody)} - smallDoc := Document{Create: true, Id: mustParseId("id:ns:type::doc2"), Operation: OperationUpdate, Fields: []byte(`{"foo": "s"}`)} + bigBody := fmt.Sprintf(`{"fields": {"foo": "%s"}}`, strings.Repeat("s", 512+1)) + bigDoc := Document{Create: true, Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Body: []byte(bigBody)} + smallDoc := Document{Create: true, Id: mustParseId("id:ns:type::doc2"), Operation: OperationUpdate, Body: []byte(`{"fields": {"foo": "s"}}`)} var result Result client.options.Compression = CompressionNone @@ -307,7 +300,7 @@ func benchmarkClientSend(b *testing.B, compression Compression, document Documen } func makeDocument(size int) Document { - return Document{Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Fields: []byte(fmt.Sprintf(`{"foo": "%s"}`, randString(size)))} + return Document{Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Body: []byte(fmt.Sprintf(`{"fields": {"foo": "%s"}}`, randString(size)))} } func BenchmarkClientSendSmallUncompressed(b *testing.B) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 362bc7b0964..a60d4d45317 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -131,6 +131,11 @@ public class VespaMetricSet { addMetric(metrics, ConfigServerMetrics.ZK_CONNECTIONS.last()); addMetric(metrics, ConfigServerMetrics.ZK_OUTSTANDING_REQUESTS.last()); + addMetric(metrics, ConfigServerMetrics.CLUSTER_COST.last()); + addMetric(metrics, ConfigServerMetrics.CLUSTER_LOAD_IDEAL_CPU.last()); + addMetric(metrics, ConfigServerMetrics.CLUSTER_LOAD_IDEAL_MEMORY.last()); + addMetric(metrics, ConfigServerMetrics.CLUSTER_LOAD_IDEAL_DISK.last()); + return metrics; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 3cb0731c43a..e1d222e0546 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -932,9 +932,11 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { int nodeCount = deployState.zone().environment().isProduction() ? 2 : 1; deployState.getDeployLogger().logApplicationPackage(Level.INFO, "Using " + nodeCount + " nodes in " + cluster); var nodesSpec = NodesSpecification.dedicated(nodeCount, context); + ClusterSpec.Id clusterId = ClusterSpec.Id.from(cluster.getName()); var hosts = nodesSpec.provision(hostSystem, ClusterSpec.Type.container, - ClusterSpec.Id.from(cluster.getName()), + clusterId, + zoneEndpoint(context, clusterId), deployState.getDeployLogger(), false, context.clusterInfo().build()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index e044b97546c..43f045940c9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -28,6 +28,7 @@ import com.yahoo.vespa.model.container.docproc.DocprocChain; import com.yahoo.vespa.model.container.docproc.DocprocChains; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.search.IndexedSearchCluster; +import com.yahoo.vespa.model.search.IndexingDocproc; import com.yahoo.vespa.model.search.IndexingDocprocChain; import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.search.SearchNode; @@ -213,13 +214,17 @@ public class Content extends ConfigModel { /** Select/creates and initializes the indexing cluster coupled to this */ private void buildIndexingClusters(Content content, ConfigModelContext modelContext, ApplicationConfigProducerRoot root) { - if ( ! content.getCluster().getSearch().hasIndexedCluster()) return; - - IndexedSearchCluster indexedSearchCluster = content.getCluster().getSearch().getIndexed(); - if (indexedSearchCluster.hasExplicitIndexingCluster()) { - setExistingIndexingCluster(indexedSearchCluster, content.containers); + var search = content.getCluster().getSearch(); + if (!search.getIndexingDocproc().isPresent()) { + return; + } + var indexingDocproc = search.getIndexingDocproc().get(); + if (indexingDocproc.hasExplicitCluster()) { + setExistingIndexingCluster(content, indexingDocproc, content.containers); } else { - setContainerAsIndexingCluster(indexedSearchCluster, content, modelContext, root); + if (search.hasIndexedCluster()) { + setContainerAsIndexingCluster(search.getIndexed(), content, modelContext, root); + } } } @@ -237,18 +242,19 @@ public class Content extends ConfigModel { targetCluster = content.containers.iterator().next().getCluster(); addDocproc(targetCluster); - indexedSearchCluster.setIndexingClusterName(targetCluster.getName()); - addIndexingChainsTo(targetCluster, indexedSearchCluster); + var indexingDocproc = indexedSearchCluster.getIndexingDocproc(); + indexingDocproc.setClusterName(targetCluster.getName()); + addIndexingChainsTo(targetCluster, content, indexingDocproc); } } - private void setExistingIndexingCluster(IndexedSearchCluster cluster, Collection<ContainerModel> containers) { - String indexingClusterName = cluster.getIndexingClusterName(); + private void setExistingIndexingCluster(Content content, IndexingDocproc indexingDocproc, Collection<ContainerModel> containers) { + String indexingClusterName = indexingDocproc.getClusterName(content.getCluster().getName()); ContainerModel containerModel = findByName(indexingClusterName, containers); if (containerModel == null) - throw new IllegalArgumentException("Content cluster '" + cluster.getClusterName() + "' refers to docproc " + + throw new IllegalArgumentException("Content cluster '" + content.getCluster().getName() + "' refers to docproc " + "cluster '" + indexingClusterName + "', but this cluster does not exist."); - addIndexingChainsTo(containerModel.getCluster(), cluster); + addIndexingChainsTo(containerModel.getCluster(), content, indexingDocproc); } private ContainerModel findByName(String name, Collection<ContainerModel> containers) { @@ -258,19 +264,19 @@ public class Content extends ConfigModel { return null; } - private void addIndexingChainsTo(ContainerCluster<?> indexer, IndexedSearchCluster cluster) { + private void addIndexingChainsTo(ContainerCluster<?> indexer, Content content, IndexingDocproc indexingDocproc) { addIndexingChain(indexer); DocprocChain indexingChain; ComponentRegistry<DocprocChain> allChains = indexer.getDocprocChains().allChains(); - if (cluster.hasExplicitIndexingChain()) { - indexingChain = allChains.getComponent(cluster.getIndexingChainName()); + if (indexingDocproc.hasExplicitChain() && !indexingDocproc.getChainName().equals(IndexingDocprocChain.NAME)) { + indexingChain = allChains.getComponent(indexingDocproc.getChainName()); if (indexingChain == null) { - throw new IllegalArgumentException(cluster + " refers to docproc " + - "chain '" + cluster.getIndexingChainName() + + throw new IllegalArgumentException(content.getCluster() + " refers to docproc " + + "chain '" + indexingDocproc.getChainName() + "' for indexing, but this chain does not exist"); } else if (indexingChain.getId().getName().equals("default")) { - throw new IllegalArgumentException(cluster + " specifies the chain " + + throw new IllegalArgumentException(content.getCluster() + " specifies the chain " + "'default' as indexing chain. As the 'default' chain is run by default, " + "using it as the indexing chain will run it twice. " + "Use a different name for the indexing chain."); @@ -282,7 +288,7 @@ public class Content extends ConfigModel { indexingChain = allChains.getComponent(IndexingDocprocChain.NAME); } - cluster.setIndexingChain(indexingChain); + indexingDocproc.setChain(indexingChain); } private TreeConfigProducer<AnyConfigProducer> getDocProc(ApplicationConfigProducerRoot root) { @@ -301,7 +307,7 @@ public class Content extends ConfigModel { Content content, ConfigModelContext modelContext, ApplicationConfigProducerRoot root) { - String indexerName = cluster.getIndexingClusterName(); + String indexerName = cluster.getIndexingDocproc().getClusterName(content.getCluster().getName()); TreeConfigProducer<AnyConfigProducer> parent = getDocProc(root); ApplicationContainerCluster indexingCluster = new ApplicationContainerCluster(parent, "cluster." + indexerName, indexerName, modelContext.getDeployState()); ContainerModel indexingClusterModel = new ContainerModel(modelContext.withParent(parent).withId(indexingCluster.getSubId())); @@ -334,7 +340,7 @@ public class Content extends ConfigModel { indexingCluster.addContainers(nodes); addIndexingChain(indexingCluster); - cluster.setIndexingChain(indexingCluster.getDocprocChains().allChains().getComponent(IndexingDocprocChain.NAME)); + cluster.getIndexingDocproc().setChain(indexingCluster.getDocprocChains().allChains().getComponent(IndexingDocprocChain.NAME)); } private ContainerCluster<?> getContainerWithDocproc(Collection<ContainerModel> containers) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index a0240d28a3c..ec7acaf819f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.search.IndexedSearchCluster; +import com.yahoo.vespa.model.search.IndexingDocproc; import com.yahoo.vespa.model.search.NodeSpec; import com.yahoo.vespa.model.search.SchemaDefinitionXMLHandler; import com.yahoo.vespa.model.search.SearchCluster; @@ -57,6 +58,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> /** The single, indexed search cluster this sets up (supporting multiple document types), or null if none */ private IndexedSearchCluster indexedCluster; + private Optional<IndexingDocproc> indexingDocproc; private Redundancy redundancy; private final String clusterName; @@ -206,6 +208,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> double fractionOfMemoryReserved) { super(parent, "search"); + this.indexingDocproc = Optional.empty(); this.clusterName = clusterName; this.documentDefinitions = documentDefinitions; this.globallyDistributedDocuments = globallyDistributedDocuments; @@ -259,6 +262,10 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> throw new IllegalArgumentException("Duplicate indexed cluster '" + indexedCluster.getClusterName() + "'"); } indexedCluster = (IndexedSearchCluster)sc; + if (indexingDocproc.isPresent()) { + throw new IllegalArgumentException("Indexing docproc has previously been setup for streaming search"); + } + indexingDocproc = Optional.of(indexedCluster.getIndexingDocproc()); } clusters.put(sc.getClusterName(), sc); } @@ -458,6 +465,12 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> public Map<String, SearchCluster> getClusters() { return clusters; } public IndexedSearchCluster getIndexed() { return indexedCluster; } public boolean hasIndexedCluster() { return indexedCluster != null; } + public Optional<IndexingDocproc> getIndexingDocproc() { return indexingDocproc; } + public void setupStreamingSearchIndexingDocProc() { + if (indexingDocproc.isEmpty()) { + indexingDocproc = Optional.of(new IndexingDocproc()); + } + } public String getClusterName() { return clusterName; } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 66a99e1993c..dfdfa9303a7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -203,19 +203,24 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem if (docprocCluster != null) { docprocCluster = docprocCluster.trim(); } - if (c.getSearch().hasIndexedCluster()) { - if (docprocCluster != null && !docprocCluster.isEmpty()) { - c.getSearch().getIndexed().setIndexingClusterName(docprocCluster); - } - } - String docprocChain = e.stringAttribute("chain"); if (docprocChain != null) { docprocChain = docprocChain.trim(); } - if (c.getSearch().hasIndexedCluster()) { - if (docprocChain != null && !docprocChain.isEmpty()) { - c.getSearch().getIndexed().setIndexingChainName(docprocChain); + if (docprocCluster != null && !docprocCluster.isEmpty()) { + if (!c.getSearch().hasIndexedCluster() && !c.getSearch().getIndexingDocproc().isPresent() && + docprocChain != null && !docprocChain.isEmpty()) { + c.getSearch().setupStreamingSearchIndexingDocProc(); + } + var indexingDocproc = c.getSearch().getIndexingDocproc(); + if (indexingDocproc.isPresent()) { + indexingDocproc.get().setClusterName(docprocCluster); + } + } + if (docprocChain != null && !docprocChain.isEmpty()) { + var indexingDocproc = c.getSearch().getIndexingDocproc(); + if (indexingDocproc.isPresent()) { + indexingDocproc.get().setChainName(docprocChain); } } } @@ -451,7 +456,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem @Override public void getConfig(MessagetyperouteselectorpolicyConfig.Builder builder) { - if ( ! getSearch().hasIndexedCluster()) return; + if ( ! getSearch().getIndexingDocproc().isPresent()) return; DocumentProtocol.getConfig(builder, getConfigId()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java index ad0312705ca..6623efb599d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java @@ -110,7 +110,7 @@ public final class DocumentProtocol implements Protocol, for (ContentCluster cluster : Content.getContentClusters(repo)) { DocumentProtocolPoliciesConfig.Cluster.Builder clusterBuilder = new DocumentProtocolPoliciesConfig.Cluster.Builder(); addSelector(cluster.getConfigId(), cluster.getRoutingSelector(), clusterBuilder); - if (cluster.getSearch().hasIndexedCluster()) + if (cluster.getSearch().getIndexingDocproc().isPresent()) addRoutes(getDirectRouteName(cluster.getConfigId()), getIndexedRouteName(cluster.getConfigId()), clusterBuilder); else clusterBuilder.defaultRoute(cluster.getConfigId()); @@ -227,10 +227,11 @@ public final class DocumentProtocol implements Protocol, for (ContentCluster cluster : content) { RouteSpec spec = new RouteSpec(cluster.getConfigId()); - if (cluster.getSearch().hasIndexedCluster()) { + if (cluster.getSearch().getIndexingDocproc().isPresent()) { + var indexingDocproc = cluster.getSearch().getIndexingDocproc().get(); table.addRoute(spec.addHop("[MessageType:" + cluster.getConfigId() + "]")); table.addRoute(new RouteSpec(getIndexedRouteName(cluster.getConfigId())) - .addHop(cluster.getSearch().getIndexed().getIndexingServiceName()) + .addHop(indexingDocproc.getServiceName()) .addHop("[Content:cluster=" + cluster.getName() + "]")); table.addRoute(new RouteSpec(getDirectRouteName(cluster.getConfigId())) .addHop("[Content:cluster=" + cluster.getName() + "]")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 670460a9f9f..080a2ca43dc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -43,11 +43,7 @@ public class IndexedSearchCluster extends SearchCluster DispatchNodesConfig.Producer, ConfigInstance.Producer { - private String indexingClusterName = null; // The name of the docproc cluster to run indexing, by config. - private String indexingChainName = null; - - private DocprocChain indexingChain; // The actual docproc chain indexing for this. - + private IndexingDocproc indexingDocproc; private Tuning tuning; private SearchCoverage searchCoverage; @@ -77,6 +73,7 @@ public class IndexedSearchCluster extends SearchCluster public IndexedSearchCluster(TreeConfigProducer<AnyConfigProducer> parent, String clusterName, int index, ModelContext.FeatureFlags featureFlags) { super(parent, clusterName, index); + indexingDocproc = new IndexingDocproc(); documentDbsConfigProducer = new MultipleDocumentDatabasesConfigProducer(this, documentDbs); rootDispatch = new DispatchGroup(this); defaultDispatchPolicy = DispatchTuning.Builder.toDispatchPolicy(featureFlags.queryDispatchPolicy()); @@ -87,58 +84,7 @@ public class IndexedSearchCluster extends SearchCluster @Override protected IndexingMode getIndexingMode() { return IndexingMode.REALTIME; } - public final boolean hasExplicitIndexingCluster() { - return indexingClusterName != null; - } - - public final boolean hasExplicitIndexingChain() { - return indexingChainName != null; - } - - /** - * Returns the name of the docproc cluster running indexing for this search cluster. This is derived from the - * services file on initialization, this can NOT be used at runtime to determine indexing chain. When initialization - * is done, the {@link #getIndexingServiceName()} method holds the actual indexing docproc chain object. - * - * @return the name of the docproc cluster associated with this - */ - public String getIndexingClusterName() { - return hasExplicitIndexingCluster() ? indexingClusterName : getClusterName() + ".indexing"; - } - - public String getIndexingChainName() { - return indexingChainName; - } - - public void setIndexingChainName(String indexingChainName) { - this.indexingChainName = indexingChainName; - } - - /** - * Sets the name of the docproc cluster running indexing for this search cluster. This is for initial configuration, - * and will not reflect the actual indexing chain. See {@link #getIndexingClusterName} for more detail. - * - * @param name the name of the docproc cluster associated with this - */ - public void setIndexingClusterName(String name) { - indexingClusterName = name; - } - - public String getIndexingServiceName() { - return indexingChain.getServiceName(); - } - - /** - * Sets the docproc chain that will be running indexing for this search cluster. This is set by the - * {@link com.yahoo.vespa.model.content.Content} model during build. - * - * @param chain the chain that is to run indexing for this cluster - * @return this, to allow chaining - */ - public SearchCluster setIndexingChain(DocprocChain chain) { - indexingChain = chain; - return this; - } + public IndexingDocproc getIndexingDocproc() { return indexingDocproc; } public DispatchGroup getRootDispatch() { return rootDispatch; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocproc.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocproc.java new file mode 100644 index 00000000000..46f3e6f459d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocproc.java @@ -0,0 +1,68 @@ +package com.yahoo.vespa.model.search; + +import com.yahoo.vespa.model.container.docproc.DocprocChain; + +/** + * Utility class to track configuration for which indexing docproc to use by a search cluster. + */ +public class IndexingDocproc { + private String clusterName; // The name of the docproc cluster to run indexing, by config. + private String chainName; + + private DocprocChain chain; // The actual docproc chain indexing for this. + + public boolean hasExplicitCluster() { + return clusterName != null; + } + + public boolean hasExplicitChain() { + return chainName != null; + } + + /** + * Returns the name of the docproc cluster running indexing for this search cluster. This is derived from the + * services file on initialization, this can NOT be used at runtime to determine indexing chain. When initialization + * is done, the {@link #getServiceName()} method holds the actual indexing docproc chain object. + * + * @return the name of the docproc cluster associated with this + */ + public String getClusterName(String searchClusterName) { + return hasExplicitCluster() ? clusterName : searchClusterName + ".indexing"; + } + + public String getChainName() { + return chainName; + } + + public void setChainName(String name) { + chainName = name; + } + + /** + * Sets the name of the docproc cluster running indexing for this search cluster. This is for initial configuration, + * and will not reflect the actual indexing chain. See {@link #getClusterName} for more detail. + * + * @param name the name of the docproc cluster associated with this + */ + public void setClusterName(String name) { + clusterName = name; + } + + public String getServiceName() { + return chain.getServiceName(); + } + + /** + * Sets the docproc chain that will be running indexing for this search cluster. This is set by the + * {@link com.yahoo.vespa.model.content.Content} model during build. + * + * @param chain the chain that is to run indexing for this cluster + */ + public void setChain(DocprocChain chain) { this.chain = chain; } + + public IndexingDocproc() { + clusterName = null; + chainName = null; + chain = null; + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 14fe7bbcc36..f9b1edf4f35 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1397,4 +1397,42 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(2, fleetControllerConfigBuilder.build().max_number_of_groups_allowed_to_be_down()); } + private void assertIndexingDocprocEnabled(boolean indexed, boolean force, boolean expEnabled) + { + String services = "<?xml version='1.0' encoding='UTF-8' ?>" + + "<services version='1.0'>" + + " <container id='default' version='1.0'>" + + " <document-processing/>" + + " </container>" + + " <content id='search' version='1.0'>" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document-processing cluster='default'" + (force ? " chain='indexing'" : "") + "/>" + + " <document type='type1' mode='" + (indexed ? "index" : "streaming") + "'/>" + + " </documents>" + + " </content>" + + "</services>"; + VespaModel model = createEnd2EndOneNode(new TestProperties(), services); + var searchCluster = model.getContentClusters().get("search").getSearch(); + assertEquals(expEnabled, searchCluster.getIndexingDocproc().isPresent()); + } + + @Test + void testIndexingDocprocEnabledWhenIndexMode() + { + assertIndexingDocprocEnabled(true, false, true); + } + + @Test + void testIndexingDocprocNotEnabledWhenStreamingMode() + { + assertIndexingDocprocEnabled(false, false, false); + } + + @Test + void testIndexingDocprocEnabledWhenStreamingModeAndForced() + { + assertIndexingDocprocEnabled(false, true, true); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java index 4476e128196..ac9d0ad8724 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java @@ -220,7 +220,7 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest { fail("Expected exception"); } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().startsWith("Indexing cluster 'musiccluster' specifies the chain 'default' as indexing chain")); + assertTrue(e.getMessage().startsWith("content cluster 'musiccluster' specifies the chain 'default' as indexing chain")); } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index d1fd409fc93..36ba12c7cf8 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -362,7 +362,7 @@ public class NodeResources { appendDouble(sb, vcpu); sb.append(", memory: "); appendDouble(sb, memoryGb); - sb.append(" Gb, disk "); + sb.append(" Gb, disk: "); appendDouble(sb, diskGb); sb.append(" Gb"); if (bandwidthGbps > 0) { diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java index ae052c03556..230d28e04aa 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java @@ -6,6 +6,8 @@ import org.junit.jupiter.api.Test; import java.util.function.Supplier; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; /** @@ -20,11 +22,11 @@ public class NodeResourcesTest { @Test void testToString() { - assertEquals("[vcpu: 1.0, memory: 10.0 Gb, disk 100.0 Gb, architecture: any]", + assertEquals("[vcpu: 1.0, memory: 10.0 Gb, disk: 100.0 Gb, architecture: any]", new NodeResources(1., 10., 100., 0).toString()); - assertEquals("[vcpu: 0.3, memory: 3.3 Gb, disk 33.3 Gb, bandwidth: 0.3 Gbps, architecture: any]", + assertEquals("[vcpu: 0.3, memory: 3.3 Gb, disk: 33.3 Gb, bandwidth: 0.3 Gbps, architecture: any]", new NodeResources(1 / 3., 10 / 3., 100 / 3., 0.3).toString()); - assertEquals("[vcpu: 0.7, memory: 9.0 Gb, disk 66.7 Gb, bandwidth: 0.7 Gbps, architecture: any]", + assertEquals("[vcpu: 0.7, memory: 9.0 Gb, disk: 66.7 Gb, bandwidth: 0.7 Gbps, architecture: any]", new NodeResources(2 / 3., 8.97, 200 / 3., 0.67).toString()); } @@ -37,6 +39,33 @@ public class NodeResourcesTest { } @Test + void testSatisfies() { + var hostResources = new NodeResources(1, 2, 3, 1); + assertTrue(hostResources.satisfies(new NodeResources(1, 2, 3, 1))); + assertTrue(hostResources.satisfies(new NodeResources(1, 1, 1, 1))); + assertFalse(hostResources.satisfies(new NodeResources(2, 2, 3, 1))); + assertFalse(hostResources.satisfies(new NodeResources(1, 3, 3, 1))); + assertFalse(hostResources.satisfies(new NodeResources(1, 2, 4, 1))); + + var gpuHostResources = new NodeResources(1, 2, 3, 1, + NodeResources.DiskSpeed.fast, + NodeResources.StorageType.local, + NodeResources.Architecture.x86_64, + new NodeResources.GpuResources(1, 16)); + assertTrue(gpuHostResources.satisfies(new NodeResources(1, 2, 3, 1, + NodeResources.DiskSpeed.fast, + NodeResources.StorageType.local, + NodeResources.Architecture.x86_64, + new NodeResources.GpuResources(1, 16)))); + assertFalse(gpuHostResources.satisfies(new NodeResources(1, 2, 3, 1, + NodeResources.DiskSpeed.fast, + NodeResources.StorageType.local, + NodeResources.Architecture.x86_64, + new NodeResources.GpuResources(1, 32)))); + assertFalse(hostResources.satisfies(gpuHostResources)); + } + + @Test void benchmark() { NodeResources [] resouces = new NodeResources[100]; for (int i = 0; i < resouces.length; i++) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index 91ece6733e1..ac896338643 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -645,10 +645,16 @@ public class DeploymentStatus { Optional<Instant> platformReadyAt = step.dependenciesCompletedAt(change.withoutApplication(), Optional.of(job)); Optional<Instant> revisionReadyAt = step.dependenciesCompletedAt(change.withoutPlatform(), Optional.of(job)); + boolean failingUpgradeOnlyTests = ! jobs().type(systemTest(job.type()), stagingTest(job.type())) + .failingHardOn(Versions.from(change.withoutApplication(), application, deploymentFor(job), () -> systemVersion)) + .isEmpty(); + // If neither change is ready, we guess based on the specified rollout. if (platformReadyAt.isEmpty() && revisionReadyAt.isEmpty()) { return switch (rollout) { - case separate -> List.of(change.withoutApplication(), change); // Platform should stay ahead. + case separate -> ! failingUpgradeOnlyTests + ? List.of(change.withoutApplication(), change) // Platform should stay ahead ... + : List.of(change); // ... unless upgrade-only is failing tests. case leading -> List.of(change); // They should eventually join. case simultaneous -> List.of(change.withoutPlatform(), change); // Revision should get ahead. }; @@ -663,9 +669,6 @@ public class DeploymentStatus { // Both changes are ready for this step, and we look to the specified rollout to decide. boolean platformReadyFirst = platformReadyAt.get().isBefore(revisionReadyAt.get()); boolean revisionReadyFirst = revisionReadyAt.get().isBefore(platformReadyAt.get()); - boolean failingUpgradeOnlyTests = ! jobs().type(systemTest(job.type()), stagingTest(job.type())) - .failingHardOn(Versions.from(change.withoutApplication(), application, deploymentFor(job), () -> systemVersion)) - .isEmpty(); return switch (rollout) { case separate -> // Let whichever change rolled out first, keep rolling first, unless upgrade alone is failing. (platformReadyFirst || platformReadyAt.get().equals(Instant.EPOCH)) // Assume platform was first if no jobs have run yet. diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 96c1d7c545d..f1e8697cf41 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -170,7 +170,10 @@ public class UpgraderTest { // --- Failing application is repaired by changing the application, causing confidence to move above 'high' threshold // Deploy application change default0.submit(applicationPackage("default")); - default0.deploy(); + default0.runJob(systemTest) + .jobAborted(stagingTest) // New revision causes run with failing upgrade alone to be aborted. + .runJob(stagingTest) + .deploy(); tester.controllerTester().computeVersionStatus(); assertEquals(VespaVersion.Confidence.high, tester.controller().readVersionStatus().systemVersion().get().confidence()); diff --git a/document/src/vespa/document/annotation/spantree.h b/document/src/vespa/document/annotation/spantree.h index a0010fafa74..8745fb4421d 100644 --- a/document/src/vespa/document/annotation/spantree.h +++ b/document/src/vespa/document/annotation/spantree.h @@ -4,7 +4,6 @@ #include "annotation.h" #include <vector> -#include <cassert> namespace document { struct SpanNode; @@ -24,7 +23,6 @@ public: SpanTree(vespalib::stringref name, std::unique_ptr<T> root) : _name(name), _root(std::move(root)) { - assert(_root.get()); } ~SpanTree(); diff --git a/document/src/vespa/document/base/testdocman.cpp b/document/src/vespa/document/base/testdocman.cpp index 471a2f8c196..d5b24b51f24 100644 --- a/document/src/vespa/document/base/testdocman.cpp +++ b/document/src/vespa/document/base/testdocman.cpp @@ -7,6 +7,7 @@ #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/vespalib/util/rand48.h> #include <sstream> +#include <cassert> namespace document { diff --git a/document/src/vespa/document/select/valuenodes.cpp b/document/src/vespa/document/select/valuenodes.cpp index b3052cc07e2..06205e6b7d1 100644 --- a/document/src/vespa/document/select/valuenodes.cpp +++ b/document/src/vespa/document/select/valuenodes.cpp @@ -11,6 +11,7 @@ #include <vespa/vespalib/util/md5.h> #include <vespa/document/util/stringutil.h> #include <vespa/vespalib/text/lowercase.h> +#include <cassert> #include <iomanip> #include <sys/time.h> diff --git a/document/src/vespa/document/serialization/annotationdeserializer.cpp b/document/src/vespa/document/serialization/annotationdeserializer.cpp index 41bc9ec8aaa..c449029440f 100644 --- a/document/src/vespa/document/serialization/annotationdeserializer.cpp +++ b/document/src/vespa/document/serialization/annotationdeserializer.cpp @@ -40,7 +40,7 @@ unique_ptr<SpanTree> AnnotationDeserializer::readSpanTree() { deserializer.read(tree_name); _nodes.clear(); SpanNode::UP root = readSpanNode(); - unique_ptr<SpanTree> span_tree(new SpanTree(tree_name.getValue(), std::move(root))); + auto span_tree = std::make_unique<SpanTree>(tree_name.getValue(), std::move(root)); uint32_t annotation_count = getInt1_2_4Bytes(_stream); span_tree->reserveAnnotations(annotation_count); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 8c75f341572..0dee7709b52 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -420,6 +420,12 @@ public class Flags { "Takes effect at next host-admin tick", ZONE_ID); + public static final UnboundListFlag<String> WEIGHTED_ENDPOINT_RECORD_TTL = defineListFlag( + "weighted-endpoint-record-ttl", List.of(), String.class, List.of("jonmv"), "2023-05-16", "2023-06-16", + "A list of endpoints and custom TTLs, on the form \"endpoint-fqdn:TTL-seconds\". " + + "Where specified, CNAME records are used instead of the default ALIAS records, which have a default 60s TTL.", + "Takes effect at redeployment from controller"); + public static final UnboundBooleanFlag ENABLE_CONDITIONAL_PUT_REMOVE_WRITE_REPAIR = defineFeatureFlag( "enable-conditional-put-remove-write-repair", false, List.of("vekterli", "havardpe"), "2023-05-10", "2023-07-01", diff --git a/linguistics-components/src/main/java/com/yahoo/language/huggingface/HuggingFaceTokenizer.java b/linguistics-components/src/main/java/com/yahoo/language/huggingface/HuggingFaceTokenizer.java index b92e0678970..d812b85b82e 100644 --- a/linguistics-components/src/main/java/com/yahoo/language/huggingface/HuggingFaceTokenizer.java +++ b/linguistics-components/src/main/java/com/yahoo/language/huggingface/HuggingFaceTokenizer.java @@ -76,6 +76,7 @@ public class HuggingFaceTokenizer extends AbstractComponent implements Embedder, public String decode(List<Long> tokens, Language language) { return resolve(language).decode(toArray(tokens)); } @Override public void close() { models.forEach((__, model) -> model.close()); } + @Override public void deconstruct() { close(); } private ai.djl.huggingface.tokenizers.HuggingFaceTokenizer resolve(Language language) { // Disregard language if there is default model diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java index b791c843357..2728249333e 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java @@ -57,7 +57,7 @@ public class SimpleTokenizer implements Tokenizer { } /** Tokenize the input, and apply the given transform to each token string. */ - public Iterable<Token> tokenize(String input, Function<String, String> tokenProocessor) { + public Iterable<Token> tokenize(String input, Function<String, String> tokenProcessor) { if (input.isEmpty()) return List.of(); List<Token> tokens = new ArrayList<>(); @@ -71,7 +71,7 @@ public class SimpleTokenizer implements Tokenizer { String original = input.substring(prev, next); tokens.add(new SimpleToken(original).setOffset(prev) .setType(tokenType) - .setTokenString(tokenProocessor.apply(original))); + .setTokenString(tokenProcessor.apply(original))); prev = next; prevType = nextType; tokenType = prevType; diff --git a/logd/src/logd/empty_forwarder.cpp b/logd/src/logd/empty_forwarder.cpp index dda03c46c01..b601ea6d890 100644 --- a/logd/src/logd/empty_forwarder.cpp +++ b/logd/src/logd/empty_forwarder.cpp @@ -5,6 +5,7 @@ #include <vespa/log/exceptions.h> #include <vespa/log/log_message.h> #include <vespa/vespalib/util/size_literals.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".logd.empty_forwarder"); diff --git a/metrics/src/main/java/ai/vespa/metrics/ConfigServerMetrics.java b/metrics/src/main/java/ai/vespa/metrics/ConfigServerMetrics.java index 013c50e77cf..d323026e4ca 100644 --- a/metrics/src/main/java/ai/vespa/metrics/ConfigServerMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/ConfigServerMetrics.java @@ -27,7 +27,6 @@ public enum ConfigServerMetrics implements VespaMetrics { MAINTENANCE_DEPLOYMENT_TRANSIENT_FAILURE("maintenanceDeployment.transientFailure", Unit.OPERATION, "Number of maintenance deployments that failed with a transient failure"), MAINTENANCE_DEPLOYMENT_FAILURE("maintenanceDeployment.failure", Unit.OPERATION, "Number of maintenance deployments that failed with a permanent failure"), - // ZooKeeper related metrics ZK_CONNECTIONS_LOST("configserver.zkConnectionLost", Unit.CONNECTION, "Number of ZooKeeper connections lost"), ZK_RECONNECTED("configserver.zkReconnected", Unit.CONNECTION, "Number of ZooKeeper reconnections"), @@ -45,8 +44,13 @@ public enum ConfigServerMetrics implements VespaMetrics { ORCHESTRATOR_LOCK_ACQUIRE_TIMEOUT("orchestrator.lock.acquire-timedout", Unit.OPERATION, "Number of times zookeeper lock couldn't be acquired within timeout"), ORCHESTRATOR_LOCK_ACQUIRE("orchestrator.lock.acquire", Unit.OPERATION, "Number of attempts to acquire zookeeper lock"), ORCHESTRATOR_LOCK_ACQUIRED("orchestrator.lock.acquired", Unit.OPERATION, "Number of times zookeeper lock was acquired"), - ORCHESTRATOR_LOCK_HOLD_LATENCY("orchestrator.lock.hold-latency", Unit.SECOND, "Time zookeeper lock was held before it was released"); + ORCHESTRATOR_LOCK_HOLD_LATENCY("orchestrator.lock.hold-latency", Unit.SECOND, "Time zookeeper lock was held before it was released"), + // Node repository metrics + CLUSTER_COST("cluster.cost", Unit.DOLLAR_PER_HOUR, "The cost of the nodes allocated to a certain cluster, in $/hr"), + CLUSTER_LOAD_IDEAL_CPU("cluster.load.ideal.cpu", Unit.FRACTION, "The ideal cpu load of a certain cluster"), + CLUSTER_LOAD_IDEAL_MEMORY("cluster.load.ideal.memory", Unit.FRACTION, "The ideal memory load of a certain cluster"), + CLUSTER_LOAD_IDEAL_DISK("cluster.load.ideal.disk", Unit.FRACTION, "The ideal disk load of a certain cluster"); private final String name; private final Unit unit; diff --git a/metrics/src/main/java/ai/vespa/metrics/Unit.java b/metrics/src/main/java/ai/vespa/metrics/Unit.java index a2123d72246..ee6ea569fc4 100644 --- a/metrics/src/main/java/ai/vespa/metrics/Unit.java +++ b/metrics/src/main/java/ai/vespa/metrics/Unit.java @@ -12,6 +12,7 @@ public enum Unit { CONNECTION(BaseUnit.CONNECTION, "A link used for communication between a client and a server"), DOCUMENT(BaseUnit.DOCUMENT, "Vespa document, a collection of fields defined in a schema file"), DOCUMENTID(BaseUnit.DOCUMENTID, "A unique document identifier"), + DOLLAR_PER_HOUR(BaseUnit.DOLLAR, BaseUnit.HOUR, "Total current cost of the cluster in $/hr"), FAILURE(BaseUnit.FAILURE, "Failures, typically for requests, operations or nodes"), FILE(BaseUnit.FILE, "Data file stored on the disk on a node"), FRACTION(BaseUnit.FRACTION, "A value in the range [0..1]. Higher values can occur for some metrics, but would indicate the value is outside of the allowed range."), @@ -80,10 +81,12 @@ public enum Unit { CONNECTION("connection"), DOCUMENT("document"), DOCUMENTID("documentid"), + DOLLAR("dollar"), FAILURE("failure"), FILE("file"), FRACTION("fraction"), HIT("hit"), + HOUR("hour"), INSTANCE("instance"), ITEM("item"), MILLISECOND("millisecond", "ms"), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index 6a81c17d362..65e0bc558b2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -197,9 +197,11 @@ public class Cluster { Duration totalDuration = Duration.ZERO; for (ScalingEvent event : scalingEvents()) { if (event.duration().isEmpty()) continue; + // Assume we have missed timely recording completion if it is longer than 4 days, so ignore + if ( ! event.duration().get().minus(Duration.ofDays(4)).isNegative()) continue; + completedEventCount++; - // Assume we have missed timely recording completion if it is longer than 4 days - totalDuration = totalDuration.plus(maximum(Duration.ofDays(4), event.duration().get())); + totalDuration = totalDuration.plus(event.duration().get()); } if (completedEventCount == 0) { // Use defaults if (clusterSpec.isStateful()) return Duration.ofHours(12); @@ -223,10 +225,4 @@ public class Cluster { return duration; } - private static Duration maximum(Duration largestAllowed, Duration duration) { - if ( ! duration.minus(largestAllowed).isNegative()) - return largestAllowed; - return duration; - } - } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index 4f94f0fab53..f01f5a30870 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import ai.vespa.metrics.ConfigServerMetrics; import com.yahoo.collections.Pair; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; @@ -18,6 +19,7 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.Node.State; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.ClusterId; import com.yahoo.vespa.hosted.provision.persistence.CacheStats; @@ -118,7 +120,7 @@ public class MetricsReporter extends NodeRepositoryMaintainer { byCluster.forEach((clusterId, clusterNodes) -> { Metric.Context context = getContext(dimensions(clusterId.application(), clusterId.cluster())); updateExclusiveSwitchMetrics(clusterNodes, nodes, context); - updateClusterCostMetrics(clusterNodes, context); + updateClusterCostMetrics(clusterId, clusterNodes, context); }); } @@ -129,9 +131,16 @@ public class MetricsReporter extends NodeRepositoryMaintainer { metric.set("nodes.exclusiveSwitchFraction", exclusiveSwitchRatio,context); } - private void updateClusterCostMetrics(List<Node> clusterNodes, Metric.Context context) { + private void updateClusterCostMetrics(ClusterId clusterId, + List<Node> clusterNodes, Metric.Context context) { + var cluster = nodeRepository().applications().get(clusterId.application()) + .flatMap(application -> application.cluster(clusterId.cluster())); + if (cluster.isEmpty()) return; double cost = clusterNodes.stream().mapToDouble(node -> node.resources().cost()).sum(); - metric.set("cluster.cost", cost, context); + metric.set(ConfigServerMetrics.CLUSTER_COST.baseName(), cost, context); + metric.set(ConfigServerMetrics.CLUSTER_LOAD_IDEAL_CPU.baseName(), cluster.get().target().ideal().cpu(), context); + metric.set(ConfigServerMetrics.CLUSTER_LOAD_IDEAL_MEMORY.baseName(), cluster.get().target().ideal().memory(), context); + metric.set(ConfigServerMetrics.CLUSTER_LOAD_IDEAL_DISK.baseName(), cluster.get().target().ideal().disk(), context); } private void updateZoneMetrics() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java index 32fe9ba9f7b..5def863113c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java @@ -50,7 +50,7 @@ public abstract class OsUpgrader { /** The duration this leaves new nodes alone before scheduling any upgrade */ private Duration gracePeriod() { - return Duration.ofDays(1); + return nodeRepository.zone().system().isCd() ? Duration.ofHours(4) : Duration.ofDays(1); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 237a6657ccc..b0c7a25095e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -269,34 +269,32 @@ public class NodeRepositoryProvisioner implements Provisioner { private IllegalArgumentException newNoAllocationPossible(ClusterSpec spec, Limits limits) { StringBuilder message = new StringBuilder("No allocation possible within ").append(limits); - if (nodeRepository.exclusiveAllocation(spec)) - message.append(". Nearest allowed node resources: ").append(findNearestNodeResources(limits)); + if (nodeRepository.exclusiveAllocation(spec) && findNearestNodeResources(limits).isPresent()) + message.append(". Nearest allowed node resources: ").append(findNearestNodeResources(limits).get()); return new IllegalArgumentException(message.toString()); } - private NodeResources findNearestNodeResources(Limits limits) { - NodeResources nearestMin = nearestFlavorResources(limits.min().nodeResources()); - NodeResources nearestMax = nearestFlavorResources(limits.max().nodeResources()); - if (limits.min().nodeResources().distanceTo(nearestMin) < limits.max().nodeResources().distanceTo(nearestMax)) + private Optional<NodeResources> findNearestNodeResources(Limits limits) { + Optional<NodeResources> nearestMin = nearestFlavorResources(limits.min().nodeResources()); + Optional<NodeResources> nearestMax = nearestFlavorResources(limits.max().nodeResources()); + if (nearestMin.isEmpty()) return nearestMax; + if (nearestMax.isEmpty()) return nearestMin; + if (limits.min().nodeResources().distanceTo(nearestMin.get()) < limits.max().nodeResources().distanceTo(nearestMax.get())) return nearestMin; else return nearestMax; } /** Returns the advertised flavor resources which are nearest to the given resources */ - private NodeResources nearestFlavorResources(NodeResources requestedResources) { - NodeResources nearestHostResources = nodeRepository.flavors().getFlavors().stream() - .map(flavor -> nodeRepository.resourcesCalculator().advertisedResourcesOf(flavor)) - .filter(resources -> resources.diskSpeed().compatibleWith(requestedResources.diskSpeed())) - .filter(resources -> resources.storageType().compatibleWith(requestedResources.storageType())) - .filter(resources -> resources.architecture().compatibleWith(requestedResources.architecture())) - .min(Comparator.comparingDouble(resources -> resources.distanceTo(requestedResources))) - .orElseThrow() - .withBandwidthGbps(requestedResources.bandwidthGbps()); - if ( nearestHostResources.storageType() == NodeResources.StorageType.remote) - nearestHostResources = nearestHostResources.withDiskGb(requestedResources.diskGb()); - return nearestHostResources; + private Optional<NodeResources> nearestFlavorResources(NodeResources requestedResources) { + return nodeRepository.flavors().getFlavors().stream() + .map(flavor -> nodeRepository.resourcesCalculator().advertisedResourcesOf(flavor)) + .filter(resources -> resources.satisfies(requestedResources)) + .min(Comparator.comparingDouble(resources -> resources.distanceTo(requestedResources))) + .map(resources -> resources.withBandwidthGbps(requestedResources.bandwidthGbps())) + .map(resources -> resources.storageType() == NodeResources.StorageType.remote ? + resources.withDiskGb(requestedResources.diskGb()) : resources); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java index d75f51680d7..3c459871490 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java @@ -154,10 +154,10 @@ public class AutoscalingMaintainerTest { @Test public void test_toString() { - assertEquals("4 nodes with [vcpu: 1.0, memory: 2.0 Gb, disk 4.0 Gb, bandwidth: 1.0 Gbps, architecture: any] (total: [vcpu: 4.0, memory: 8.0 Gb, disk 16.0 Gb, bandwidth: 4.0 Gbps, architecture: any])", + assertEquals("4 nodes with [vcpu: 1.0, memory: 2.0 Gb, disk: 4.0 Gb, bandwidth: 1.0 Gbps, architecture: any] (total: [vcpu: 4.0, memory: 8.0 Gb, disk: 16.0 Gb, bandwidth: 4.0 Gbps, architecture: any])", AutoscalingMaintainer.toString(new ClusterResources(4, 1, new NodeResources(1, 2, 4, 1)))); - assertEquals("4 nodes (in 2 groups) with [vcpu: 1.0, memory: 2.0 Gb, disk 4.0 Gb, bandwidth: 1.0 Gbps, architecture: any] (total: [vcpu: 4.0, memory: 8.0 Gb, disk 16.0 Gb, bandwidth: 4.0 Gbps, architecture: any])", + assertEquals("4 nodes (in 2 groups) with [vcpu: 1.0, memory: 2.0 Gb, disk: 4.0 Gb, bandwidth: 1.0 Gbps, architecture: any] (total: [vcpu: 4.0, memory: 8.0 Gb, disk: 16.0 Gb, bandwidth: 4.0 Gbps, architecture: any])", AutoscalingMaintainer.toString(new ClusterResources(4, 2, new NodeResources(1, 2, 4, 1)))); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 487355a0b75..de2c060a0eb 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -19,8 +19,11 @@ import com.yahoo.vespa.curator.stats.LockStats; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.autoscale.Autoscaling; +import com.yahoo.vespa.hosted.provision.autoscale.Load; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; +import com.yahoo.vespa.hosted.provision.node.ClusterId; import com.yahoo.vespa.hosted.provision.node.Generation; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; @@ -240,7 +243,7 @@ public class MetricsReporterTest { } @Test - public void non_active_metric() { + public void node_and_cluster_metrics() { ProvisioningTester tester = new ProvisioningTester.Builder().build(); tester.makeReadyHosts(5, new NodeResources(64, 256, 2000, 10)); tester.activateTenantHosts(); @@ -248,18 +251,36 @@ public class MetricsReporterTest { MetricsReporter metricsReporter = metricsReporter(metric, tester); // Application is deployed - ApplicationId application = ApplicationId.from("t1", "a1", "default"); - Map<String, String> dimensions = Map.of("applicationId", application.toFullString()); + ApplicationId applicationId = ApplicationId.from("t1", "a1", "default"); + ClusterSpec clusterSpec = ProvisioningTester.contentClusterSpec(); NodeResources resources = new NodeResources(2, 8, 100, 1); - List<Node> activeNodes = tester.deploy(application, ProvisioningTester.contentClusterSpec(), Capacity.from(new ClusterResources(4, 1, resources))); + Capacity capacity = Capacity.from(new ClusterResources(4, 1, resources)); + + List<Node> activeNodes = tester.deploy(applicationId, clusterSpec, capacity); + var application = tester.nodeRepository().applications().require(applicationId); + application = application.withCluster(clusterSpec.id(), false, capacity); + var cluster = application.cluster(clusterSpec.id()).get().withTarget(new Autoscaling(Autoscaling.Status.ideal, + "test", + Optional.empty(), + tester.clock().instant(), + Load.zero(), + new Load(0.1, 0.2, 0.3), + Autoscaling.Metrics.zero())); + tester.nodeRepository().applications().put(application.with(cluster), tester.nodeRepository().applications().lock(applicationId)); + metricsReporter.maintain(); + Map<String, String> dimensions = Map.of("applicationId", applicationId.toFullString()); assertEquals(0D, getMetric("nodes.nonActiveFraction", metric, dimensions)); assertEquals(4, getMetric("nodes.active", metric, dimensions)); assertEquals(0, getMetric("nodes.nonActive", metric, dimensions)); - Map<String, String> clusterDimensions = Map.of("applicationId", application.toFullString(), - "clusterid", ProvisioningTester.contentClusterSpec().id().value()); + + Map<String, String> clusterDimensions = Map.of("applicationId", applicationId.toFullString(), + "clusterid", clusterSpec.id().value()); assertEquals(1.392, getMetric("cluster.cost", metric, clusterDimensions)); + assertEquals(0.1, getMetric("cluster.load.ideal.cpu", metric, clusterDimensions)); + assertEquals(0.2, getMetric("cluster.load.ideal.memory", metric, clusterDimensions)); + assertEquals(0.3, getMetric("cluster.load.ideal.disk", metric, clusterDimensions)); // One node fails tester.fail(activeNodes.get(0).hostname()); @@ -269,7 +290,7 @@ public class MetricsReporterTest { assertEquals(1, getMetric("nodes.nonActive", metric, dimensions)); // Cluster is removed - tester.deactivate(application); + tester.deactivate(applicationId); metricsReporter.maintain(); assertEquals(1D, getMetric("nodes.nonActiveFraction", metric, dimensions).doubleValue(), Double.MIN_VALUE); assertEquals(0, getMetric("nodes.active", metric, dimensions)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java index f73d6f2ce01..bc54e552270 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -73,9 +73,9 @@ public class ScalingSuggestionsMaintainerTest { new TestMetric()); maintainer.maintain(); - assertEquals("8 nodes with [vcpu: 3.2, memory: 4.5 Gb, disk 10.0 Gb, bandwidth: 0.1 Gbps, architecture: any]", + assertEquals("8 nodes with [vcpu: 3.2, memory: 4.5 Gb, disk: 10.0 Gb, bandwidth: 0.1 Gbps, architecture: any]", suggestionOf(app1, cluster1, tester).resources().get().toString()); - assertEquals("8 nodes with [vcpu: 3.6, memory: 4.4 Gb, disk 11.8 Gb, bandwidth: 0.1 Gbps, architecture: any]", + assertEquals("8 nodes with [vcpu: 3.6, memory: 4.4 Gb, disk: 11.8 Gb, bandwidth: 0.1 Gbps, architecture: any]", suggestionOf(app2, cluster2, tester).resources().get().toString()); // Utilization goes way down @@ -83,14 +83,14 @@ public class ScalingSuggestionsMaintainerTest { addMeasurements(0.10f, 0.10f, 0.10f, 0, 500, app1, tester.nodeRepository()); maintainer.maintain(); assertEquals("Suggestion stays at the peak value observed", - "8 nodes with [vcpu: 3.2, memory: 4.5 Gb, disk 10.0 Gb, bandwidth: 0.1 Gbps, architecture: any]", + "8 nodes with [vcpu: 3.2, memory: 4.5 Gb, disk: 10.0 Gb, bandwidth: 0.1 Gbps, architecture: any]", suggestionOf(app1, cluster1, tester).resources().get().toString()); // Utilization is still way down and a week has passed tester.clock().advance(Duration.ofDays(7)); addMeasurements(0.10f, 0.10f, 0.10f, 0, 500, app1, tester.nodeRepository()); maintainer.maintain(); assertEquals("Peak suggestion has been outdated", - "3 nodes with [vcpu: 1.2, memory: 4.0 Gb, disk 10.0 Gb, bandwidth: 0.1 Gbps, architecture: any]", + "3 nodes with [vcpu: 1.2, memory: 4.0 Gb, disk: 10.0 Gb, bandwidth: 0.1 Gbps, architecture: any]", suggestionOf(app1, cluster1, tester).resources().get().toString()); assertTrue(shouldSuggest(app1, cluster1, tester)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java index f40c8037f41..0b4d345b8a5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java @@ -440,7 +440,7 @@ public class VirtualNodeProvisioningTest { catch (Exception e) { assertEquals("No room for 3 nodes as 2 of 4 hosts are exclusive", "Could not satisfy request for 3 nodes with " + - "[vcpu: 2.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, architecture: any] " + + "[vcpu: 2.0, memory: 4.0 Gb, disk: 100.0 Gb, bandwidth: 1.0 Gbps, architecture: any] " + "in tenant2.app2 container cluster 'my-container' 6.39: " + "Node allocation failure on group 0: " + "Not enough suitable nodes available due to host exclusivity constraints", @@ -467,7 +467,7 @@ public class VirtualNodeProvisioningTest { } catch (NodeAllocationException e) { assertEquals("Could not satisfy request for 2 nodes with " + - "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: remote, architecture: any] " + + "[vcpu: 1.0, memory: 4.0 Gb, disk: 100.0 Gb, bandwidth: 1.0 Gbps, storage type: remote, architecture: any] " + "in tenant.app1 content cluster 'my-content'" + " 6.42: Node allocation failure on group 0", e.getMessage()); @@ -549,8 +549,8 @@ public class VirtualNodeProvisioningTest { } catch (IllegalArgumentException e) { assertEquals("No allocation possible within limits: " + - "from 2 nodes with [vcpu: 1.0, memory: 5.0 Gb, disk 10.0 Gb, bandwidth: 1.0 Gbps, architecture: any] " + - "to 4 nodes with [vcpu: 1.0, memory: 5.0 Gb, disk 10.0 Gb, bandwidth: 1.0 Gbps, architecture: any]", + "from 2 nodes with [vcpu: 1.0, memory: 5.0 Gb, disk: 10.0 Gb, bandwidth: 1.0 Gbps, architecture: any] " + + "to 4 nodes with [vcpu: 1.0, memory: 5.0 Gb, disk: 10.0 Gb, bandwidth: 1.0 Gbps, architecture: any]", e.getMessage()); } } @@ -573,9 +573,9 @@ public class VirtualNodeProvisioningTest { } catch (IllegalArgumentException e) { assertEquals("No allocation possible within limits: " + - "from 2 nodes with [vcpu: 20.0, memory: 37.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, architecture: any] " + - "to 4 nodes with [vcpu: 20.0, memory: 37.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, architecture: any]. " + - "Nearest allowed node resources: [vcpu: 20.0, memory: 40.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: remote, architecture: any]", + "from 2 nodes with [vcpu: 20.0, memory: 37.0 Gb, disk: 100.0 Gb, bandwidth: 1.0 Gbps, architecture: any] " + + "to 4 nodes with [vcpu: 20.0, memory: 37.0 Gb, disk: 100.0 Gb, bandwidth: 1.0 Gbps, architecture: any]. " + + "Nearest allowed node resources: [vcpu: 20.0, memory: 40.0 Gb, disk: 100.0 Gb, bandwidth: 1.0 Gbps, storage type: remote, architecture: any]", e.getMessage()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-zone.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-zone.json index 0e14dd8b36f..87b823fbb33 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-zone.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-zone.json @@ -3,7 +3,7 @@ "couldLoseHosts": 4, "failedTenantParent": "dockerhost1.yahoo.com", "failedTenant": "host4.yahoo.com", - "failedTenantResources": "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", + "failedTenantResources": "[vcpu: 1.0, memory: 4.0 Gb, disk: 100.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", "failedTenantAllocation": "allocated to tenant3.application3.instance3 as 'content/id3/0/0/stateful'", "hostCandidateRejectionReasons": { "singularReasonFailures": { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-container1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-container1.json index 8ef88eae97d..b2212179647 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-container1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-container1.json @@ -5,7 +5,7 @@ "type": "tenant", "hostname": "test-node-pool-102-2", "parentHostname": "dockerhost3.yahoo.com", - "flavor": "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: remote, architecture: x86_64]", + "flavor": "[vcpu: 1.0, memory: 4.0 Gb, disk: 100.0 Gb, bandwidth: 1.0 Gbps, storage type: remote, architecture: x86_64]", "resources":{"vcpu":1.0,"memoryGb":4.0,"diskGb":100.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"remote","architecture":"x86_64"}, "realResources":{"vcpu":1.0,"memoryGb":4.0,"diskGb":100.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"remote","architecture":"x86_64"}, "environment": "DOCKER_CONTAINER", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node1.json index d90ed692f1c..0cd34f3551e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node1.json @@ -4,7 +4,7 @@ "state": "active", "type": "tenant", "hostname": "host1.yahoo.com", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", + "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk: 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local","architecture":"x86_64"}, "realResources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local","architecture":"x86_64"}, "environment": "DOCKER_CONTAINER", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node10.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node10.json index bec194ea325..4125f6e4913 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node10.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node10.json @@ -5,7 +5,7 @@ "type": "tenant", "hostname": "host10.yahoo.com", "parentHostname": "parent1.yahoo.com", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", + "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk: 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local","architecture":"x86_64"}, "realResources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local","architecture":"x86_64"}, "environment": "DOCKER_CONTAINER", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node11.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node11.json index d7e07f02f3a..892bb412edf 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node11.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node11.json @@ -5,7 +5,7 @@ "type": "tenant", "hostname": "host11.yahoo.com", "parentHostname": "parent.host.yahoo.com", - "flavor": "[vcpu: 1.0, memory: 1.0 Gb, disk 100.0 Gb, bandwidth: 0.3 Gbps, architecture: any]", + "flavor": "[vcpu: 1.0, memory: 1.0 Gb, disk: 100.0 Gb, bandwidth: 0.3 Gbps, architecture: any]", "resources":{"vcpu":1.0,"memoryGb":1.0,"diskGb":100.0,"bandwidthGbps":0.3,"diskSpeed":"fast","storageType":"any","architecture":"any"}, "realResources":{"vcpu":1.0,"memoryGb":1.0,"diskGb":100.0,"bandwidthGbps":0.3,"diskSpeed":"fast","storageType":"any","architecture":"any"}, "environment": "DOCKER_CONTAINER", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node13.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node13.json index 73c34a7fa9e..f5152efd7cb 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node13.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node13.json @@ -4,7 +4,7 @@ "state": "active", "type": "tenant", "hostname": "host13.yahoo.com", - "flavor": "[vcpu: 10.0, memory: 48.0 Gb, disk 500.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", + "flavor": "[vcpu: 10.0, memory: 48.0 Gb, disk: 500.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", "resources":{"vcpu":10.0,"memoryGb":48.0,"diskGb":500.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local","architecture":"x86_64"}, "realResources":{"vcpu":10.0,"memoryGb":48.0,"diskGb":500.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local","architecture":"x86_64"}, "environment": "DOCKER_CONTAINER", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node14.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node14.json index abb0ba57e49..f48e52b18bf 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node14.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node14.json @@ -4,7 +4,7 @@ "state": "active", "type": "tenant", "hostname": "host14.yahoo.com", - "flavor": "[vcpu: 10.0, memory: 48.0 Gb, disk 500.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", + "flavor": "[vcpu: 10.0, memory: 48.0 Gb, disk: 500.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", "resources":{"vcpu":10.0,"memoryGb":48.0,"diskGb":500.0,"bandwidthGbps":1.0, "diskSpeed":"fast","storageType":"local","architecture":"x86_64"}, "realResources":{"vcpu":10.0,"memoryGb":48.0,"diskGb":500.0,"bandwidthGbps":1.0, "diskSpeed":"fast","storageType":"local","architecture":"x86_64"}, "environment": "DOCKER_CONTAINER", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node2.json index 9cd675163f0..815a9a3d2d2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node2.json @@ -4,7 +4,7 @@ "state": "active", "type": "tenant", "hostname": "host2.yahoo.com", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", + "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk: 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local","architecture":"x86_64"}, "realResources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local","architecture":"x86_64"}, "environment": "DOCKER_CONTAINER", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node3.json index 1c560c2f95b..23f3594bba4 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node3.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node3.json @@ -4,7 +4,7 @@ "state": "ready", "type": "tenant", "hostname": "host3.yahoo.com", - "flavor": "[vcpu: 0.5, memory: 48.0 Gb, disk 500.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", + "flavor": "[vcpu: 0.5, memory: 48.0 Gb, disk: 500.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", "resources":{"vcpu":0.5,"memoryGb":48.0,"diskGb":500.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local","architecture":"x86_64"}, "realResources":{"vcpu":0.5,"memoryGb":48.0,"diskGb":500.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local","architecture":"x86_64"}, "environment": "DOCKER_CONTAINER", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-wg.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-wg.json index a1883ba4b25..0993c3fbcc2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-wg.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-wg.json @@ -5,7 +5,7 @@ "type": "tenant", "hostname": "host4.yahoo.com", "parentHostname": "dockerhost1.yahoo.com", - "flavor": "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", + "flavor": "[vcpu: 1.0, memory: 4.0 Gb, disk: 100.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", "resources":{"vcpu":1.0,"memoryGb":4.0,"diskGb":100.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local","architecture":"x86_64"}, "realResources":{"vcpu":1.0,"memoryGb":4.0,"diskGb":100.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local","architecture":"x86_64"}, "environment": "DOCKER_CONTAINER", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-with-hostnames.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-with-hostnames.json index 50007fd6610..8232e6c7085 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-with-hostnames.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-with-hostnames.json @@ -5,7 +5,7 @@ "type": "tenant", "hostname": "host4.yahoo.com", "parentHostname": "dockerhost1.yahoo.com", - "flavor": "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", + "flavor": "[vcpu: 1.0, memory: 4.0 Gb, disk: 100.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", "resources":{"vcpu":1.0,"memoryGb":4.0,"diskGb":100.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local","architecture":"x86_64"}, "realResources":{"vcpu":1.0,"memoryGb":4.0,"diskGb":100.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local","architecture":"x86_64"}, "environment": "DOCKER_CONTAINER", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4.json index f206adf4366..5f25bac926b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4.json @@ -5,7 +5,7 @@ "type": "tenant", "hostname": "host4.yahoo.com", "parentHostname": "dockerhost1.yahoo.com", - "flavor": "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", + "flavor": "[vcpu: 1.0, memory: 4.0 Gb, disk: 100.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", "resources":{"vcpu":1.0,"memoryGb":4.0,"diskGb":100.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local","architecture":"x86_64"}, "realResources":{"vcpu":1.0,"memoryGb":4.0,"diskGb":100.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local","architecture":"x86_64"}, "environment": "DOCKER_CONTAINER", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5-after-changes.json index 77dd81c736a..18829647a2a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5-after-changes.json @@ -5,7 +5,7 @@ "type": "tenant", "hostname": "host5.yahoo.com", "parentHostname": "dockerhost2.yahoo.com", - "flavor": "[vcpu: 1.0, memory: 8.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, disk speed: slow, storage type: remote, architecture: x86_64]", + "flavor": "[vcpu: 1.0, memory: 8.0 Gb, disk: 100.0 Gb, bandwidth: 1.0 Gbps, disk speed: slow, storage type: remote, architecture: x86_64]", "resources":{"vcpu":1.0,"memoryGb":8.0,"diskGb":100.0,"bandwidthGbps":1.0,"diskSpeed":"slow","storageType":"remote","architecture":"x86_64"}, "realResources":{"vcpu":1.0,"memoryGb":8.0,"diskGb":100.0,"bandwidthGbps":1.0,"diskSpeed":"slow","storageType":"remote","architecture":"x86_64"}, "environment": "DOCKER_CONTAINER", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5.json index 8a397fe5faa..a2633c5d71c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5.json @@ -5,7 +5,7 @@ "type": "tenant", "hostname": "host5.yahoo.com", "parentHostname": "dockerhost2.yahoo.com", - "flavor": "[vcpu: 1.0, memory: 8.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, disk speed: slow, storage type: remote, architecture: x86_64]", + "flavor": "[vcpu: 1.0, memory: 8.0 Gb, disk: 100.0 Gb, bandwidth: 1.0 Gbps, disk speed: slow, storage type: remote, architecture: x86_64]", "resources":{"vcpu":1.0,"memoryGb":8.0,"diskGb":100.0,"bandwidthGbps":1.0,"diskSpeed":"slow","storageType":"remote","architecture":"x86_64"}, "realResources":{"vcpu":1.0,"memoryGb":8.0,"diskGb":100.0,"bandwidthGbps":1.0,"diskSpeed":"slow","storageType":"remote","architecture":"x86_64"}, "environment": "DOCKER_CONTAINER", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json index 2bbe4a3024e..be0efad7f82 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json @@ -4,7 +4,7 @@ "state": "dirty", "type": "tenant", "hostname": "host55.yahoo.com", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", + "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk: 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local","architecture":"x86_64"}, "realResources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local","architecture":"x86_64"}, "environment": "DOCKER_CONTAINER", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6.json index 69316b1ca7f..b16b9dfe3cd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6.json @@ -4,7 +4,7 @@ "state": "active", "type": "tenant", "hostname": "host6.yahoo.com", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", + "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk: 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local","architecture":"x86_64"}, "realResources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local","architecture":"x86_64"}, "environment": "DOCKER_CONTAINER", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node7.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node7.json index 19fa81b82e0..806e1ddc705 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node7.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node7.json @@ -4,7 +4,7 @@ "state": "provisioned", "type": "tenant", "hostname": "host7.yahoo.com", - "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", + "flavor": "[vcpu: 2.0, memory: 8.0 Gb, disk: 50.0 Gb, bandwidth: 1.0 Gbps, storage type: local, architecture: x86_64]", "resources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local","architecture":"x86_64"}, "realResources":{"vcpu":2.0,"memoryGb":8.0,"diskGb":50.0,"bandwidthGbps":1.0,"diskSpeed":"fast","storageType":"local","architecture":"x86_64"}, "environment": "DOCKER_CONTAINER", diff --git a/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp b/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp index c0c6f729509..9364be4570e 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp @@ -16,6 +16,7 @@ #include <vespa/searchcommon/common/undefinedvalues.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/exceptions.h> +#include <cassert> using document::FieldValue; using document::BoolFieldValue; diff --git a/searchcore/src/vespa/searchcore/proton/matching/partial_result.h b/searchcore/src/vespa/searchcore/proton/matching/partial_result.h index f4dc2e31d4d..314fefa3cc0 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/partial_result.h +++ b/searchcore/src/vespa/searchcore/proton/matching/partial_result.h @@ -48,7 +48,7 @@ public: _sortData.push_back(sd); _sortDataSize += sd.second; } - virtual void merge(Source &rhs) override; + void merge(Source &rhs) override; }; } diff --git a/searchcore/src/vespa/searchcorespi/index/iindexmaintaineroperations.h b/searchcore/src/vespa/searchcorespi/index/iindexmaintaineroperations.h index 9025b56dc27..0b089daff7c 100644 --- a/searchcore/src/vespa/searchcorespi/index/iindexmaintaineroperations.h +++ b/searchcore/src/vespa/searchcorespi/index/iindexmaintaineroperations.h @@ -19,7 +19,7 @@ struct IIndexMaintainerOperations { using IFieldLengthInspector = search::index::IFieldLengthInspector; using Schema = search::index::Schema; using SelectorArray = search::diskindex::SelectorArray; - virtual ~IIndexMaintainerOperations() {} + virtual ~IIndexMaintainerOperations() = default; /** * Creates a new memory index using the given schema. diff --git a/searchlib/src/apps/uniform/uniform.cpp b/searchlib/src/apps/uniform/uniform.cpp index 807b8d61a9e..95d2bb1a7d1 100644 --- a/searchlib/src/apps/uniform/uniform.cpp +++ b/searchlib/src/apps/uniform/uniform.cpp @@ -3,6 +3,7 @@ #include <vespa/vespalib/util/signalhandler.h> #include <vespa/searchlib/bitcompression/compression.h> #include <cinttypes> +#include <cassert> static uint64_t maxExpGolombVal(uint64_t kValue, uint64_t maxBits) diff --git a/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp b/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp index 54a7d6ea286..4ac4c92f658 100644 --- a/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp +++ b/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp @@ -27,6 +27,26 @@ struct State { State::State() : term(), md(), f3(nullptr), f5(nullptr), f7(nullptr), array() {} State::~State() = default; +/** + * convenience adapter for easy iteration + **/ +class SimpleTermFieldRangeAdapter +{ + SimpleTermData& _ref; + size_t _idx; + size_t _lim; +public: + explicit SimpleTermFieldRangeAdapter(SimpleTermData& ref) + : _ref(ref), _idx(0), _lim(ref.numFields()) + {} + + [[nodiscard]] bool valid() const { return (_idx < _lim); } + + [[nodiscard]] SimpleTermFieldData& get() const { return _ref.field(_idx); } + + void next() { assert(valid()); ++_idx; } +}; + void testInvalidId() { const TermFieldMatchData empty; using search::queryeval::SearchIterator; @@ -44,7 +64,7 @@ void testSetup(State &state) { state.term.addField(5); // docfreq = 3 using FRA = search::fef::ITermFieldRangeAdapter; - using SFR = search::fef::SimpleTermFieldRangeAdapter; + using SFR = SimpleTermFieldRangeAdapter; // lookup terms { diff --git a/searchlib/src/vespa/searchcommon/attribute/config.cpp b/searchlib/src/vespa/searchcommon/attribute/config.cpp index 70c2377289f..91495025dee 100644 --- a/searchlib/src/vespa/searchcommon/attribute/config.cpp +++ b/searchlib/src/vespa/searchcommon/attribute/config.cpp @@ -65,4 +65,11 @@ Config::operator==(const Config &b) const _hnsw_index_params == b._hnsw_index_params; } +Config& +Config::set_hnsw_index_params(const HnswIndexParams& params) { + assert(_distance_metric == params.distance_metric()); + _hnsw_index_params = params; + return *this; +} + } diff --git a/searchlib/src/vespa/searchcommon/attribute/config.h b/searchlib/src/vespa/searchcommon/attribute/config.h index 0102f362532..32cac7ec9d6 100644 --- a/searchlib/src/vespa/searchcommon/attribute/config.h +++ b/searchlib/src/vespa/searchcommon/attribute/config.h @@ -10,7 +10,6 @@ #include <vespa/searchcommon/common/dictionary_config.h> #include <vespa/eval/eval/value_type.h> #include <vespa/vespalib/datastore/compaction_strategy.h> -#include <cassert> #include <optional> namespace search::attribute { @@ -72,11 +71,7 @@ public: _distance_metric = value; return *this; } - Config& set_hnsw_index_params(const HnswIndexParams& params) { - assert(_distance_metric == params.distance_metric()); - _hnsw_index_params = params; - return *this; - } + Config& set_hnsw_index_params(const HnswIndexParams& params); Config& clear_hnsw_index_params() { _hnsw_index_params.reset(); return *this; diff --git a/searchlib/src/vespa/searchlib/attribute/load_utils.hpp b/searchlib/src/vespa/searchlib/attribute/load_utils.hpp index 463a62ab01a..614e327942a 100644 --- a/searchlib/src/vespa/searchlib/attribute/load_utils.hpp +++ b/searchlib/src/vespa/searchlib/attribute/load_utils.hpp @@ -5,6 +5,7 @@ #include "load_utils.h" #include "attributevector.h" #include <vespa/searchcommon/attribute/multivalue.h> +#include <cassert> namespace search::attribute { diff --git a/searchlib/src/vespa/searchlib/attribute/readerbase.cpp b/searchlib/src/vespa/searchlib/attribute/readerbase.cpp index e4bc2c02ad6..382d9ccb110 100644 --- a/searchlib/src/vespa/searchlib/attribute/readerbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/readerbase.cpp @@ -6,6 +6,7 @@ #include <vespa/fastlib/io/bufferedfile.h> #include <vespa/searchlib/util/filesizecalculator.h> #include <vespa/vespalib/util/size_literals.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".search.attribute.readerbase"); @@ -73,6 +74,13 @@ ReaderBase::ReaderBase(AttributeVector &attr) ReaderBase::~ReaderBase() = default; +size_t +ReaderBase::getEnumCount() const { + size_t dataSize = _datFile.data_size(); + assert((dataSize % sizeof(uint32_t)) == 0); + return dataSize / sizeof(uint32_t); +} + bool ReaderBase::hasWeight() const { return _weightFile.valid(); diff --git a/searchlib/src/vespa/searchlib/attribute/readerbase.h b/searchlib/src/vespa/searchlib/attribute/readerbase.h index 070dc1f99fb..ff400acc824 100644 --- a/searchlib/src/vespa/searchlib/attribute/readerbase.h +++ b/searchlib/src/vespa/searchlib/attribute/readerbase.h @@ -4,7 +4,6 @@ #include <vespa/searchlib/util/file_with_header.h> #include <vespa/searchlib/util/fileutil.h> -#include <cassert> namespace search { @@ -25,11 +24,7 @@ public: return (_idxFile.data_size()) /sizeof(uint32_t); } - size_t getEnumCount() const { - size_t dataSize = _datFile.data_size(); - assert((dataSize % sizeof(uint32_t)) == 0); - return dataSize / sizeof(uint32_t); - } + size_t getEnumCount() const; size_t getNumValues(); int32_t getNextWeight() { return _weightReader.readHostOrder(); } diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.h b/searchlib/src/vespa/searchlib/bitcompression/compression.h index a77d82d9e8f..2d6b8083d43 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/compression.h +++ b/searchlib/src/vespa/searchlib/bitcompression/compression.h @@ -4,7 +4,6 @@ #include <vespa/searchlib/util/comprfile.h> #include <vespa/vespalib/stllike/string.h> -#include <cassert> namespace vespalib { @@ -1400,7 +1399,6 @@ public: const uint8_t * getByteCompr() const { - assert((_preRead & 7) == 0); return reinterpret_cast<const uint8_t *>(getCompr()) + (getBitOffset() >> 3); } diff --git a/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp b/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp index cdae8058d76..7c38931df77 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp @@ -2,6 +2,7 @@ #include "countcompression.h" #include <vespa/searchlib/index/postinglistcounts.h> +#include <cassert> namespace search::bitcompression { diff --git a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.h b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.h index 9208a5be3b8..b162bdc3f2b 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.h +++ b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.h @@ -3,8 +3,8 @@ #pragma once #include "countcompression.h" -#include <limits> #include <vespa/vespalib/stllike/string.h> +#include <cassert> namespace search::bitcompression { diff --git a/searchlib/src/vespa/searchlib/bitcompression/posocc_fields_params.cpp b/searchlib/src/vespa/searchlib/bitcompression/posocc_fields_params.cpp index d2269787068..9d6258ce26f 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/posocc_fields_params.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/posocc_fields_params.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/index/schemautil.h> #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".posocc_fields_params"); @@ -38,6 +39,12 @@ PosOccFieldsParams::operator=(const PosOccFieldsParams &rhs) return *this; } +void +PosOccFieldsParams::assertCachedParamsRef() const { + assert(_numFields == _params.size()); + assert(_fieldParams == (_params.empty() ? nullptr : &_params[0])); +} + bool PosOccFieldsParams::operator==(const PosOccFieldsParams &rhs) const diff --git a/searchlib/src/vespa/searchlib/bitcompression/posocc_fields_params.h b/searchlib/src/vespa/searchlib/bitcompression/posocc_fields_params.h index 963a80f06dc..8748557e5a7 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/posocc_fields_params.h +++ b/searchlib/src/vespa/searchlib/bitcompression/posocc_fields_params.h @@ -4,7 +4,6 @@ #include "posocc_field_params.h" #include <vector> -#include <cassert> namespace search::bitcompression { @@ -32,10 +31,7 @@ public: _fieldParams = _params.empty() ? nullptr : &_params[0]; } - void assertCachedParamsRef() const { - assert(_numFields == _params.size()); - assert(_fieldParams == (_params.empty() ? nullptr : &_params[0])); - } + void assertCachedParamsRef() const; uint32_t getNumFields() const { return _numFields; } const PosOccFieldParams *getFieldParams() const { return _fieldParams; } diff --git a/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp b/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp index fd6c723e901..8e1bfd2875c 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp @@ -7,6 +7,7 @@ #include <vespa/searchlib/index/postinglistparams.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/data/fileheader.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".posocccompression"); diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldreader.h b/searchlib/src/vespa/searchlib/diskindex/fieldreader.h index 73b56559115..e97d5deb95c 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldreader.h +++ b/searchlib/src/vespa/searchlib/diskindex/fieldreader.h @@ -1,14 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "wordnummapper.h" +#include "docidmapper.h" +#include "fieldwriter.h" #include <vespa/searchlib/index/postinglistcounts.h> #include <vespa/searchlib/index/dictionaryfile.h> #include <vespa/searchlib/index/docidandfeatures.h> #include <vespa/searchlib/index/postinglistfile.h> #include <vespa/searchlib/index/schemautil.h> -#include "wordnummapper.h" -#include "docidmapper.h" -#include "fieldwriter.h" namespace search::diskindex { @@ -40,7 +40,7 @@ public: using PostingListCounts = index::PostingListCounts; using PostingListParams = index::PostingListParams; - uint64_t _wordNum; + uint64_t _wordNum; DocIdAndFeatures _docIdAndFeatures; protected: std::unique_ptr<DictionaryFileSeqRead> _dictFile; diff --git a/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp b/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp index 5399d70fbe7..432651278e0 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/data/fileheader.h> #include <vespa/fastos/file.h> #include <cinttypes> +#include <cassert> #include <arpa/inet.h> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp index 4fd9d116244..d12081ee89c 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp @@ -8,11 +8,9 @@ #include <vespa/searchlib/common/documentsummary.h> #include <vespa/searchlib/common/i_flush_token.h> #include <vespa/searchlib/index/schemautil.h> -#include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/error.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/lambdatask.h> -#include <vespa/document/util/queue.h> #include <filesystem> #include <system_error> diff --git a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_reader.cpp b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_reader.cpp index 2170777dbd3..460fac36acc 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_reader.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_reader.cpp @@ -3,6 +3,7 @@ #include "zc4_posting_reader.h" #include "zc4_posting_header.h" #include <vespa/searchlib/index/docidandfeatures.h> +#include <cassert> namespace search::diskindex { @@ -19,9 +20,7 @@ Zc4PostingReader<bigEndian>::Zc4PostingReader(bool dynamic_k) } template <bool bigEndian> -Zc4PostingReader<bigEndian>::~Zc4PostingReader() -{ -} +Zc4PostingReader<bigEndian>::~Zc4PostingReader() = default; template <bool bigEndian> void diff --git a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_reader_base.cpp b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_reader_base.cpp index b50835a648d..c71404d449b 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_reader_base.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_reader_base.cpp @@ -3,7 +3,7 @@ #include "zc4_posting_reader_base.h" #include "zc4_posting_header.h" #include <vespa/searchlib/index/docidandfeatures.h> - +#include <cassert> namespace search::diskindex { using index::PostingListCounts; diff --git a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer.cpp b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer.cpp index 202ed5a23cd..3a1b7928c93 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer.cpp @@ -3,6 +3,7 @@ #include "zc4_posting_writer.h" #include <vespa/searchlib/index/docidandfeatures.h> #include <vespa/searchlib/index/postinglistcounts.h> +#include <cassert> using search::index::DocIdAndFeatures; using search::index::PostingListCounts; diff --git a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.cpp b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.cpp index 3f44b56706a..8a84ccc5731 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.cpp @@ -3,6 +3,7 @@ #include "zc4_posting_writer_base.h" #include <vespa/searchlib/index/postinglistcounts.h> #include <vespa/searchlib/index/postinglistparams.h> +#include <cassert> using search::index::PostingListCounts; using search::index::PostingListParams; diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp index b67a8409581..df33091a4e8 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp @@ -1,9 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "zcposocciterators.h" +#include "zc4_posting_params.h" #include <vespa/searchlib/bitcompression/posocc_fields_params.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> -#include "zc4_posting_params.h" +#include <cassert> namespace search::diskindex { diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp index ef540365208..c9a1563a8e3 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp @@ -8,6 +8,7 @@ #include <vespa/searchlib/index/postinglistparams.h> #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/vespalib/data/fileheader.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".diskindex.zcposting"); diff --git a/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.cpp b/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.cpp index 66404c7a0ff..83a4ae20db5 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.cpp @@ -4,6 +4,7 @@ #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/searchlib/bitcompression/posocccompression.h> +#include <cassert> namespace search::diskindex { diff --git a/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp b/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp index bd13c032a03..16f5ee04be4 100644 --- a/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp @@ -6,6 +6,7 @@ #include <vespa/document/datatype/documenttype.h> #include <vespa/vespalib/encoding/base64.h> #include <vespa/vespalib/locale/c.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".searchlib.documentfieldnode"); diff --git a/searchlib/src/vespa/searchlib/fef/objectstore.h b/searchlib/src/vespa/searchlib/fef/objectstore.h index 06575c61eb5..7ba08284111 100644 --- a/searchlib/src/vespa/searchlib/fef/objectstore.h +++ b/searchlib/src/vespa/searchlib/fef/objectstore.h @@ -2,7 +2,6 @@ #pragma once #include <vespa/vespalib/stllike/hash_map.h> -#include <cassert> namespace search::fef { @@ -66,7 +65,6 @@ const T & as_value(const Anything &val) { using WrapperType = AnyWrapper<T>; const auto *wrapper = dynamic_cast<const WrapperType *>(&val); - assert(wrapper != nullptr); return wrapper->getValue(); } diff --git a/searchlib/src/vespa/searchlib/fef/phrase_splitter_query_env.cpp b/searchlib/src/vespa/searchlib/fef/phrase_splitter_query_env.cpp index dfa1d9886f7..bbbdbd69c67 100644 --- a/searchlib/src/vespa/searchlib/fef/phrase_splitter_query_env.cpp +++ b/searchlib/src/vespa/searchlib/fef/phrase_splitter_query_env.cpp @@ -43,7 +43,6 @@ PhraseSplitterQueryEnv::PhraseSplitterQueryEnv(const IQueryEnvironment & queryEn TermFieldHandle numHandles = 0; // how many handles existed in underlying data for (uint32_t i = 0; i < queryEnv.getNumTerms(); ++i) { const ITermData *td = queryEnv.getTerm(i); - assert(td != nullptr); considerTerm(i, *td, fieldId); numHandles += td->numFields(); } diff --git a/searchlib/src/vespa/searchlib/fef/phrasesplitter.cpp b/searchlib/src/vespa/searchlib/fef/phrasesplitter.cpp index 90a058eda00..b74f12bdb97 100644 --- a/searchlib/src/vespa/searchlib/fef/phrasesplitter.cpp +++ b/searchlib/src/vespa/searchlib/fef/phrasesplitter.cpp @@ -42,7 +42,6 @@ PhraseSplitter::update() for (const auto ©_info : _phrase_splitter_query_env.get_copy_info()) { const TermFieldMatchData *src = _matchData->resolveTermField(copy_info.orig_handle); TermFieldMatchData *dst = resolveSplittedTermField(copy_info.split_handle); - assert(src != nullptr && dst != nullptr); copyTermFieldMatchData(*dst, *src, copy_info.offsetInPhrase); } diff --git a/searchlib/src/vespa/searchlib/fef/simpletermdata.h b/searchlib/src/vespa/searchlib/fef/simpletermdata.h index d501d0848e8..391a00e4c8a 100644 --- a/searchlib/src/vespa/searchlib/fef/simpletermdata.h +++ b/searchlib/src/vespa/searchlib/fef/simpletermdata.h @@ -7,7 +7,6 @@ #include "simpletermfielddata.h" #include <vespa/searchlib/query/weight.h> #include <vector> -#include <cassert> namespace search::fef { @@ -128,24 +127,4 @@ public: } }; -/** - * convenience adapter for easy iteration - **/ -class SimpleTermFieldRangeAdapter -{ - SimpleTermData& _ref; - size_t _idx; - size_t _lim; -public: - explicit SimpleTermFieldRangeAdapter(SimpleTermData& ref) - : _ref(ref), _idx(0), _lim(ref.numFields()) - {} - - [[nodiscard]] bool valid() const { return (_idx < _lim); } - - [[nodiscard]] SimpleTermFieldData& get() const { return _ref.field(_idx); } - - void next() { assert(valid()); ++_idx; } -}; - } diff --git a/searchlib/src/vespa/searchlib/fef/termfieldmatchdataarray.h b/searchlib/src/vespa/searchlib/fef/termfieldmatchdataarray.h index 3c1b76ad40e..46d370ee8fe 100644 --- a/searchlib/src/vespa/searchlib/fef/termfieldmatchdataarray.h +++ b/searchlib/src/vespa/searchlib/fef/termfieldmatchdataarray.h @@ -3,7 +3,6 @@ #pragma once #include <vector> -#include <cassert> #include <cstddef> namespace search::fef { @@ -43,7 +42,6 @@ public: * @param value the pointer to be added **/ TermFieldMatchDataArray &add(TermFieldMatchData *value) { - assert(value != nullptr); _array.push_back(value); return *this; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.cpp index c79d856676d..c1537c6b290 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.cpp @@ -10,6 +10,7 @@ #include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/util/stringfmt.h> #include <stdexcept> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".memoryindex.url_field_inverter"); diff --git a/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp b/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp index 00f17f7963c..51882e6d185 100644 --- a/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp @@ -3,6 +3,7 @@ #include "termnodes.h" #include <vespa/vespalib/util/exceptions.h> #include <charconv> +#include <cassert> using vespalib::IllegalArgumentException; using vespalib::stringref; diff --git a/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp b/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp index 94814dbb9b3..d2aa72011e6 100644 --- a/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp @@ -5,9 +5,20 @@ #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/vespalib/objects/visit.h> #include <vespa/searchcommon/attribute/i_search_context.h> +#include <cassert> namespace search::queryeval { +FakeSearch::FakeSearch(const vespalib::string &tag, const vespalib::string &field, + const vespalib::string &term, const FakeResult &res, + fef::TermFieldMatchDataArray tfmda) + : _tag(tag), _field(field), _term(term), + _result(res), _offset(0), _tfmda(std::move(tfmda)), + _ctx(nullptr) +{ + assert(_tfmda.size() == 1); +} + void FakeSearch::doSeek(uint32_t docid) { diff --git a/searchlib/src/vespa/searchlib/queryeval/fake_search.h b/searchlib/src/vespa/searchlib/queryeval/fake_search.h index 5cd04f80499..7b7fdf0f078 100644 --- a/searchlib/src/vespa/searchlib/queryeval/fake_search.h +++ b/searchlib/src/vespa/searchlib/queryeval/fake_search.h @@ -29,13 +29,7 @@ public: const vespalib::string &field, const vespalib::string &term, const FakeResult &res, - fef::TermFieldMatchDataArray tfmda) - : _tag(tag), _field(field), _term(term), - _result(res), _offset(0), _tfmda(std::move(tfmda)), - _ctx(nullptr) - { - assert(_tfmda.size() == 1); - } + fef::TermFieldMatchDataArray tfmda); void attr_ctx(const attribute::ISearchContext *ctx) { _ctx = ctx; } bool is_attr() const { return (_ctx != nullptr); } void doSeek(uint32_t docid) override; diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_search.cpp b/searchlib/src/vespa/searchlib/queryeval/same_element_search.cpp index 98c51d7f1ca..5db1e0057cd 100644 --- a/searchlib/src/vespa/searchlib/queryeval/same_element_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_search.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/objects/visit.hpp> #include <algorithm> #include <functional> +#include <cassert> using TFMD = search::fef::TermFieldMatchData; diff --git a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.cpp b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.cpp index f5069fd4f53..ea264935d42 100644 --- a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.cpp @@ -4,6 +4,7 @@ #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/vespalib/objects/visit.h> #include <functional> +#include <cassert> using search::fef::TermFieldMatchData; using std::unique_ptr; diff --git a/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.cpp b/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.cpp index bc92b691ce8..04c8b6b3904 100644 --- a/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.cpp +++ b/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "threelevelcountbuffers.h" +#include <cassert> namespace search::diskindex { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp index db4ac0f32b0..f1fee7e9ac3 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp @@ -14,6 +14,7 @@ #include <vespa/searchlib/parsequery/stackdumpiterator.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/issue.h> +#include <cassert> using search::common::GeoLocationParser; using search::common::GeoLocationSpec; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp index 6d668561651..07e4bde54d0 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp @@ -10,6 +10,7 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/issue.h> #include <climits> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docsummary.geoposdfw"); diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp index 596525e17d7..146dd487769 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp @@ -11,6 +11,7 @@ #include <vespa/document/datatype/datatype.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/vespalib/data/slime/inserter.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".vsm.docsumfilter"); diff --git a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/EnabledInRegions.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/EnabledInRegions.java index db2e5ac5f95..9ccd0588d6d 100644 --- a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/EnabledInRegions.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/EnabledInRegions.java @@ -35,7 +35,7 @@ class EnabledInRegionsCondition implements ExecutionCondition { return ConditionEvaluationResult.enabled(EnabledInRegions.class.getSimpleName() + " is not present"); List<String> enablingRegions = List.of(annotation.get().value()); - String thisRegion = TestRuntime.get().application().instance(); + String thisRegion = TestRuntime.get().zone().region(); String reason = "Enabled in: %s. Current region: %s.".formatted(enablingRegions.isEmpty() ? "no regions" : "regions " + String.join(", ", enablingRegions), thisRegion); return enablingRegions.contains(thisRegion) ? ConditionEvaluationResult.enabled(reason) : ConditionEvaluationResult.disabled(reason); } diff --git a/vespalib/src/tests/btree/btree_test.cpp b/vespalib/src/tests/btree/btree_test.cpp index afad3523fa3..6305eca41ac 100644 --- a/vespalib/src/tests/btree/btree_test.cpp +++ b/vespalib/src/tests/btree/btree_test.cpp @@ -2,7 +2,6 @@ #include <string> #include <vespa/vespalib/btree/btreeroot.h> -#include <vespa/vespalib/btree/btreebuilder.h> #include <vespa/vespalib/btree/btreenodeallocator.h> #include <vespa/vespalib/btree/btree.h> #include <vespa/vespalib/btree/btreestore.h> @@ -297,6 +296,12 @@ BTreeTest::assertMemoryUsage(const vespalib::MemoryUsage & exp, const vespalib:: return result; } +TEST_F(BTreeTest, control_iterator_size) { + EXPECT_EQ(208u, sizeof(BTreeIteratorBase<uint32_t, uint32_t, NoAggregated>)); + EXPECT_EQ(208u, sizeof(BTreeIteratorBase<uint32_t, BTreeNoLeafData, NoAggregated>)); + EXPECT_EQ(544u, sizeof(MyTree::Iterator)); +} + TEST_F(BTreeTest, require_that_node_insert_works) { GenerationHandler g; diff --git a/vespalib/src/vespa/vespalib/btree/btree.h b/vespalib/src/vespa/vespalib/btree/btree.h index c2f5aac01b7..0099da718a3 100644 --- a/vespalib/src/vespa/vespalib/btree/btree.h +++ b/vespalib/src/vespa/vespalib/btree/btree.h @@ -39,50 +39,22 @@ public: using ConstIterator = typename TreeType::ConstIterator; using FrozenView = typename TreeType::FrozenView; using AggrCalcType = typename TreeType::AggrCalcType; -private: - NodeAllocatorType _alloc; - TreeType _tree; - - BTree(const BTree &rhs); - BTree & - operator=(BTree &rhs); - -public: + BTree(const BTree &rhs) = delete; + BTree & operator=(BTree &rhs) = delete; BTree(); ~BTree(); const NodeAllocatorType &getAllocator() const { return _alloc; } NodeAllocatorType &getAllocator() { return _alloc; } - - void - disableFreeLists() { - _alloc.disableFreeLists(); - } - - void - disable_entry_hold_list() - { - _alloc.disable_entry_hold_list(); - } - - // Inherit doc from BTreeRoot - void clear() { - _tree.clear(_alloc); - } - void assign(Builder & rhs) { - _tree.assign(rhs, _alloc); - } + void disableFreeLists() { _alloc.disableFreeLists(); } + void disable_entry_hold_list() { _alloc.disable_entry_hold_list(); } + void clear() { _tree.clear(_alloc); } + void assign(Builder & rhs) { _tree.assign(rhs, _alloc); } bool insert(const KeyType & key, const DataType & data, CompareT comp = CompareT()) { return _tree.insert(key, data, _alloc, comp); } - - void - insert(Iterator &itr, - const KeyType &key, const DataType &data) - { - _tree.insert(itr, key, data); - } + void insert(Iterator &itr, const KeyType &key, const DataType &data) { _tree.insert(itr, key, data); } Iterator find(const KeyType & key, CompareT comp = CompareT()) const { return _tree.find(key, _alloc, comp); @@ -97,55 +69,23 @@ public: return _tree.remove(key, _alloc, comp); } - void - remove(Iterator &itr) - { - _tree.remove(itr); - } - - Iterator begin() const { - return _tree.begin(_alloc); - } - FrozenView getFrozenView() const { - return _tree.getFrozenView(_alloc); - } - size_t size() const { - return _tree.size(_alloc); - } - vespalib::string toString() const { - return _tree.toString(_alloc); - } - bool isValid(CompareT comp = CompareT()) const { - return _tree.isValid(_alloc, comp); - } - bool isValidFrozen(CompareT comp = CompareT()) const { - return _tree.isValidFrozen(_alloc, comp); - } - size_t bitSize() const { - return _tree.bitSize(_alloc); - } + void remove(Iterator &itr) { _tree.remove(itr); } + Iterator begin() const { return _tree.begin(_alloc); } + FrozenView getFrozenView() const { return _tree.getFrozenView(_alloc); } + size_t size() const { return _tree.size(_alloc); } + vespalib::string toString() const { return _tree.toString(_alloc); } + bool isValid(CompareT comp = CompareT()) const { return _tree.isValid(_alloc, comp); } + bool isValidFrozen(CompareT comp = CompareT()) const { return _tree.isValidFrozen(_alloc, comp); } + size_t bitSize() const { return _tree.bitSize(_alloc); } size_t bitSize(BTreeNode::Ref node) const { return _tree.bitSize(node, _alloc); } - void setRoot(BTreeNode::Ref newRoot) { - _tree.setRoot(newRoot, _alloc); - } - BTreeNode::Ref getRoot() const { - return _tree.getRoot(); - } - vespalib::MemoryUsage getMemoryUsage() const { - return _alloc.getMemoryUsage(); - } - - const AggrT & - getAggregated() const - { - return _tree.getAggregated(_alloc); - } + void setRoot(BTreeNode::Ref newRoot) { _tree.setRoot(newRoot, _alloc); } + BTreeNode::Ref getRoot() const { return _tree.getRoot(); } + vespalib::MemoryUsage getMemoryUsage() const { return _alloc.getMemoryUsage(); } + const AggrT & getAggregated() const { return _tree.getAggregated(_alloc); } - void - thaw(Iterator &itr) - { + void thaw(Iterator &itr) { assert(&itr.getAllocator() == &getAllocator()); _tree.thaw(itr); } @@ -153,18 +93,17 @@ public: void compact_worst(const datastore::CompactionStrategy& compaction_strategy); template <typename FunctionType> - void - foreach_key(FunctionType func) const - { + void foreach_key(FunctionType func) const { _alloc.getNodeStore().foreach_key(_tree.getRoot(), func); } template <typename FunctionType> - void - foreach(FunctionType func) const - { + void foreach(FunctionType func) const { _alloc.getNodeStore().foreach(_tree.getRoot(), func); } +private: + NodeAllocatorType _alloc; + TreeType _tree; }; } diff --git a/vespalib/src/vespa/vespalib/btree/btreeiterator.h b/vespalib/src/vespa/vespalib/btree/btreeiterator.h index 4b99edf592a..7a754880aa3 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeiterator.h +++ b/vespalib/src/vespa/vespalib/btree/btreeiterator.h @@ -36,115 +36,46 @@ class NodeElement using KeyType = typename NodeType::KeyType; using DataType = typename NodeType::DataType; const NodeType *_node; - uint32_t _idx; - - NodeType * - getWNode() const - { - return const_cast<NodeType *>(_node); - } + uint32_t _idx; + NodeType * getWNode() const { return const_cast<NodeType *>(_node); } public: - NodeElement() - : _node(nullptr), - _idx(0u) - { - } + NodeElement() : _node(nullptr), _idx(0u) { } + NodeElement(const NodeType *node, uint32_t idx) : _node(node), _idx(idx) { } - NodeElement(const NodeType *node, uint32_t idx) - : _node(node), - _idx(idx) - { - } + void setNode(const NodeType *node) { _node = node; } + const NodeType * getNode() const { return _node; } + void setIdx(uint32_t idx) { _idx = idx; } + uint32_t getIdx() const { return _idx; } + void incIdx() { ++_idx; } + void decIdx() { --_idx; } - void - setNode(const NodeType *node) - { + void setNodeAndIdx(const NodeType *node, uint32_t idx) { _node = node; - } - - const NodeType * - getNode() const - { - return _node; - } - - void - setIdx(uint32_t idx) - { _idx = idx; } - uint32_t - getIdx() const - { - return _idx; - } - - void - incIdx() - { - ++_idx; - } - - void - decIdx() - { - --_idx; - } - - void - setNodeAndIdx(const NodeType *node, uint32_t idx) - { - _node = node; - _idx = idx; - } - - const KeyType & - getKey() const - { - return _node->getKey(_idx); - } - - const DataType & - getData() const - { - return _node->getData(_idx); - } - + const KeyType & getKey() const { return _node->getKey(_idx); } + const DataType & getData() const { return _node->getData(_idx); } // Only use during compaction when changing reference to moved value DataType &getWData() { return getWNode()->getWData(_idx); } - - bool - valid() const - { - return _node != nullptr; - } - - void - adjustLeftVictimKilled() - { + bool valid() const { return _node != nullptr; } + void adjustLeftVictimKilled() { assert(_idx > 0); --_idx; } - void - adjustSteal(uint32_t stolen) - { + void adjustSteal(uint32_t stolen) { assert(_idx + stolen < _node->validSlots()); _idx += stolen; } - void - adjustSplit(bool inRightSplit) - { + void adjustSplit(bool inRightSplit) { if (inRightSplit) ++_idx; } - bool - adjustSplit(bool inRightSplit, const NodeType *splitNode) - { + bool adjustSplit(bool inRightSplit, const NodeType *splitNode) { adjustSplit(inRightSplit); if (_idx >= _node->validSlots()) { _idx -= _node->validSlots(); @@ -154,18 +85,13 @@ public: return false; } - void - swap(NodeElement &rhs) - { + void swap(NodeElement &rhs) { std::swap(_node, rhs._node); std::swap(_idx, rhs._idx); } - bool - operator!=(const NodeElement &rhs) const - { - return _node != rhs._node || - _idx != rhs._idx; + bool operator!=(const NodeElement &rhs) const { + return (_node != rhs._node) || (_idx != rhs._idx); } }; @@ -243,8 +169,7 @@ protected: * * @param pidx Number of levels above leaf nodes to take into account. */ - size_t - position(uint32_t pidx) const; + size_t position(uint32_t pidx) const; /** * Create iterator pointing to first element in the tree referenced @@ -273,8 +198,7 @@ protected: /** * Step iterator forwards. If at end then leave it at end. */ - BTreeIteratorBase & - operator++() { + BTreeIteratorBase & operator++() { if (_leaf.getNode() == nullptr) { return *this; } @@ -290,8 +214,7 @@ protected: * Step iterator backwards. If at end then place it at last valid * position in tree (cf. rbegin()) */ - BTreeIteratorBase & - operator--(); + BTreeIteratorBase & operator--(); ~BTreeIteratorBase(); BTreeIteratorBase(const BTreeIteratorBase &other); @@ -311,9 +234,7 @@ protected: * from this iterator position to end of subtree. */ template <typename FunctionType> - void - foreach_key_range_start(uint32_t level, FunctionType func) const - { + void foreach_key_range_start(uint32_t level, FunctionType func) const { if (level > 0u) { --level; foreach_key_range_start(level, func); @@ -332,9 +253,7 @@ protected: * subtree before this iterator position). */ template <typename FunctionType> - void - foreach_key_range_end(uint32_t level, FunctionType func) const - { + void foreach_key_range_end(uint32_t level, FunctionType func) const { if (level > 0u) { --level; auto &store = _allocator->getNodeStore(); @@ -348,8 +267,7 @@ protected: } public: - bool - operator==(const BTreeIteratorBase & rhs) const { + bool operator==(const BTreeIteratorBase & rhs) const { if (_leaf.getIdx() != rhs._leaf.getIdx()) { return false; } @@ -367,83 +285,55 @@ public: return true; } - bool - operator!=(const BTreeIteratorBase & rhs) const - { - return !operator==(rhs); - } + bool operator!=(const BTreeIteratorBase & rhs) const { return !operator==(rhs); } /** * Swap iterator with the other. * * @param rhs Other iterator. */ - void - swap(BTreeIteratorBase & rhs); + void swap(BTreeIteratorBase & rhs); /** * Get key at current iterator location. */ - const KeyType & - getKey() const - { - return _leaf.getKey(); - } + const KeyType & getKey() const { return _leaf.getKey(); } /** * Get data at current iterator location. */ - const DataType & - getData() const - { - return _leaf.getData(); - } + const DataType & getData() const { return _leaf.getData(); } /** * Check if iterator is at a valid element, i.e. not at end. */ - bool - valid() const - { - return _leaf.valid(); - } + bool valid() const { return _leaf.valid(); } /** * Return the number of elements in the tree. */ - size_t - size() const; + size_t size() const; /** * Return the current position in the tree. */ - size_t - position() const - { - return position(_pathSize); - } + size_t position() const { return position(_pathSize); } /** * Return the distance between two positions in the tree. */ - ssize_t - operator-(const BTreeIteratorBase &rhs) const; + ssize_t operator-(const BTreeIteratorBase &rhs) const; /** * Return if the tree has data or not (e.g. keys and data or only keys). */ - static bool - hasData() - { - return LeafNodeType::hasData(); - } + static bool hasData() { return LeafNodeType::hasData(); } /** * Move the iterator directly to end. Used by findHelper method in BTree. */ - void - setupEnd(); + void setupEnd(); /** * Setup iterator to be empty and not be associated with any tree. @@ -453,50 +343,41 @@ public: /** * Move iterator to beyond last element in the current tree. */ - void - end() __attribute__((noinline)); + void end() __attribute__((noinline)); /** * Move iterator to beyond last element in the given tree. * * @param rootRef Reference to root of tree. */ - void - end(BTreeNode::Ref rootRef); + void end(BTreeNode::Ref rootRef); /** * Move iterator to first element in the current tree. */ - void - begin(); + void begin(); /** * Move iterator to first element in the given tree. * * @param rootRef Reference to root of tree. */ - void - begin(BTreeNode::Ref rootRef); + void begin(BTreeNode::Ref rootRef); /** * Move iterator to last element in the current tree. */ - void - rbegin(); + void rbegin(); /* * Get aggregated values for the current tree. */ - const AggrT & - getAggregated() const; + const AggrT & getAggregated() const; - bool - identical(const BTreeIteratorBase &rhs) const; + bool identical(const BTreeIteratorBase &rhs) const; template <typename FunctionType> - void - foreach_key(FunctionType func) const - { + void foreach_key(FunctionType func) const { if (_pathSize > 0) { _path[_pathSize - 1].getNode()-> foreach_key(_allocator->getNodeStore(), func); @@ -511,9 +392,7 @@ public: * range [this iterator, end_itr)). */ template <typename FunctionType> - void - foreach_key_range(const BTreeIteratorBase &end_itr, FunctionType func) const - { + void foreach_key_range(const BTreeIteratorBase &end_itr, FunctionType func) const { if (!valid()) { return; } @@ -584,9 +463,7 @@ class BTreeConstIterator : public BTreeIteratorBase<KeyT, DataT, AggrT, TraitsT::PATH_SIZE> { protected: - using ParentType = BTreeIteratorBase<KeyT, - DataT, - AggrT, + using ParentType = BTreeIteratorBase<KeyT, DataT, AggrT, TraitsT::INTERNAL_SLOTS, TraitsT::LEAF_SLOTS, TraitsT::PATH_SIZE>; @@ -645,17 +522,12 @@ public: /** * Default constructor. Iterator is not associated with a tree. */ - BTreeConstIterator() - : ParentType() - { - } + BTreeConstIterator() : ParentType() { } /** * Step iterator forwards. If at end then leave it at end. */ - BTreeConstIterator & - operator++() - { + BTreeConstIterator & operator++() { ParentType::operator++(); return *this; } @@ -664,9 +536,7 @@ public: * Step iterator backwards. If at end then place it at last valid * position in tree (cf. rbegin()) */ - BTreeConstIterator & - operator--() - { + BTreeConstIterator & operator--() { ParentType::operator--(); return *this; } @@ -679,8 +549,7 @@ public: * @param key Key to search for * @param comp Comparator for the tree ordering. */ - void - lower_bound(const KeyType & key, CompareT comp = CompareT()); + void lower_bound(const KeyType & key, CompareT comp = CompareT()); /** * Position iterator at first position with a key that is greater @@ -689,9 +558,7 @@ public: * @param key Key to search for * @param comp Comparator for the tree ordering. */ - void - lower_bound(BTreeNode::Ref rootRef, - const KeyType & key, CompareT comp = CompareT()); + void lower_bound(BTreeNode::Ref rootRef, const KeyType & key, CompareT comp = CompareT()); /** * Step iterator forwards until it is at a position with a key @@ -704,8 +571,7 @@ public: * @param key Key to search for * @param comp Comparator for the tree ordering. */ - void - seek(const KeyType &key, CompareT comp = CompareT()); + void seek(const KeyType &key, CompareT comp = CompareT()); /** * Step iterator forwards until it is at a position with a key @@ -717,8 +583,7 @@ public: * @param key Key to search for * @param comp Comparator for the tree ordering. */ - void - binarySeek(const KeyType &key, CompareT comp = CompareT()); + void binarySeek(const KeyType &key, CompareT comp = CompareT()); /** * Step iterator forwards until it is at a position with a key @@ -730,8 +595,7 @@ public: * @param key Key to search for * @param comp Comparator for the tree ordering. */ - void - linearSeek(const KeyType &key, CompareT comp = CompareT()); + void linearSeek(const KeyType &key, CompareT comp = CompareT()); /** * Step iterator forwards until it is at a position with a key @@ -744,8 +608,7 @@ public: * @param key Key to search for * @param comp Comparator for the tree ordering. */ - void - seekPast(const KeyType &key, CompareT comp = CompareT()); + void seekPast(const KeyType &key, CompareT comp = CompareT()); /** * Step iterator forwards until it is at a position with a key @@ -757,8 +620,7 @@ public: * @param key Key to search for * @param comp Comparator for the tree ordering. */ - void - binarySeekPast(const KeyType &key, CompareT comp = CompareT()); + void binarySeekPast(const KeyType &key, CompareT comp = CompareT()); /** * Step iterator forwards until it is at a position with a key @@ -770,8 +632,7 @@ public: * @param key Key to search for * @param comp Comparator for the tree ordering. */ - void - linearSeekPast(const KeyType &key, CompareT comp = CompareT()); + void linearSeekPast(const KeyType &key, CompareT comp = CompareT()); /** * Validate the iterator as a valid iterator or positioned at @@ -781,8 +642,7 @@ public: * @param rootRef Reference to root of tree to operate on * @param comp Comparator for the tree ordering. */ - void - validate(BTreeNode::Ref rootRef, CompareT comp = CompareT()); + void validate(BTreeNode::Ref rootRef, CompareT comp = CompareT()); }; @@ -795,15 +655,10 @@ template <typename KeyT, typename AggrT = NoAggregated, typename CompareT = std::less<KeyT>, typename TraitsT = BTreeDefaultTraits> -class BTreeIterator : public BTreeConstIterator<KeyT, DataT, AggrT, - CompareT, TraitsT> +class BTreeIterator : public BTreeConstIterator<KeyT, DataT, AggrT, CompareT, TraitsT> { public: - using ParentType = BTreeConstIterator<KeyT, - DataT, - AggrT, - CompareT, - TraitsT>; + using ParentType = BTreeConstIterator<KeyT, DataT, AggrT, CompareT, TraitsT>; using NodeAllocatorType = typename ParentType::NodeAllocatorType; using InternalNodeType = typename ParentType::InternalNodeType; using LeafNodeType = typename ParentType::LeafNodeType; @@ -844,40 +699,27 @@ public: { } - BTreeIterator() - : ParentType() - { - } + BTreeIterator() : ParentType() { } - BTreeIterator & - operator++() - { + BTreeIterator & operator++() { ParentType::operator++(); return *this; } - BTreeIterator & - operator--() - { + BTreeIterator & operator--() { ParentType::operator--(); return *this; } - NodeAllocatorType & - getAllocator() const - { + NodeAllocatorType & getAllocator() const { return const_cast<NodeAllocatorType &>(*_allocator); } - BTreeNode::Ref - moveFirstLeafNode(BTreeNode::Ref rootRef); + BTreeNode::Ref moveFirstLeafNode(BTreeNode::Ref rootRef); - void - moveNextLeafNode(); + void moveNextLeafNode(); - void - writeData(const DataType &data) - { + void writeData(const DataType &data) { _leaf.getWNode()->writeData(_leaf.getIdx(), data); } @@ -889,8 +731,7 @@ public: * The new key must have the same semantic meaning as the old key. * Typically used when compacting data store containing keys. */ - void - writeKey(const KeyType &key); + void writeKey(const KeyType &key); /** * Updata data at the current iterator position. The tree should @@ -900,71 +741,33 @@ public: * @param aggrCalc Calculator for updating aggregated information. */ template <class AggrCalcT> - void - updateData(const DataType &data, const AggrCalcT &aggrCalc); + void updateData(const DataType &data, const AggrCalcT &aggrCalc); /** * Thaw a path from the root node down the the current leaf node in * the current tree, allowing for updates to be performed without * disturbing the frozen version of the tree. */ - BTreeNode::Ref - thaw(BTreeNode::Ref rootRef); + BTreeNode::Ref thaw(BTreeNode::Ref rootRef); private: /* Insert into empty tree */ template <class AggrCalcT> - BTreeNode::Ref - insertFirst(const KeyType &key, const DataType &data, - const AggrCalcT &aggrCalc); - - LeafNodeType * - getLeafNode() const - { - return _leaf.getWNode(); - } - - bool - setLeafNodeIdx(uint32_t idx, const LeafNodeType *splitLeafNode); - - void - setLeafNodeIdx(uint32_t idx) - { - _leaf.setIdx(idx); - } - - uint32_t - getLeafNodeIdx() const - { - return _leaf.getIdx(); - } - - uint32_t - getPathSize() const - { - return _pathSize; - } - - PathElement & - getPath(uint32_t pidx) - { - return _path[pidx]; - } + BTreeNode::Ref insertFirst(const KeyType &key, const DataType &data, const AggrCalcT &aggrCalc); + LeafNodeType * getLeafNode() const { return _leaf.getWNode(); } + bool setLeafNodeIdx(uint32_t idx, const LeafNodeType *splitLeafNode); + void setLeafNodeIdx(uint32_t idx) { _leaf.setIdx(idx); } + uint32_t getLeafNodeIdx() const { return _leaf.getIdx(); } + uint32_t getPathSize() const { return _pathSize; } + PathElement & getPath(uint32_t pidx) { return _path[pidx]; } template <class AggrCalcT> - BTreeNode::Ref - addLevel(BTreeNode::Ref rootRef, BTreeNode::Ref splitNodeRef, - bool inRightSplit, const AggrCalcT &aggrCalc); + BTreeNode::Ref addLevel(BTreeNode::Ref rootRef, BTreeNode::Ref splitNodeRef, bool inRightSplit, const AggrCalcT &aggrCalc); - BTreeNode::Ref - removeLevel(BTreeNode::Ref rootRef, InternalNodeType *rootNode); + BTreeNode::Ref removeLevel(BTreeNode::Ref rootRef, InternalNodeType *rootNode); + void removeLast(BTreeNode::Ref rootRef); - void - removeLast(BTreeNode::Ref rootRef); - - void - adjustSteal(uint32_t level, bool leftVictimKilled, uint32_t stolen) - { + void adjustSteal(uint32_t level, bool leftVictimKilled, uint32_t stolen) { assert(_pathSize > level); if (leftVictimKilled) { _path[level].adjustLeftVictimKilled(); diff --git a/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h b/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h index b537602c703..c7c635b4471 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h +++ b/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h @@ -34,10 +34,6 @@ public: using DataStoreBase = datastore::DataStoreBase; private: - BTreeNodeAllocator(const BTreeNodeAllocator &rhs); - - BTreeNodeAllocator & operator=(const BTreeNodeAllocator &rhs); - NodeStore _nodeStore; using RefVector = vespalib::Array<BTreeNode::Ref>; @@ -53,6 +49,8 @@ private: RefVector _leafHoldUntilFreeze; public: + BTreeNodeAllocator(const BTreeNodeAllocator &rhs) = delete; + BTreeNodeAllocator & operator=(const BTreeNodeAllocator &rhs) = delete; BTreeNodeAllocator(); ~BTreeNodeAllocator(); diff --git a/vespalib/src/vespa/vespalib/geo/zcurve.cpp b/vespalib/src/vespa/vespalib/geo/zcurve.cpp index c207f966704..d04a04fda0a 100644 --- a/vespalib/src/vespa/vespalib/geo/zcurve.cpp +++ b/vespalib/src/vespa/vespalib/geo/zcurve.cpp @@ -10,15 +10,38 @@ namespace vespalib::geo { namespace { + /** + * An area defined by its upper left and lower right corners. The + * z-coordinates between these corners act as a spacial + * over-estimation of the actual area. These areas may never cross + * signed borders, since that would break the whole concept of + * hierarchical spatial partitioning. + **/ +struct Area { + const ZCurve::Point min; + const ZCurve::Point max; + Area(const Area &rhs) = default; + Area(int32_t min_x, int32_t min_y, + int32_t max_x, int32_t max_y) + : min(min_x, min_y), max(max_x, max_y) + { + assert((min_x <= max_x) && ((min_x < 0) == (max_x < 0))); + assert((min_y <= max_y) && ((min_y < 0) == (max_y < 0))); + } + Area &operator=(Area &&rhs) { new ((void*)this) Area(rhs); return *this; } + int64_t size() const { return (static_cast<int64_t>(max.x) - min.x + 1) * (static_cast<int64_t>(max.y) - min.y + 1); } + int64_t estimate() const { return (max.z - min.z + 1); } + int64_t error() const { return estimate() - size(); } +}; + class ZAreaQueue { private: struct MaxAreaErrorCmp { - bool operator()(const ZCurve::Area &a, const ZCurve::Area &b) const { + bool operator()(const Area &a, const Area &b) const { return (a.error() > b.error()); } }; - using Area = ZCurve::Area; using Range = ZCurve::Range; using RangeVector = ZCurve::RangeVector; using Queue = PriorityQueue<Area, MaxAreaErrorCmp, LeftArrayHeap>; @@ -61,7 +84,6 @@ public: class ZAreaSplitter { private: - using Area = ZCurve::Area; using RangeVector = ZCurve::RangeVector; ZAreaQueue _queue; diff --git a/vespalib/src/vespa/vespalib/geo/zcurve.h b/vespalib/src/vespa/vespalib/geo/zcurve.h index 2f92b3a019b..c5fbdc08dce 100644 --- a/vespalib/src/vespa/vespalib/geo/zcurve.h +++ b/vespalib/src/vespa/vespalib/geo/zcurve.h @@ -3,7 +3,6 @@ #pragma once #include <cstdint> -#include <cassert> #include <vector> namespace vespalib::geo { @@ -163,30 +162,6 @@ public: Point(int32_t x_, int32_t y_) : x(x_), y(y_), z(encode(x_, y_)) {} }; - /** - * An area defined by its upper left and lower right corners. The - * z-coordinates between these corners act as a spacial - * over-estimation of the actual area. These areas may never cross - * signed borders, since that would break the whole concept of - * hierarchical spatial partitioning. - **/ - struct Area { - const Point min; - const Point max; - Area(const Area &rhs) = default; - Area(int32_t min_x, int32_t min_y, - int32_t max_x, int32_t max_y) - : min(min_x, min_y), max(max_x, max_y) - { - assert((min_x <= max_x) && ((min_x < 0) == (max_x < 0))); - assert((min_y <= max_y) && ((min_y < 0) == (max_y < 0))); - } - Area &operator=(Area &&rhs) { new ((void*)this) Area(rhs); return *this; } - int64_t size() const { return (static_cast<int64_t>(max.x) - min.x + 1) * (static_cast<int64_t>(max.y) - min.y + 1); } - int64_t estimate() const { return (max.z - min.z + 1); } - int64_t error() const { return estimate() - size(); } - }; - class Range { private: @@ -212,11 +187,9 @@ public: static RangeVector find_ranges(int min_x, int min_y, int max_x, int max_y); - static int64_t - encodeSlow(int32_t x, int32_t y); + static int64_t encodeSlow(int32_t x, int32_t y); - static void - decodeSlow(int64_t enc, int32_t *xp, int32_t *yp); + static void decodeSlow(int64_t enc, int32_t *xp, int32_t *yp); }; } diff --git a/vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp b/vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp index 4b6b82697f7..534177e480a 100644 --- a/vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp +++ b/vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "simple_metrics_manager.h" #include "simple_tick.h" +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".vespalib.metrics.simple_metrics_manager"); diff --git a/vespalib/src/vespa/vespalib/metrics/stable_store.h b/vespalib/src/vespa/vespalib/metrics/stable_store.h index f249fd7729e..d456150ab7e 100644 --- a/vespalib/src/vespa/vespalib/metrics/stable_store.h +++ b/vespalib/src/vespa/vespalib/metrics/stable_store.h @@ -4,7 +4,6 @@ #include <memory> #include <vector> -#include <assert.h> namespace vespalib { @@ -54,8 +53,8 @@ private: StableStore(size_t sz, UP &&more, std::vector<T> &&mine); - size_t _size; - UP _more; + size_t _size; + UP _more; std::vector<T> _mine; }; diff --git a/vespalib/src/vespa/vespalib/util/fiddle.h b/vespalib/src/vespa/vespalib/util/fiddle.h index f4d2ac33695..b6799d9c778 100644 --- a/vespalib/src/vespa/vespalib/util/fiddle.h +++ b/vespalib/src/vespa/vespalib/util/fiddle.h @@ -4,8 +4,7 @@ #include <cassert> -namespace vespalib { -namespace bits { +namespace vespalib::bits { //----------------------------------------------------------------------------- @@ -79,6 +78,5 @@ uint32_t split_range(uint32_t min, uint32_t max, //----------------------------------------------------------------------------- -} // namespace bits -} // namespace vespalib +} diff --git a/vespalib/src/vespa/vespalib/util/latch.h b/vespalib/src/vespa/vespalib/util/latch.h index 3ae49aeb11f..9110b898372 100644 --- a/vespalib/src/vespa/vespalib/util/latch.h +++ b/vespalib/src/vespa/vespalib/util/latch.h @@ -4,7 +4,6 @@ #include <mutex> #include <condition_variable> -#include <cassert> namespace vespalib { |