diff options
72 files changed, 506 insertions, 681 deletions
diff --git a/client/go/internal/vespa/document/dispatcher_test.go b/client/go/internal/vespa/document/dispatcher_test.go index 252bd94dff9..382d21501c3 100644 --- a/client/go/internal/vespa/document/dispatcher_test.go +++ b/client/go/internal/vespa/document/dispatcher_test.go @@ -61,8 +61,8 @@ func TestDispatcher(t *testing.T) { breaker := NewCircuitBreaker(time.Second, 0) dispatcher := NewDispatcher(feeder, throttler, breaker, io.Discard, false) docs := []Document{ - {Id: mustParseId("id:ns:type::doc1"), Operation: OperationPut, Fields: []byte(`{"foo": "123"}`)}, - {Id: mustParseId("id:ns:type::doc2"), Operation: OperationPut, Fields: []byte(`{"bar": "456"}`)}, + {Id: mustParseId("id:ns:type::doc1"), Operation: OperationPut, Body: []byte(`{"fields": {"foo": "123"}}`)}, + {Id: mustParseId("id:ns:type::doc2"), Operation: OperationPut, Body: []byte(`{"fields": {"bar": "456"}}`)}, } for _, d := range docs { dispatcher.Enqueue(d) @@ -192,7 +192,7 @@ func BenchmarkDocumentDispatching(b *testing.B) { throttler := newThrottler(8, clock.now) breaker := NewCircuitBreaker(time.Second, 0) dispatcher := NewDispatcher(feeder, throttler, breaker, io.Discard, false) - doc := Document{Id: mustParseId("id:ns:type::doc1"), Operation: OperationPut, Fields: []byte(`{"foo": "123"}`)} + doc := Document{Id: mustParseId("id:ns:type::doc1"), Operation: OperationPut, Body: []byte(`{"fields": {"foo": "123"}}`)} b.ResetTimer() // ignore setup time for n := 0; n < b.N; n++ { diff --git a/client/go/internal/vespa/document/document.go b/client/go/internal/vespa/document/document.go index a33c4a3c5af..8f884b223d7 100644 --- a/client/go/internal/vespa/document/document.go +++ b/client/go/internal/vespa/document/document.go @@ -20,8 +20,6 @@ import ( "github.com/go-json-experiment/json" ) -var asciiSpace = [256]uint8{'\t': 1, '\n': 1, '\v': 1, '\f': 1, '\r': 1, ' ': 1} - type Operation int const ( @@ -36,6 +34,11 @@ const ( jsonString json.Kind = '"' ) +var ( + fieldsPrefix = []byte(`{"fields":`) + fieldsSuffix = []byte("}") +) + // Id represents a Vespa document ID. type Id struct { id string @@ -110,14 +113,13 @@ func ParseId(serialized string) (Id, error) { type Document struct { Id Id Condition string - Fields []byte + Body []byte Operation Operation Create bool } // Decoder decodes documents from a JSON structure which is either an array of objects, or objects separated by newline. type Decoder struct { - r *bufio.Reader dec *json.Decoder buf bytes.Buffer @@ -145,37 +147,28 @@ func (d Document) String() string { if d.Create { sb.WriteString(", create=true") } - if d.Fields != nil { - sb.WriteString(", fields=") - sb.WriteString(string(d.Fields)) + if d.Body != nil { + sb.WriteString(", body=") + sb.WriteString(string(d.Body)) } return sb.String() } func (d *Decoder) guessMode() error { - for !d.array && !d.jsonl { - b, err := d.r.ReadByte() - if err != nil { - return err - } - // Skip leading whitespace - if b < 0x80 && asciiSpace[b] != 0 { - continue - } - switch json.Kind(b) { - case jsonObjectStart: - d.jsonl = true - case jsonArrayStart: - d.array = true - default: - return fmt.Errorf("unexpected token: %q", string(b)) - } - if err := d.r.UnreadByte(); err != nil { - return err - } - if err := d.readArrayDelim(true); err != nil { + if d.array || d.jsonl { + return nil + } + kind := d.dec.PeekKind() + switch kind { + case jsonArrayStart: + if _, err := d.readNext(jsonArrayStart); err != nil { return err } + d.array = true + case jsonObjectStart: + d.jsonl = true + default: + return fmt.Errorf("expected %s or %s, got %s", jsonArrayStart, jsonObjectStart, kind) } return nil } @@ -191,18 +184,6 @@ func (d *Decoder) readNext(kind json.Kind) (json.Token, error) { return t, nil } -func (d *Decoder) readArrayDelim(open bool) error { - if !d.array { - return nil - } - kind := jsonArrayEnd - if open { - kind = jsonArrayStart - } - _, err := d.readNext(kind) - return err -} - func (d *Decoder) readString() (string, error) { t, err := d.readNext(jsonString) if err != nil { @@ -231,7 +212,7 @@ func (d *Decoder) Decode() (Document, error) { return doc, err } -func (d *Decoder) readField(name string, doc *Document) error { +func (d *Decoder) readField(name string, offset int64, doc *Document) error { readId := false switch name { case "id", "put": @@ -259,9 +240,9 @@ func (d *Decoder) readField(name string, doc *Document) error { if _, err := d.readNext(jsonObjectStart); err != nil { return err } - start := d.dec.InputOffset() - 1 - // Skip data between the most recent ending position of fields and current offset - d.buf.Next(int(start - d.fieldsEnd)) + // Skip data between start of operation and start of fields + fieldsStart := d.dec.InputOffset() - 1 + d.buf.Next(int(fieldsStart - offset)) depth := 1 for depth > 0 { t, err := d.dec.ReadToken() @@ -276,10 +257,11 @@ func (d *Decoder) readField(name string, doc *Document) error { } } d.fieldsEnd = d.dec.InputOffset() - doc.Fields = make([]byte, int(d.fieldsEnd-start)) - if _, err := d.buf.Read(doc.Fields); err != nil { - return err - } + fields := d.buf.Next(int(d.fieldsEnd - fieldsStart)) + doc.Body = make([]byte, 0, len(fieldsPrefix)+len(fields)+len(fieldsSuffix)) + doc.Body = append(doc.Body, fieldsPrefix...) + doc.Body = append(doc.Body, fields...) + doc.Body = append(doc.Body, fieldsSuffix...) } if readId { s, err := d.readString() @@ -296,12 +278,13 @@ func (d *Decoder) readField(name string, doc *Document) error { } func (d *Decoder) decode() (Document, error) { + start := d.dec.InputOffset() if err := d.guessMode(); err != nil { return Document{}, err } - if d.dec.PeekKind() == jsonArrayEnd { + if d.array && d.dec.PeekKind() == jsonArrayEnd { // Reached end of the array holding document operations - if err := d.readArrayDelim(false); err != nil { + if _, err := d.readNext(jsonArrayEnd); err != nil { return Document{}, err } return Document{}, io.EOF @@ -319,13 +302,17 @@ loop: if err != nil { return Document{}, err } - if err := d.readField(t.String(), &doc); err != nil { + if err := d.readField(t.String(), start, &doc); err != nil { return Document{}, err } default: if _, err := d.readNext(jsonObjectEnd); err != nil { return Document{}, err } + // Drop operation from the buffer + start = max(start, d.fieldsEnd) + end := d.dec.InputOffset() + d.buf.Next(int(end - start)) break loop } } @@ -333,9 +320,9 @@ loop: } func NewDecoder(r io.Reader) *Decoder { - sz := 1 << 26 - d := &Decoder{r: bufio.NewReaderSize(r, sz)} - d.dec = json.NewDecoder(io.TeeReader(d.r, &d.buf)) + br := bufio.NewReaderSize(r, 1<<26) + d := &Decoder{} + d.dec = json.NewDecoder(io.TeeReader(br, &d.buf)) return d } diff --git a/client/go/internal/vespa/document/document_test.go b/client/go/internal/vespa/document/document_test.go index 71400314634..fbaa076ab9d 100644 --- a/client/go/internal/vespa/document/document_test.go +++ b/client/go/internal/vespa/document/document_test.go @@ -116,7 +116,8 @@ func feedInput(jsonl bool) string { "fields": { "foo" : "123", "bar": {"a": [1, 2, 3]}} }`, ` -{ + + { "put": "id:ns:type::doc2", "create": false, "condition": "foo", @@ -129,9 +130,13 @@ func feedInput(jsonl bool) string { `, ` { + "fields": {"qux": "789"}, "put": "id:ns:type::doc4", - "create": true, - "fields": {"qux": "789"} + "create": true +}`, + ` +{ + "remove": "id:ns:type::doc5" }`} if jsonl { return strings.Join(operations, "\n") @@ -141,16 +146,17 @@ func feedInput(jsonl bool) string { func testDocumentDecoder(t *testing.T, jsonLike string) { t.Helper() - r := NewDecoder(strings.NewReader(jsonLike)) + dec := NewDecoder(strings.NewReader(jsonLike)) want := []Document{ - {Id: mustParseId("id:ns:type::doc1"), Operation: OperationPut, Fields: []byte(`{ "foo" : "123", "bar": {"a": [1, 2, 3]}}`)}, - {Id: mustParseId("id:ns:type::doc2"), Operation: OperationPut, Condition: "foo", Fields: []byte(`{"bar": "456"}`)}, + {Id: mustParseId("id:ns:type::doc1"), Operation: OperationPut, Body: []byte(`{"fields":{ "foo" : "123", "bar": {"a": [1, 2, 3]}}}`)}, + {Id: mustParseId("id:ns:type::doc2"), Operation: OperationPut, Condition: "foo", Body: []byte(`{"fields":{"bar": "456"}}`)}, {Id: mustParseId("id:ns:type::doc3"), Operation: OperationRemove}, - {Id: mustParseId("id:ns:type::doc4"), Operation: OperationPut, Create: true, Fields: []byte(`{"qux": "789"}`)}, + {Id: mustParseId("id:ns:type::doc4"), Operation: OperationPut, Create: true, Body: []byte(`{"fields":{"qux": "789"}}`)}, + {Id: mustParseId("id:ns:type::doc5"), Operation: OperationRemove}, } got := []Document{} for { - doc, err := r.Decode() + doc, err := dec.Decode() if err == io.EOF { break } @@ -159,6 +165,13 @@ func testDocumentDecoder(t *testing.T, jsonLike string) { } got = append(got, doc) } + wantBufLen := 0 + if dec.array { + wantBufLen = 1 + } + if l := dec.buf.Len(); l != wantBufLen { + t.Errorf("got dec.buf.Len() = %d, want %d", l, wantBufLen) + } if !reflect.DeepEqual(got, want) { t.Errorf("got %+v, want %+v", got, want) } @@ -179,13 +192,13 @@ func TestDocumentDecoderInvalid(t *testing.T) { "fields": {"foo": "invalid } ` - r := NewDecoder(strings.NewReader(jsonLike)) - _, err := r.Decode() // first object is valid + dec := NewDecoder(strings.NewReader(jsonLike)) + _, err := dec.Decode() // first object is valid if err != nil { t.Errorf("unexpected error: %s", err) } - _, err = r.Decode() - wantErr := "invalid json at byte offset 109: json: invalid character '\\n' within string (expecting non-control character)" + _, err = dec.Decode() + wantErr := "invalid json at byte offset 110: json: invalid character '\\n' within string (expecting non-control character)" if err.Error() != wantErr { t.Errorf("want error %q, got %q", wantErr, err.Error()) } diff --git a/client/go/internal/vespa/document/http.go b/client/go/internal/vespa/document/http.go index 8f7ac5bfe63..ce57ac55f03 100644 --- a/client/go/internal/vespa/document/http.go +++ b/client/go/internal/vespa/document/http.go @@ -30,9 +30,6 @@ const ( ) var ( - fieldsPrefix = []byte(`{"fields":`) - fieldsSuffix = []byte("}") - defaultHeaders http.Header = map[string][]string{ "User-Agent": {fmt.Sprintf("Vespa CLI/%s", build.Version)}, "Content-Type": {"application/json; charset=utf-8"}, @@ -132,15 +129,6 @@ func writeQueryParam(sb *bytes.Buffer, start int, escape bool, k, v string) { } } -func writeRequestBody(w io.Writer, body []byte) error { - for _, b := range [][]byte{fieldsPrefix, body, fieldsSuffix} { - if _, err := w.Write(b); err != nil { - return err - } - } - return nil -} - func (c *Client) methodAndURL(d Document, sb *bytes.Buffer) (string, string) { httpMethod := "" switch d.Operation { @@ -229,7 +217,7 @@ func (c *Client) preparePending() { for pd := range c.pending { pd.buf = c.buffer() method, url := c.methodAndURL(pd.document, pd.buf) - pd.request, pd.err = c.createRequest(method, url, pd.document.Fields, pd.buf) + pd.request, pd.err = c.createRequest(method, url, pd.document.Body, pd.buf) pd.prepared <- true } } @@ -259,24 +247,23 @@ func (c *Client) createRequest(method, url string, body []byte, buf *bytes.Buffe if len(body) == 0 { return newRequest(method, url, nil, false) } - bodySize := len(fieldsPrefix) + len(body) + len(fieldsSuffix) - useGzip := c.options.Compression == CompressionGzip || (c.options.Compression == CompressionAuto && bodySize > 512) - buf.Grow(min(1024, bodySize)) + useGzip := c.options.Compression == CompressionGzip || (c.options.Compression == CompressionAuto && len(body) > 512) + var r io.Reader if useGzip { + buf.Grow(min(1024, len(body))) zw := c.gzipWriter(buf) defer c.gzippers.Put(zw) - if err := writeRequestBody(zw, body); err != nil { + if _, err := zw.Write(body); err != nil { return nil, err } if err := zw.Close(); err != nil { return nil, err } + r = buf } else { - if err := writeRequestBody(buf, body); err != nil { - return nil, err - } + r = bytes.NewReader(body) } - return newRequest(method, url, buf, useGzip) + return newRequest(method, url, r, useGzip) } func (c *Client) clientTimeout() time.Duration { @@ -295,7 +282,10 @@ func (c *Client) Send(document Document) Result { if err != nil { return resultWithErr(result, err) } - bodySize := buf.Len() + bodySize := len(document.Body) + if buf.Len() > 0 { + bodySize = buf.Len() + } resp, err := c.leastBusyClient().Do(req, c.clientTimeout()) if err != nil { return resultWithErr(result, err) diff --git a/client/go/internal/vespa/document/http_test.go b/client/go/internal/vespa/document/http_test.go index 7d636aa8d5c..6eda5f04fd6 100644 --- a/client/go/internal/vespa/document/http_test.go +++ b/client/go/internal/vespa/document/http_test.go @@ -3,7 +3,6 @@ package document import ( "bytes" "fmt" - "net/http" "reflect" "strings" "testing" @@ -62,16 +61,16 @@ func TestClientSend(t *testing.T) { method string url string }{ - {Document{Create: true, Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Fields: []byte(`{"foo": "123"}`)}, + {Document{Create: true, Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Body: []byte(`{"fields":{"foo": "123"}}`)}, "PUT", "https://example.com:1337/document/v1/ns/type/docid/doc1?timeout=5000ms&create=true"}, - {Document{Id: mustParseId("id:ns:type::doc2"), Operation: OperationUpdate, Fields: []byte(`{"foo": "456"}`)}, + {Document{Id: mustParseId("id:ns:type::doc2"), Operation: OperationUpdate, Body: []byte(`{"fields":{"foo": "456"}}`)}, "PUT", "https://example.com:1337/document/v1/ns/type/docid/doc2?timeout=5000ms"}, {Document{Id: mustParseId("id:ns:type::doc3"), Operation: OperationRemove}, "DELETE", "https://example.com:1337/document/v1/ns/type/docid/doc3?timeout=5000ms"}, - {Document{Condition: "foo", Id: mustParseId("id:ns:type::doc4"), Operation: OperationUpdate, Fields: []byte(`{"baz": "789"}`)}, + {Document{Condition: "foo", Id: mustParseId("id:ns:type::doc4"), Operation: OperationUpdate, Body: []byte(`{"fields":{"baz": "789"}}`)}, "PUT", "https://example.com:1337/document/v1/ns/type/docid/doc4?timeout=5000ms&condition=foo"}, } @@ -95,7 +94,6 @@ func TestClientSend(t *testing.T) { MaxLatency: time.Second, }, } - var wantBody bytes.Buffer if i < 3 { httpClient.NextResponseString(200, `{"message":"All good!"}`) wantRes.Status = StatusSuccess @@ -112,11 +110,6 @@ func TestClientSend(t *testing.T) { wantRes.Stats.Errors = 1 wantRes.Stats.BytesRecv = 36 } - if tt.method == http.MethodPut { - wantBody.WriteString(`{"fields":`) - wantBody.Write(doc.Fields) - wantBody.WriteString("}") - } res := client.Send(doc) wantRes.Stats.BytesSent = int64(len(httpClient.LastBody)) if !reflect.DeepEqual(res, wantRes) { @@ -133,8 +126,8 @@ func TestClientSend(t *testing.T) { if r.URL.String() != tt.url { t.Errorf("got r.URL = %q, want %q", r.URL, tt.url) } - if !bytes.Equal(httpClient.LastBody, wantBody.Bytes()) { - t.Errorf("got r.Body = %q, want %q", string(httpClient.LastBody), wantBody.String()) + if !bytes.Equal(httpClient.LastBody, doc.Body) { + t.Errorf("got r.Body = %q, want %q", string(httpClient.LastBody), doc.Body) } } want := Stats{ @@ -164,9 +157,9 @@ func TestClientSendCompressed(t *testing.T) { Timeout: time.Duration(5 * time.Second), }, []util.HTTPClient{httpClient}) - bigBody := fmt.Sprintf(`{"foo": "%s"}`, strings.Repeat("s", 512+1)) - bigDoc := Document{Create: true, Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Fields: []byte(bigBody)} - smallDoc := Document{Create: true, Id: mustParseId("id:ns:type::doc2"), Operation: OperationUpdate, Fields: []byte(`{"foo": "s"}`)} + bigBody := fmt.Sprintf(`{"fields": {"foo": "%s"}}`, strings.Repeat("s", 512+1)) + bigDoc := Document{Create: true, Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Body: []byte(bigBody)} + smallDoc := Document{Create: true, Id: mustParseId("id:ns:type::doc2"), Operation: OperationUpdate, Body: []byte(`{"fields": {"foo": "s"}}`)} var result Result client.options.Compression = CompressionNone @@ -307,7 +300,7 @@ func benchmarkClientSend(b *testing.B, compression Compression, document Documen } func makeDocument(size int) Document { - return Document{Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Fields: []byte(fmt.Sprintf(`{"foo": "%s"}`, randString(size)))} + return Document{Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Body: []byte(fmt.Sprintf(`{"fields": {"foo": "%s"}}`, randString(size)))} } func BenchmarkClientSendSmallUncompressed(b *testing.B) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index e044b97546c..43f045940c9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -28,6 +28,7 @@ import com.yahoo.vespa.model.container.docproc.DocprocChain; import com.yahoo.vespa.model.container.docproc.DocprocChains; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.search.IndexedSearchCluster; +import com.yahoo.vespa.model.search.IndexingDocproc; import com.yahoo.vespa.model.search.IndexingDocprocChain; import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.search.SearchNode; @@ -213,13 +214,17 @@ public class Content extends ConfigModel { /** Select/creates and initializes the indexing cluster coupled to this */ private void buildIndexingClusters(Content content, ConfigModelContext modelContext, ApplicationConfigProducerRoot root) { - if ( ! content.getCluster().getSearch().hasIndexedCluster()) return; - - IndexedSearchCluster indexedSearchCluster = content.getCluster().getSearch().getIndexed(); - if (indexedSearchCluster.hasExplicitIndexingCluster()) { - setExistingIndexingCluster(indexedSearchCluster, content.containers); + var search = content.getCluster().getSearch(); + if (!search.getIndexingDocproc().isPresent()) { + return; + } + var indexingDocproc = search.getIndexingDocproc().get(); + if (indexingDocproc.hasExplicitCluster()) { + setExistingIndexingCluster(content, indexingDocproc, content.containers); } else { - setContainerAsIndexingCluster(indexedSearchCluster, content, modelContext, root); + if (search.hasIndexedCluster()) { + setContainerAsIndexingCluster(search.getIndexed(), content, modelContext, root); + } } } @@ -237,18 +242,19 @@ public class Content extends ConfigModel { targetCluster = content.containers.iterator().next().getCluster(); addDocproc(targetCluster); - indexedSearchCluster.setIndexingClusterName(targetCluster.getName()); - addIndexingChainsTo(targetCluster, indexedSearchCluster); + var indexingDocproc = indexedSearchCluster.getIndexingDocproc(); + indexingDocproc.setClusterName(targetCluster.getName()); + addIndexingChainsTo(targetCluster, content, indexingDocproc); } } - private void setExistingIndexingCluster(IndexedSearchCluster cluster, Collection<ContainerModel> containers) { - String indexingClusterName = cluster.getIndexingClusterName(); + private void setExistingIndexingCluster(Content content, IndexingDocproc indexingDocproc, Collection<ContainerModel> containers) { + String indexingClusterName = indexingDocproc.getClusterName(content.getCluster().getName()); ContainerModel containerModel = findByName(indexingClusterName, containers); if (containerModel == null) - throw new IllegalArgumentException("Content cluster '" + cluster.getClusterName() + "' refers to docproc " + + throw new IllegalArgumentException("Content cluster '" + content.getCluster().getName() + "' refers to docproc " + "cluster '" + indexingClusterName + "', but this cluster does not exist."); - addIndexingChainsTo(containerModel.getCluster(), cluster); + addIndexingChainsTo(containerModel.getCluster(), content, indexingDocproc); } private ContainerModel findByName(String name, Collection<ContainerModel> containers) { @@ -258,19 +264,19 @@ public class Content extends ConfigModel { return null; } - private void addIndexingChainsTo(ContainerCluster<?> indexer, IndexedSearchCluster cluster) { + private void addIndexingChainsTo(ContainerCluster<?> indexer, Content content, IndexingDocproc indexingDocproc) { addIndexingChain(indexer); DocprocChain indexingChain; ComponentRegistry<DocprocChain> allChains = indexer.getDocprocChains().allChains(); - if (cluster.hasExplicitIndexingChain()) { - indexingChain = allChains.getComponent(cluster.getIndexingChainName()); + if (indexingDocproc.hasExplicitChain() && !indexingDocproc.getChainName().equals(IndexingDocprocChain.NAME)) { + indexingChain = allChains.getComponent(indexingDocproc.getChainName()); if (indexingChain == null) { - throw new IllegalArgumentException(cluster + " refers to docproc " + - "chain '" + cluster.getIndexingChainName() + + throw new IllegalArgumentException(content.getCluster() + " refers to docproc " + + "chain '" + indexingDocproc.getChainName() + "' for indexing, but this chain does not exist"); } else if (indexingChain.getId().getName().equals("default")) { - throw new IllegalArgumentException(cluster + " specifies the chain " + + throw new IllegalArgumentException(content.getCluster() + " specifies the chain " + "'default' as indexing chain. As the 'default' chain is run by default, " + "using it as the indexing chain will run it twice. " + "Use a different name for the indexing chain."); @@ -282,7 +288,7 @@ public class Content extends ConfigModel { indexingChain = allChains.getComponent(IndexingDocprocChain.NAME); } - cluster.setIndexingChain(indexingChain); + indexingDocproc.setChain(indexingChain); } private TreeConfigProducer<AnyConfigProducer> getDocProc(ApplicationConfigProducerRoot root) { @@ -301,7 +307,7 @@ public class Content extends ConfigModel { Content content, ConfigModelContext modelContext, ApplicationConfigProducerRoot root) { - String indexerName = cluster.getIndexingClusterName(); + String indexerName = cluster.getIndexingDocproc().getClusterName(content.getCluster().getName()); TreeConfigProducer<AnyConfigProducer> parent = getDocProc(root); ApplicationContainerCluster indexingCluster = new ApplicationContainerCluster(parent, "cluster." + indexerName, indexerName, modelContext.getDeployState()); ContainerModel indexingClusterModel = new ContainerModel(modelContext.withParent(parent).withId(indexingCluster.getSubId())); @@ -334,7 +340,7 @@ public class Content extends ConfigModel { indexingCluster.addContainers(nodes); addIndexingChain(indexingCluster); - cluster.setIndexingChain(indexingCluster.getDocprocChains().allChains().getComponent(IndexingDocprocChain.NAME)); + cluster.getIndexingDocproc().setChain(indexingCluster.getDocprocChains().allChains().getComponent(IndexingDocprocChain.NAME)); } private ContainerCluster<?> getContainerWithDocproc(Collection<ContainerModel> containers) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index a0240d28a3c..ec7acaf819f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; import com.yahoo.vespa.model.content.cluster.ContentCluster; import com.yahoo.vespa.model.search.IndexedSearchCluster; +import com.yahoo.vespa.model.search.IndexingDocproc; import com.yahoo.vespa.model.search.NodeSpec; import com.yahoo.vespa.model.search.SchemaDefinitionXMLHandler; import com.yahoo.vespa.model.search.SearchCluster; @@ -57,6 +58,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> /** The single, indexed search cluster this sets up (supporting multiple document types), or null if none */ private IndexedSearchCluster indexedCluster; + private Optional<IndexingDocproc> indexingDocproc; private Redundancy redundancy; private final String clusterName; @@ -206,6 +208,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> double fractionOfMemoryReserved) { super(parent, "search"); + this.indexingDocproc = Optional.empty(); this.clusterName = clusterName; this.documentDefinitions = documentDefinitions; this.globallyDistributedDocuments = globallyDistributedDocuments; @@ -259,6 +262,10 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> throw new IllegalArgumentException("Duplicate indexed cluster '" + indexedCluster.getClusterName() + "'"); } indexedCluster = (IndexedSearchCluster)sc; + if (indexingDocproc.isPresent()) { + throw new IllegalArgumentException("Indexing docproc has previously been setup for streaming search"); + } + indexingDocproc = Optional.of(indexedCluster.getIndexingDocproc()); } clusters.put(sc.getClusterName(), sc); } @@ -458,6 +465,12 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> public Map<String, SearchCluster> getClusters() { return clusters; } public IndexedSearchCluster getIndexed() { return indexedCluster; } public boolean hasIndexedCluster() { return indexedCluster != null; } + public Optional<IndexingDocproc> getIndexingDocproc() { return indexingDocproc; } + public void setupStreamingSearchIndexingDocProc() { + if (indexingDocproc.isEmpty()) { + indexingDocproc = Optional.of(new IndexingDocproc()); + } + } public String getClusterName() { return clusterName; } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 66a99e1993c..dfdfa9303a7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -203,19 +203,24 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem if (docprocCluster != null) { docprocCluster = docprocCluster.trim(); } - if (c.getSearch().hasIndexedCluster()) { - if (docprocCluster != null && !docprocCluster.isEmpty()) { - c.getSearch().getIndexed().setIndexingClusterName(docprocCluster); - } - } - String docprocChain = e.stringAttribute("chain"); if (docprocChain != null) { docprocChain = docprocChain.trim(); } - if (c.getSearch().hasIndexedCluster()) { - if (docprocChain != null && !docprocChain.isEmpty()) { - c.getSearch().getIndexed().setIndexingChainName(docprocChain); + if (docprocCluster != null && !docprocCluster.isEmpty()) { + if (!c.getSearch().hasIndexedCluster() && !c.getSearch().getIndexingDocproc().isPresent() && + docprocChain != null && !docprocChain.isEmpty()) { + c.getSearch().setupStreamingSearchIndexingDocProc(); + } + var indexingDocproc = c.getSearch().getIndexingDocproc(); + if (indexingDocproc.isPresent()) { + indexingDocproc.get().setClusterName(docprocCluster); + } + } + if (docprocChain != null && !docprocChain.isEmpty()) { + var indexingDocproc = c.getSearch().getIndexingDocproc(); + if (indexingDocproc.isPresent()) { + indexingDocproc.get().setChainName(docprocChain); } } } @@ -451,7 +456,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem @Override public void getConfig(MessagetyperouteselectorpolicyConfig.Builder builder) { - if ( ! getSearch().hasIndexedCluster()) return; + if ( ! getSearch().getIndexingDocproc().isPresent()) return; DocumentProtocol.getConfig(builder, getConfigId()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java index ad0312705ca..6623efb599d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/routing/DocumentProtocol.java @@ -110,7 +110,7 @@ public final class DocumentProtocol implements Protocol, for (ContentCluster cluster : Content.getContentClusters(repo)) { DocumentProtocolPoliciesConfig.Cluster.Builder clusterBuilder = new DocumentProtocolPoliciesConfig.Cluster.Builder(); addSelector(cluster.getConfigId(), cluster.getRoutingSelector(), clusterBuilder); - if (cluster.getSearch().hasIndexedCluster()) + if (cluster.getSearch().getIndexingDocproc().isPresent()) addRoutes(getDirectRouteName(cluster.getConfigId()), getIndexedRouteName(cluster.getConfigId()), clusterBuilder); else clusterBuilder.defaultRoute(cluster.getConfigId()); @@ -227,10 +227,11 @@ public final class DocumentProtocol implements Protocol, for (ContentCluster cluster : content) { RouteSpec spec = new RouteSpec(cluster.getConfigId()); - if (cluster.getSearch().hasIndexedCluster()) { + if (cluster.getSearch().getIndexingDocproc().isPresent()) { + var indexingDocproc = cluster.getSearch().getIndexingDocproc().get(); table.addRoute(spec.addHop("[MessageType:" + cluster.getConfigId() + "]")); table.addRoute(new RouteSpec(getIndexedRouteName(cluster.getConfigId())) - .addHop(cluster.getSearch().getIndexed().getIndexingServiceName()) + .addHop(indexingDocproc.getServiceName()) .addHop("[Content:cluster=" + cluster.getName() + "]")); table.addRoute(new RouteSpec(getDirectRouteName(cluster.getConfigId())) .addHop("[Content:cluster=" + cluster.getName() + "]")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 670460a9f9f..080a2ca43dc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -43,11 +43,7 @@ public class IndexedSearchCluster extends SearchCluster DispatchNodesConfig.Producer, ConfigInstance.Producer { - private String indexingClusterName = null; // The name of the docproc cluster to run indexing, by config. - private String indexingChainName = null; - - private DocprocChain indexingChain; // The actual docproc chain indexing for this. - + private IndexingDocproc indexingDocproc; private Tuning tuning; private SearchCoverage searchCoverage; @@ -77,6 +73,7 @@ public class IndexedSearchCluster extends SearchCluster public IndexedSearchCluster(TreeConfigProducer<AnyConfigProducer> parent, String clusterName, int index, ModelContext.FeatureFlags featureFlags) { super(parent, clusterName, index); + indexingDocproc = new IndexingDocproc(); documentDbsConfigProducer = new MultipleDocumentDatabasesConfigProducer(this, documentDbs); rootDispatch = new DispatchGroup(this); defaultDispatchPolicy = DispatchTuning.Builder.toDispatchPolicy(featureFlags.queryDispatchPolicy()); @@ -87,58 +84,7 @@ public class IndexedSearchCluster extends SearchCluster @Override protected IndexingMode getIndexingMode() { return IndexingMode.REALTIME; } - public final boolean hasExplicitIndexingCluster() { - return indexingClusterName != null; - } - - public final boolean hasExplicitIndexingChain() { - return indexingChainName != null; - } - - /** - * Returns the name of the docproc cluster running indexing for this search cluster. This is derived from the - * services file on initialization, this can NOT be used at runtime to determine indexing chain. When initialization - * is done, the {@link #getIndexingServiceName()} method holds the actual indexing docproc chain object. - * - * @return the name of the docproc cluster associated with this - */ - public String getIndexingClusterName() { - return hasExplicitIndexingCluster() ? indexingClusterName : getClusterName() + ".indexing"; - } - - public String getIndexingChainName() { - return indexingChainName; - } - - public void setIndexingChainName(String indexingChainName) { - this.indexingChainName = indexingChainName; - } - - /** - * Sets the name of the docproc cluster running indexing for this search cluster. This is for initial configuration, - * and will not reflect the actual indexing chain. See {@link #getIndexingClusterName} for more detail. - * - * @param name the name of the docproc cluster associated with this - */ - public void setIndexingClusterName(String name) { - indexingClusterName = name; - } - - public String getIndexingServiceName() { - return indexingChain.getServiceName(); - } - - /** - * Sets the docproc chain that will be running indexing for this search cluster. This is set by the - * {@link com.yahoo.vespa.model.content.Content} model during build. - * - * @param chain the chain that is to run indexing for this cluster - * @return this, to allow chaining - */ - public SearchCluster setIndexingChain(DocprocChain chain) { - indexingChain = chain; - return this; - } + public IndexingDocproc getIndexingDocproc() { return indexingDocproc; } public DispatchGroup getRootDispatch() { return rootDispatch; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocproc.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocproc.java new file mode 100644 index 00000000000..46f3e6f459d --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexingDocproc.java @@ -0,0 +1,68 @@ +package com.yahoo.vespa.model.search; + +import com.yahoo.vespa.model.container.docproc.DocprocChain; + +/** + * Utility class to track configuration for which indexing docproc to use by a search cluster. + */ +public class IndexingDocproc { + private String clusterName; // The name of the docproc cluster to run indexing, by config. + private String chainName; + + private DocprocChain chain; // The actual docproc chain indexing for this. + + public boolean hasExplicitCluster() { + return clusterName != null; + } + + public boolean hasExplicitChain() { + return chainName != null; + } + + /** + * Returns the name of the docproc cluster running indexing for this search cluster. This is derived from the + * services file on initialization, this can NOT be used at runtime to determine indexing chain. When initialization + * is done, the {@link #getServiceName()} method holds the actual indexing docproc chain object. + * + * @return the name of the docproc cluster associated with this + */ + public String getClusterName(String searchClusterName) { + return hasExplicitCluster() ? clusterName : searchClusterName + ".indexing"; + } + + public String getChainName() { + return chainName; + } + + public void setChainName(String name) { + chainName = name; + } + + /** + * Sets the name of the docproc cluster running indexing for this search cluster. This is for initial configuration, + * and will not reflect the actual indexing chain. See {@link #getClusterName} for more detail. + * + * @param name the name of the docproc cluster associated with this + */ + public void setClusterName(String name) { + clusterName = name; + } + + public String getServiceName() { + return chain.getServiceName(); + } + + /** + * Sets the docproc chain that will be running indexing for this search cluster. This is set by the + * {@link com.yahoo.vespa.model.content.Content} model during build. + * + * @param chain the chain that is to run indexing for this cluster + */ + public void setChain(DocprocChain chain) { this.chain = chain; } + + public IndexingDocproc() { + clusterName = null; + chainName = null; + chain = null; + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 14fe7bbcc36..f9b1edf4f35 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1397,4 +1397,42 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(2, fleetControllerConfigBuilder.build().max_number_of_groups_allowed_to_be_down()); } + private void assertIndexingDocprocEnabled(boolean indexed, boolean force, boolean expEnabled) + { + String services = "<?xml version='1.0' encoding='UTF-8' ?>" + + "<services version='1.0'>" + + " <container id='default' version='1.0'>" + + " <document-processing/>" + + " </container>" + + " <content id='search' version='1.0'>" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document-processing cluster='default'" + (force ? " chain='indexing'" : "") + "/>" + + " <document type='type1' mode='" + (indexed ? "index" : "streaming") + "'/>" + + " </documents>" + + " </content>" + + "</services>"; + VespaModel model = createEnd2EndOneNode(new TestProperties(), services); + var searchCluster = model.getContentClusters().get("search").getSearch(); + assertEquals(expEnabled, searchCluster.getIndexingDocproc().isPresent()); + } + + @Test + void testIndexingDocprocEnabledWhenIndexMode() + { + assertIndexingDocprocEnabled(true, false, true); + } + + @Test + void testIndexingDocprocNotEnabledWhenStreamingMode() + { + assertIndexingDocprocEnabled(false, false, false); + } + + @Test + void testIndexingDocprocEnabledWhenStreamingModeAndForced() + { + assertIndexingDocprocEnabled(false, true, true); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java index 4476e128196..ac9d0ad8724 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexingAndDocprocRoutingTest.java @@ -220,7 +220,7 @@ public class IndexingAndDocprocRoutingTest extends ContentBaseTest { fail("Expected exception"); } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().startsWith("Indexing cluster 'musiccluster' specifies the chain 'default' as indexing chain")); + assertTrue(e.getMessage().startsWith("content cluster 'musiccluster' specifies the chain 'default' as indexing chain")); } } diff --git a/document/src/vespa/document/annotation/spantree.h b/document/src/vespa/document/annotation/spantree.h index a0010fafa74..8745fb4421d 100644 --- a/document/src/vespa/document/annotation/spantree.h +++ b/document/src/vespa/document/annotation/spantree.h @@ -4,7 +4,6 @@ #include "annotation.h" #include <vector> -#include <cassert> namespace document { struct SpanNode; @@ -24,7 +23,6 @@ public: SpanTree(vespalib::stringref name, std::unique_ptr<T> root) : _name(name), _root(std::move(root)) { - assert(_root.get()); } ~SpanTree(); diff --git a/document/src/vespa/document/base/testdocman.cpp b/document/src/vespa/document/base/testdocman.cpp index 471a2f8c196..d5b24b51f24 100644 --- a/document/src/vespa/document/base/testdocman.cpp +++ b/document/src/vespa/document/base/testdocman.cpp @@ -7,6 +7,7 @@ #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/vespalib/util/rand48.h> #include <sstream> +#include <cassert> namespace document { diff --git a/document/src/vespa/document/select/valuenodes.cpp b/document/src/vespa/document/select/valuenodes.cpp index b3052cc07e2..06205e6b7d1 100644 --- a/document/src/vespa/document/select/valuenodes.cpp +++ b/document/src/vespa/document/select/valuenodes.cpp @@ -11,6 +11,7 @@ #include <vespa/vespalib/util/md5.h> #include <vespa/document/util/stringutil.h> #include <vespa/vespalib/text/lowercase.h> +#include <cassert> #include <iomanip> #include <sys/time.h> diff --git a/document/src/vespa/document/serialization/annotationdeserializer.cpp b/document/src/vespa/document/serialization/annotationdeserializer.cpp index 41bc9ec8aaa..c449029440f 100644 --- a/document/src/vespa/document/serialization/annotationdeserializer.cpp +++ b/document/src/vespa/document/serialization/annotationdeserializer.cpp @@ -40,7 +40,7 @@ unique_ptr<SpanTree> AnnotationDeserializer::readSpanTree() { deserializer.read(tree_name); _nodes.clear(); SpanNode::UP root = readSpanNode(); - unique_ptr<SpanTree> span_tree(new SpanTree(tree_name.getValue(), std::move(root))); + auto span_tree = std::make_unique<SpanTree>(tree_name.getValue(), std::move(root)); uint32_t annotation_count = getInt1_2_4Bytes(_stream); span_tree->reserveAnnotations(annotation_count); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 8c75f341572..0dee7709b52 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -420,6 +420,12 @@ public class Flags { "Takes effect at next host-admin tick", ZONE_ID); + public static final UnboundListFlag<String> WEIGHTED_ENDPOINT_RECORD_TTL = defineListFlag( + "weighted-endpoint-record-ttl", List.of(), String.class, List.of("jonmv"), "2023-05-16", "2023-06-16", + "A list of endpoints and custom TTLs, on the form \"endpoint-fqdn:TTL-seconds\". " + + "Where specified, CNAME records are used instead of the default ALIAS records, which have a default 60s TTL.", + "Takes effect at redeployment from controller"); + public static final UnboundBooleanFlag ENABLE_CONDITIONAL_PUT_REMOVE_WRITE_REPAIR = defineFeatureFlag( "enable-conditional-put-remove-write-repair", false, List.of("vekterli", "havardpe"), "2023-05-10", "2023-07-01", diff --git a/logd/src/logd/empty_forwarder.cpp b/logd/src/logd/empty_forwarder.cpp index dda03c46c01..b601ea6d890 100644 --- a/logd/src/logd/empty_forwarder.cpp +++ b/logd/src/logd/empty_forwarder.cpp @@ -5,6 +5,7 @@ #include <vespa/log/exceptions.h> #include <vespa/log/log_message.h> #include <vespa/vespalib/util/size_literals.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".logd.empty_forwarder"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java index 32fe9ba9f7b..5def863113c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java @@ -50,7 +50,7 @@ public abstract class OsUpgrader { /** The duration this leaves new nodes alone before scheduling any upgrade */ private Duration gracePeriod() { - return Duration.ofDays(1); + return nodeRepository.zone().system().isCd() ? Duration.ofHours(4) : Duration.ofDays(1); } } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp b/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp index c0c6f729509..9364be4570e 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/document_field_extractor.cpp @@ -16,6 +16,7 @@ #include <vespa/searchcommon/common/undefinedvalues.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/exceptions.h> +#include <cassert> using document::FieldValue; using document::BoolFieldValue; diff --git a/searchcore/src/vespa/searchcore/proton/matching/partial_result.h b/searchcore/src/vespa/searchcore/proton/matching/partial_result.h index f4dc2e31d4d..314fefa3cc0 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/partial_result.h +++ b/searchcore/src/vespa/searchcore/proton/matching/partial_result.h @@ -48,7 +48,7 @@ public: _sortData.push_back(sd); _sortDataSize += sd.second; } - virtual void merge(Source &rhs) override; + void merge(Source &rhs) override; }; } diff --git a/searchcore/src/vespa/searchcorespi/index/iindexmaintaineroperations.h b/searchcore/src/vespa/searchcorespi/index/iindexmaintaineroperations.h index 9025b56dc27..0b089daff7c 100644 --- a/searchcore/src/vespa/searchcorespi/index/iindexmaintaineroperations.h +++ b/searchcore/src/vespa/searchcorespi/index/iindexmaintaineroperations.h @@ -19,7 +19,7 @@ struct IIndexMaintainerOperations { using IFieldLengthInspector = search::index::IFieldLengthInspector; using Schema = search::index::Schema; using SelectorArray = search::diskindex::SelectorArray; - virtual ~IIndexMaintainerOperations() {} + virtual ~IIndexMaintainerOperations() = default; /** * Creates a new memory index using the given schema. diff --git a/searchlib/src/apps/uniform/uniform.cpp b/searchlib/src/apps/uniform/uniform.cpp index 807b8d61a9e..95d2bb1a7d1 100644 --- a/searchlib/src/apps/uniform/uniform.cpp +++ b/searchlib/src/apps/uniform/uniform.cpp @@ -3,6 +3,7 @@ #include <vespa/vespalib/util/signalhandler.h> #include <vespa/searchlib/bitcompression/compression.h> #include <cinttypes> +#include <cassert> static uint64_t maxExpGolombVal(uint64_t kValue, uint64_t maxBits) diff --git a/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp b/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp index 54a7d6ea286..4ac4c92f658 100644 --- a/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp +++ b/searchlib/src/tests/fef/termfieldmodel/termfieldmodel_test.cpp @@ -27,6 +27,26 @@ struct State { State::State() : term(), md(), f3(nullptr), f5(nullptr), f7(nullptr), array() {} State::~State() = default; +/** + * convenience adapter for easy iteration + **/ +class SimpleTermFieldRangeAdapter +{ + SimpleTermData& _ref; + size_t _idx; + size_t _lim; +public: + explicit SimpleTermFieldRangeAdapter(SimpleTermData& ref) + : _ref(ref), _idx(0), _lim(ref.numFields()) + {} + + [[nodiscard]] bool valid() const { return (_idx < _lim); } + + [[nodiscard]] SimpleTermFieldData& get() const { return _ref.field(_idx); } + + void next() { assert(valid()); ++_idx; } +}; + void testInvalidId() { const TermFieldMatchData empty; using search::queryeval::SearchIterator; @@ -44,7 +64,7 @@ void testSetup(State &state) { state.term.addField(5); // docfreq = 3 using FRA = search::fef::ITermFieldRangeAdapter; - using SFR = search::fef::SimpleTermFieldRangeAdapter; + using SFR = SimpleTermFieldRangeAdapter; // lookup terms { diff --git a/searchlib/src/vespa/searchcommon/attribute/config.cpp b/searchlib/src/vespa/searchcommon/attribute/config.cpp index 70c2377289f..91495025dee 100644 --- a/searchlib/src/vespa/searchcommon/attribute/config.cpp +++ b/searchlib/src/vespa/searchcommon/attribute/config.cpp @@ -65,4 +65,11 @@ Config::operator==(const Config &b) const _hnsw_index_params == b._hnsw_index_params; } +Config& +Config::set_hnsw_index_params(const HnswIndexParams& params) { + assert(_distance_metric == params.distance_metric()); + _hnsw_index_params = params; + return *this; +} + } diff --git a/searchlib/src/vespa/searchcommon/attribute/config.h b/searchlib/src/vespa/searchcommon/attribute/config.h index 0102f362532..32cac7ec9d6 100644 --- a/searchlib/src/vespa/searchcommon/attribute/config.h +++ b/searchlib/src/vespa/searchcommon/attribute/config.h @@ -10,7 +10,6 @@ #include <vespa/searchcommon/common/dictionary_config.h> #include <vespa/eval/eval/value_type.h> #include <vespa/vespalib/datastore/compaction_strategy.h> -#include <cassert> #include <optional> namespace search::attribute { @@ -72,11 +71,7 @@ public: _distance_metric = value; return *this; } - Config& set_hnsw_index_params(const HnswIndexParams& params) { - assert(_distance_metric == params.distance_metric()); - _hnsw_index_params = params; - return *this; - } + Config& set_hnsw_index_params(const HnswIndexParams& params); Config& clear_hnsw_index_params() { _hnsw_index_params.reset(); return *this; diff --git a/searchlib/src/vespa/searchlib/attribute/load_utils.hpp b/searchlib/src/vespa/searchlib/attribute/load_utils.hpp index 463a62ab01a..614e327942a 100644 --- a/searchlib/src/vespa/searchlib/attribute/load_utils.hpp +++ b/searchlib/src/vespa/searchlib/attribute/load_utils.hpp @@ -5,6 +5,7 @@ #include "load_utils.h" #include "attributevector.h" #include <vespa/searchcommon/attribute/multivalue.h> +#include <cassert> namespace search::attribute { diff --git a/searchlib/src/vespa/searchlib/attribute/readerbase.cpp b/searchlib/src/vespa/searchlib/attribute/readerbase.cpp index e4bc2c02ad6..382d9ccb110 100644 --- a/searchlib/src/vespa/searchlib/attribute/readerbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/readerbase.cpp @@ -6,6 +6,7 @@ #include <vespa/fastlib/io/bufferedfile.h> #include <vespa/searchlib/util/filesizecalculator.h> #include <vespa/vespalib/util/size_literals.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".search.attribute.readerbase"); @@ -73,6 +74,13 @@ ReaderBase::ReaderBase(AttributeVector &attr) ReaderBase::~ReaderBase() = default; +size_t +ReaderBase::getEnumCount() const { + size_t dataSize = _datFile.data_size(); + assert((dataSize % sizeof(uint32_t)) == 0); + return dataSize / sizeof(uint32_t); +} + bool ReaderBase::hasWeight() const { return _weightFile.valid(); diff --git a/searchlib/src/vespa/searchlib/attribute/readerbase.h b/searchlib/src/vespa/searchlib/attribute/readerbase.h index 070dc1f99fb..ff400acc824 100644 --- a/searchlib/src/vespa/searchlib/attribute/readerbase.h +++ b/searchlib/src/vespa/searchlib/attribute/readerbase.h @@ -4,7 +4,6 @@ #include <vespa/searchlib/util/file_with_header.h> #include <vespa/searchlib/util/fileutil.h> -#include <cassert> namespace search { @@ -25,11 +24,7 @@ public: return (_idxFile.data_size()) /sizeof(uint32_t); } - size_t getEnumCount() const { - size_t dataSize = _datFile.data_size(); - assert((dataSize % sizeof(uint32_t)) == 0); - return dataSize / sizeof(uint32_t); - } + size_t getEnumCount() const; size_t getNumValues(); int32_t getNextWeight() { return _weightReader.readHostOrder(); } diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.h b/searchlib/src/vespa/searchlib/bitcompression/compression.h index a77d82d9e8f..2d6b8083d43 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/compression.h +++ b/searchlib/src/vespa/searchlib/bitcompression/compression.h @@ -4,7 +4,6 @@ #include <vespa/searchlib/util/comprfile.h> #include <vespa/vespalib/stllike/string.h> -#include <cassert> namespace vespalib { @@ -1400,7 +1399,6 @@ public: const uint8_t * getByteCompr() const { - assert((_preRead & 7) == 0); return reinterpret_cast<const uint8_t *>(getCompr()) + (getBitOffset() >> 3); } diff --git a/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp b/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp index cdae8058d76..7c38931df77 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/countcompression.cpp @@ -2,6 +2,7 @@ #include "countcompression.h" #include <vespa/searchlib/index/postinglistcounts.h> +#include <cassert> namespace search::bitcompression { diff --git a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.h b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.h index 9208a5be3b8..b162bdc3f2b 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.h +++ b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.h @@ -3,8 +3,8 @@ #pragma once #include "countcompression.h" -#include <limits> #include <vespa/vespalib/stllike/string.h> +#include <cassert> namespace search::bitcompression { diff --git a/searchlib/src/vespa/searchlib/bitcompression/posocc_fields_params.cpp b/searchlib/src/vespa/searchlib/bitcompression/posocc_fields_params.cpp index d2269787068..9d6258ce26f 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/posocc_fields_params.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/posocc_fields_params.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/index/schemautil.h> #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/stllike/asciistream.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".posocc_fields_params"); @@ -38,6 +39,12 @@ PosOccFieldsParams::operator=(const PosOccFieldsParams &rhs) return *this; } +void +PosOccFieldsParams::assertCachedParamsRef() const { + assert(_numFields == _params.size()); + assert(_fieldParams == (_params.empty() ? nullptr : &_params[0])); +} + bool PosOccFieldsParams::operator==(const PosOccFieldsParams &rhs) const diff --git a/searchlib/src/vespa/searchlib/bitcompression/posocc_fields_params.h b/searchlib/src/vespa/searchlib/bitcompression/posocc_fields_params.h index 963a80f06dc..8748557e5a7 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/posocc_fields_params.h +++ b/searchlib/src/vespa/searchlib/bitcompression/posocc_fields_params.h @@ -4,7 +4,6 @@ #include "posocc_field_params.h" #include <vector> -#include <cassert> namespace search::bitcompression { @@ -32,10 +31,7 @@ public: _fieldParams = _params.empty() ? nullptr : &_params[0]; } - void assertCachedParamsRef() const { - assert(_numFields == _params.size()); - assert(_fieldParams == (_params.empty() ? nullptr : &_params[0])); - } + void assertCachedParamsRef() const; uint32_t getNumFields() const { return _numFields; } const PosOccFieldParams *getFieldParams() const { return _fieldParams; } diff --git a/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp b/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp index fd6c723e901..8e1bfd2875c 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/posocccompression.cpp @@ -7,6 +7,7 @@ #include <vespa/searchlib/index/postinglistparams.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/data/fileheader.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".posocccompression"); diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldreader.h b/searchlib/src/vespa/searchlib/diskindex/fieldreader.h index 73b56559115..e97d5deb95c 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldreader.h +++ b/searchlib/src/vespa/searchlib/diskindex/fieldreader.h @@ -1,14 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "wordnummapper.h" +#include "docidmapper.h" +#include "fieldwriter.h" #include <vespa/searchlib/index/postinglistcounts.h> #include <vespa/searchlib/index/dictionaryfile.h> #include <vespa/searchlib/index/docidandfeatures.h> #include <vespa/searchlib/index/postinglistfile.h> #include <vespa/searchlib/index/schemautil.h> -#include "wordnummapper.h" -#include "docidmapper.h" -#include "fieldwriter.h" namespace search::diskindex { @@ -40,7 +40,7 @@ public: using PostingListCounts = index::PostingListCounts; using PostingListParams = index::PostingListParams; - uint64_t _wordNum; + uint64_t _wordNum; DocIdAndFeatures _docIdAndFeatures; protected: std::unique_ptr<DictionaryFileSeqRead> _dictFile; diff --git a/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp b/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp index 5399d70fbe7..432651278e0 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/data/fileheader.h> #include <vespa/fastos/file.h> #include <cinttypes> +#include <cassert> #include <arpa/inet.h> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp index 4fd9d116244..d12081ee89c 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp @@ -8,11 +8,9 @@ #include <vespa/searchlib/common/documentsummary.h> #include <vespa/searchlib/common/i_flush_token.h> #include <vespa/searchlib/index/schemautil.h> -#include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/error.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/lambdatask.h> -#include <vespa/document/util/queue.h> #include <filesystem> #include <system_error> diff --git a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_reader.cpp b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_reader.cpp index 2170777dbd3..460fac36acc 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_reader.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_reader.cpp @@ -3,6 +3,7 @@ #include "zc4_posting_reader.h" #include "zc4_posting_header.h" #include <vespa/searchlib/index/docidandfeatures.h> +#include <cassert> namespace search::diskindex { @@ -19,9 +20,7 @@ Zc4PostingReader<bigEndian>::Zc4PostingReader(bool dynamic_k) } template <bool bigEndian> -Zc4PostingReader<bigEndian>::~Zc4PostingReader() -{ -} +Zc4PostingReader<bigEndian>::~Zc4PostingReader() = default; template <bool bigEndian> void diff --git a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_reader_base.cpp b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_reader_base.cpp index b50835a648d..c71404d449b 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_reader_base.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_reader_base.cpp @@ -3,7 +3,7 @@ #include "zc4_posting_reader_base.h" #include "zc4_posting_header.h" #include <vespa/searchlib/index/docidandfeatures.h> - +#include <cassert> namespace search::diskindex { using index::PostingListCounts; diff --git a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer.cpp b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer.cpp index 202ed5a23cd..3a1b7928c93 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer.cpp @@ -3,6 +3,7 @@ #include "zc4_posting_writer.h" #include <vespa/searchlib/index/docidandfeatures.h> #include <vespa/searchlib/index/postinglistcounts.h> +#include <cassert> using search::index::DocIdAndFeatures; using search::index::PostingListCounts; diff --git a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.cpp b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.cpp index 3f44b56706a..8a84ccc5731 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.cpp @@ -3,6 +3,7 @@ #include "zc4_posting_writer_base.h" #include <vespa/searchlib/index/postinglistcounts.h> #include <vespa/searchlib/index/postinglistparams.h> +#include <cassert> using search::index::PostingListCounts; using search::index::PostingListParams; diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp index b67a8409581..df33091a4e8 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.cpp @@ -1,9 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "zcposocciterators.h" +#include "zc4_posting_params.h" #include <vespa/searchlib/bitcompression/posocc_fields_params.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> -#include "zc4_posting_params.h" +#include <cassert> namespace search::diskindex { diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp index ef540365208..c9a1563a8e3 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp @@ -8,6 +8,7 @@ #include <vespa/searchlib/index/postinglistparams.h> #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/vespalib/data/fileheader.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".diskindex.zcposting"); diff --git a/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.cpp b/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.cpp index 66404c7a0ff..83a4ae20db5 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.cpp @@ -4,6 +4,7 @@ #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/searchlib/bitcompression/posocccompression.h> +#include <cassert> namespace search::diskindex { diff --git a/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp b/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp index bd13c032a03..16f5ee04be4 100644 --- a/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp @@ -6,6 +6,7 @@ #include <vespa/document/datatype/documenttype.h> #include <vespa/vespalib/encoding/base64.h> #include <vespa/vespalib/locale/c.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".searchlib.documentfieldnode"); diff --git a/searchlib/src/vespa/searchlib/fef/objectstore.h b/searchlib/src/vespa/searchlib/fef/objectstore.h index 06575c61eb5..7ba08284111 100644 --- a/searchlib/src/vespa/searchlib/fef/objectstore.h +++ b/searchlib/src/vespa/searchlib/fef/objectstore.h @@ -2,7 +2,6 @@ #pragma once #include <vespa/vespalib/stllike/hash_map.h> -#include <cassert> namespace search::fef { @@ -66,7 +65,6 @@ const T & as_value(const Anything &val) { using WrapperType = AnyWrapper<T>; const auto *wrapper = dynamic_cast<const WrapperType *>(&val); - assert(wrapper != nullptr); return wrapper->getValue(); } diff --git a/searchlib/src/vespa/searchlib/fef/phrase_splitter_query_env.cpp b/searchlib/src/vespa/searchlib/fef/phrase_splitter_query_env.cpp index dfa1d9886f7..bbbdbd69c67 100644 --- a/searchlib/src/vespa/searchlib/fef/phrase_splitter_query_env.cpp +++ b/searchlib/src/vespa/searchlib/fef/phrase_splitter_query_env.cpp @@ -43,7 +43,6 @@ PhraseSplitterQueryEnv::PhraseSplitterQueryEnv(const IQueryEnvironment & queryEn TermFieldHandle numHandles = 0; // how many handles existed in underlying data for (uint32_t i = 0; i < queryEnv.getNumTerms(); ++i) { const ITermData *td = queryEnv.getTerm(i); - assert(td != nullptr); considerTerm(i, *td, fieldId); numHandles += td->numFields(); } diff --git a/searchlib/src/vespa/searchlib/fef/phrasesplitter.cpp b/searchlib/src/vespa/searchlib/fef/phrasesplitter.cpp index 90a058eda00..b74f12bdb97 100644 --- a/searchlib/src/vespa/searchlib/fef/phrasesplitter.cpp +++ b/searchlib/src/vespa/searchlib/fef/phrasesplitter.cpp @@ -42,7 +42,6 @@ PhraseSplitter::update() for (const auto ©_info : _phrase_splitter_query_env.get_copy_info()) { const TermFieldMatchData *src = _matchData->resolveTermField(copy_info.orig_handle); TermFieldMatchData *dst = resolveSplittedTermField(copy_info.split_handle); - assert(src != nullptr && dst != nullptr); copyTermFieldMatchData(*dst, *src, copy_info.offsetInPhrase); } diff --git a/searchlib/src/vespa/searchlib/fef/simpletermdata.h b/searchlib/src/vespa/searchlib/fef/simpletermdata.h index d501d0848e8..391a00e4c8a 100644 --- a/searchlib/src/vespa/searchlib/fef/simpletermdata.h +++ b/searchlib/src/vespa/searchlib/fef/simpletermdata.h @@ -7,7 +7,6 @@ #include "simpletermfielddata.h" #include <vespa/searchlib/query/weight.h> #include <vector> -#include <cassert> namespace search::fef { @@ -128,24 +127,4 @@ public: } }; -/** - * convenience adapter for easy iteration - **/ -class SimpleTermFieldRangeAdapter -{ - SimpleTermData& _ref; - size_t _idx; - size_t _lim; -public: - explicit SimpleTermFieldRangeAdapter(SimpleTermData& ref) - : _ref(ref), _idx(0), _lim(ref.numFields()) - {} - - [[nodiscard]] bool valid() const { return (_idx < _lim); } - - [[nodiscard]] SimpleTermFieldData& get() const { return _ref.field(_idx); } - - void next() { assert(valid()); ++_idx; } -}; - } diff --git a/searchlib/src/vespa/searchlib/fef/termfieldmatchdataarray.h b/searchlib/src/vespa/searchlib/fef/termfieldmatchdataarray.h index 3c1b76ad40e..46d370ee8fe 100644 --- a/searchlib/src/vespa/searchlib/fef/termfieldmatchdataarray.h +++ b/searchlib/src/vespa/searchlib/fef/termfieldmatchdataarray.h @@ -3,7 +3,6 @@ #pragma once #include <vector> -#include <cassert> #include <cstddef> namespace search::fef { @@ -43,7 +42,6 @@ public: * @param value the pointer to be added **/ TermFieldMatchDataArray &add(TermFieldMatchData *value) { - assert(value != nullptr); _array.push_back(value); return *this; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.cpp index c79d856676d..c1537c6b290 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.cpp @@ -10,6 +10,7 @@ #include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/util/stringfmt.h> #include <stdexcept> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".memoryindex.url_field_inverter"); diff --git a/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp b/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp index 00f17f7963c..51882e6d185 100644 --- a/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp @@ -3,6 +3,7 @@ #include "termnodes.h" #include <vespa/vespalib/util/exceptions.h> #include <charconv> +#include <cassert> using vespalib::IllegalArgumentException; using vespalib::stringref; diff --git a/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp b/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp index 94814dbb9b3..d2aa72011e6 100644 --- a/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/fake_search.cpp @@ -5,9 +5,20 @@ #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/vespalib/objects/visit.h> #include <vespa/searchcommon/attribute/i_search_context.h> +#include <cassert> namespace search::queryeval { +FakeSearch::FakeSearch(const vespalib::string &tag, const vespalib::string &field, + const vespalib::string &term, const FakeResult &res, + fef::TermFieldMatchDataArray tfmda) + : _tag(tag), _field(field), _term(term), + _result(res), _offset(0), _tfmda(std::move(tfmda)), + _ctx(nullptr) +{ + assert(_tfmda.size() == 1); +} + void FakeSearch::doSeek(uint32_t docid) { diff --git a/searchlib/src/vespa/searchlib/queryeval/fake_search.h b/searchlib/src/vespa/searchlib/queryeval/fake_search.h index 5cd04f80499..7b7fdf0f078 100644 --- a/searchlib/src/vespa/searchlib/queryeval/fake_search.h +++ b/searchlib/src/vespa/searchlib/queryeval/fake_search.h @@ -29,13 +29,7 @@ public: const vespalib::string &field, const vespalib::string &term, const FakeResult &res, - fef::TermFieldMatchDataArray tfmda) - : _tag(tag), _field(field), _term(term), - _result(res), _offset(0), _tfmda(std::move(tfmda)), - _ctx(nullptr) - { - assert(_tfmda.size() == 1); - } + fef::TermFieldMatchDataArray tfmda); void attr_ctx(const attribute::ISearchContext *ctx) { _ctx = ctx; } bool is_attr() const { return (_ctx != nullptr); } void doSeek(uint32_t docid) override; diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_search.cpp b/searchlib/src/vespa/searchlib/queryeval/same_element_search.cpp index 98c51d7f1ca..5db1e0057cd 100644 --- a/searchlib/src/vespa/searchlib/queryeval/same_element_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_search.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/objects/visit.hpp> #include <algorithm> #include <functional> +#include <cassert> using TFMD = search::fef::TermFieldMatchData; diff --git a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.cpp b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.cpp index f5069fd4f53..ea264935d42 100644 --- a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_search.cpp @@ -4,6 +4,7 @@ #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/vespalib/objects/visit.h> #include <functional> +#include <cassert> using search::fef::TermFieldMatchData; using std::unique_ptr; diff --git a/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.cpp b/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.cpp index bc92b691ce8..04c8b6b3904 100644 --- a/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.cpp +++ b/searchlib/src/vespa/searchlib/test/diskindex/threelevelcountbuffers.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "threelevelcountbuffers.h" +#include <cassert> namespace search::diskindex { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp index db4ac0f32b0..f1fee7e9ac3 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp @@ -14,6 +14,7 @@ #include <vespa/searchlib/parsequery/stackdumpiterator.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/issue.h> +#include <cassert> using search::common::GeoLocationParser; using search::common::GeoLocationSpec; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp index 6d668561651..07e4bde54d0 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/geoposdfw.cpp @@ -10,6 +10,7 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/issue.h> #include <climits> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docsummary.geoposdfw"); diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp index 596525e17d7..146dd487769 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp @@ -11,6 +11,7 @@ #include <vespa/document/datatype/datatype.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/vespalib/data/slime/inserter.h> +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".vsm.docsumfilter"); diff --git a/vespalib/src/tests/btree/btree_test.cpp b/vespalib/src/tests/btree/btree_test.cpp index afad3523fa3..6305eca41ac 100644 --- a/vespalib/src/tests/btree/btree_test.cpp +++ b/vespalib/src/tests/btree/btree_test.cpp @@ -2,7 +2,6 @@ #include <string> #include <vespa/vespalib/btree/btreeroot.h> -#include <vespa/vespalib/btree/btreebuilder.h> #include <vespa/vespalib/btree/btreenodeallocator.h> #include <vespa/vespalib/btree/btree.h> #include <vespa/vespalib/btree/btreestore.h> @@ -297,6 +296,12 @@ BTreeTest::assertMemoryUsage(const vespalib::MemoryUsage & exp, const vespalib:: return result; } +TEST_F(BTreeTest, control_iterator_size) { + EXPECT_EQ(208u, sizeof(BTreeIteratorBase<uint32_t, uint32_t, NoAggregated>)); + EXPECT_EQ(208u, sizeof(BTreeIteratorBase<uint32_t, BTreeNoLeafData, NoAggregated>)); + EXPECT_EQ(544u, sizeof(MyTree::Iterator)); +} + TEST_F(BTreeTest, require_that_node_insert_works) { GenerationHandler g; diff --git a/vespalib/src/vespa/vespalib/btree/btree.h b/vespalib/src/vespa/vespalib/btree/btree.h index c2f5aac01b7..0099da718a3 100644 --- a/vespalib/src/vespa/vespalib/btree/btree.h +++ b/vespalib/src/vespa/vespalib/btree/btree.h @@ -39,50 +39,22 @@ public: using ConstIterator = typename TreeType::ConstIterator; using FrozenView = typename TreeType::FrozenView; using AggrCalcType = typename TreeType::AggrCalcType; -private: - NodeAllocatorType _alloc; - TreeType _tree; - - BTree(const BTree &rhs); - BTree & - operator=(BTree &rhs); - -public: + BTree(const BTree &rhs) = delete; + BTree & operator=(BTree &rhs) = delete; BTree(); ~BTree(); const NodeAllocatorType &getAllocator() const { return _alloc; } NodeAllocatorType &getAllocator() { return _alloc; } - - void - disableFreeLists() { - _alloc.disableFreeLists(); - } - - void - disable_entry_hold_list() - { - _alloc.disable_entry_hold_list(); - } - - // Inherit doc from BTreeRoot - void clear() { - _tree.clear(_alloc); - } - void assign(Builder & rhs) { - _tree.assign(rhs, _alloc); - } + void disableFreeLists() { _alloc.disableFreeLists(); } + void disable_entry_hold_list() { _alloc.disable_entry_hold_list(); } + void clear() { _tree.clear(_alloc); } + void assign(Builder & rhs) { _tree.assign(rhs, _alloc); } bool insert(const KeyType & key, const DataType & data, CompareT comp = CompareT()) { return _tree.insert(key, data, _alloc, comp); } - - void - insert(Iterator &itr, - const KeyType &key, const DataType &data) - { - _tree.insert(itr, key, data); - } + void insert(Iterator &itr, const KeyType &key, const DataType &data) { _tree.insert(itr, key, data); } Iterator find(const KeyType & key, CompareT comp = CompareT()) const { return _tree.find(key, _alloc, comp); @@ -97,55 +69,23 @@ public: return _tree.remove(key, _alloc, comp); } - void - remove(Iterator &itr) - { - _tree.remove(itr); - } - - Iterator begin() const { - return _tree.begin(_alloc); - } - FrozenView getFrozenView() const { - return _tree.getFrozenView(_alloc); - } - size_t size() const { - return _tree.size(_alloc); - } - vespalib::string toString() const { - return _tree.toString(_alloc); - } - bool isValid(CompareT comp = CompareT()) const { - return _tree.isValid(_alloc, comp); - } - bool isValidFrozen(CompareT comp = CompareT()) const { - return _tree.isValidFrozen(_alloc, comp); - } - size_t bitSize() const { - return _tree.bitSize(_alloc); - } + void remove(Iterator &itr) { _tree.remove(itr); } + Iterator begin() const { return _tree.begin(_alloc); } + FrozenView getFrozenView() const { return _tree.getFrozenView(_alloc); } + size_t size() const { return _tree.size(_alloc); } + vespalib::string toString() const { return _tree.toString(_alloc); } + bool isValid(CompareT comp = CompareT()) const { return _tree.isValid(_alloc, comp); } + bool isValidFrozen(CompareT comp = CompareT()) const { return _tree.isValidFrozen(_alloc, comp); } + size_t bitSize() const { return _tree.bitSize(_alloc); } size_t bitSize(BTreeNode::Ref node) const { return _tree.bitSize(node, _alloc); } - void setRoot(BTreeNode::Ref newRoot) { - _tree.setRoot(newRoot, _alloc); - } - BTreeNode::Ref getRoot() const { - return _tree.getRoot(); - } - vespalib::MemoryUsage getMemoryUsage() const { - return _alloc.getMemoryUsage(); - } - - const AggrT & - getAggregated() const - { - return _tree.getAggregated(_alloc); - } + void setRoot(BTreeNode::Ref newRoot) { _tree.setRoot(newRoot, _alloc); } + BTreeNode::Ref getRoot() const { return _tree.getRoot(); } + vespalib::MemoryUsage getMemoryUsage() const { return _alloc.getMemoryUsage(); } + const AggrT & getAggregated() const { return _tree.getAggregated(_alloc); } - void - thaw(Iterator &itr) - { + void thaw(Iterator &itr) { assert(&itr.getAllocator() == &getAllocator()); _tree.thaw(itr); } @@ -153,18 +93,17 @@ public: void compact_worst(const datastore::CompactionStrategy& compaction_strategy); template <typename FunctionType> - void - foreach_key(FunctionType func) const - { + void foreach_key(FunctionType func) const { _alloc.getNodeStore().foreach_key(_tree.getRoot(), func); } template <typename FunctionType> - void - foreach(FunctionType func) const - { + void foreach(FunctionType func) const { _alloc.getNodeStore().foreach(_tree.getRoot(), func); } +private: + NodeAllocatorType _alloc; + TreeType _tree; }; } diff --git a/vespalib/src/vespa/vespalib/btree/btreeiterator.h b/vespalib/src/vespa/vespalib/btree/btreeiterator.h index 4b99edf592a..7a754880aa3 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeiterator.h +++ b/vespalib/src/vespa/vespalib/btree/btreeiterator.h @@ -36,115 +36,46 @@ class NodeElement using KeyType = typename NodeType::KeyType; using DataType = typename NodeType::DataType; const NodeType *_node; - uint32_t _idx; - - NodeType * - getWNode() const - { - return const_cast<NodeType *>(_node); - } + uint32_t _idx; + NodeType * getWNode() const { return const_cast<NodeType *>(_node); } public: - NodeElement() - : _node(nullptr), - _idx(0u) - { - } + NodeElement() : _node(nullptr), _idx(0u) { } + NodeElement(const NodeType *node, uint32_t idx) : _node(node), _idx(idx) { } - NodeElement(const NodeType *node, uint32_t idx) - : _node(node), - _idx(idx) - { - } + void setNode(const NodeType *node) { _node = node; } + const NodeType * getNode() const { return _node; } + void setIdx(uint32_t idx) { _idx = idx; } + uint32_t getIdx() const { return _idx; } + void incIdx() { ++_idx; } + void decIdx() { --_idx; } - void - setNode(const NodeType *node) - { + void setNodeAndIdx(const NodeType *node, uint32_t idx) { _node = node; - } - - const NodeType * - getNode() const - { - return _node; - } - - void - setIdx(uint32_t idx) - { _idx = idx; } - uint32_t - getIdx() const - { - return _idx; - } - - void - incIdx() - { - ++_idx; - } - - void - decIdx() - { - --_idx; - } - - void - setNodeAndIdx(const NodeType *node, uint32_t idx) - { - _node = node; - _idx = idx; - } - - const KeyType & - getKey() const - { - return _node->getKey(_idx); - } - - const DataType & - getData() const - { - return _node->getData(_idx); - } - + const KeyType & getKey() const { return _node->getKey(_idx); } + const DataType & getData() const { return _node->getData(_idx); } // Only use during compaction when changing reference to moved value DataType &getWData() { return getWNode()->getWData(_idx); } - - bool - valid() const - { - return _node != nullptr; - } - - void - adjustLeftVictimKilled() - { + bool valid() const { return _node != nullptr; } + void adjustLeftVictimKilled() { assert(_idx > 0); --_idx; } - void - adjustSteal(uint32_t stolen) - { + void adjustSteal(uint32_t stolen) { assert(_idx + stolen < _node->validSlots()); _idx += stolen; } - void - adjustSplit(bool inRightSplit) - { + void adjustSplit(bool inRightSplit) { if (inRightSplit) ++_idx; } - bool - adjustSplit(bool inRightSplit, const NodeType *splitNode) - { + bool adjustSplit(bool inRightSplit, const NodeType *splitNode) { adjustSplit(inRightSplit); if (_idx >= _node->validSlots()) { _idx -= _node->validSlots(); @@ -154,18 +85,13 @@ public: return false; } - void - swap(NodeElement &rhs) - { + void swap(NodeElement &rhs) { std::swap(_node, rhs._node); std::swap(_idx, rhs._idx); } - bool - operator!=(const NodeElement &rhs) const - { - return _node != rhs._node || - _idx != rhs._idx; + bool operator!=(const NodeElement &rhs) const { + return (_node != rhs._node) || (_idx != rhs._idx); } }; @@ -243,8 +169,7 @@ protected: * * @param pidx Number of levels above leaf nodes to take into account. */ - size_t - position(uint32_t pidx) const; + size_t position(uint32_t pidx) const; /** * Create iterator pointing to first element in the tree referenced @@ -273,8 +198,7 @@ protected: /** * Step iterator forwards. If at end then leave it at end. */ - BTreeIteratorBase & - operator++() { + BTreeIteratorBase & operator++() { if (_leaf.getNode() == nullptr) { return *this; } @@ -290,8 +214,7 @@ protected: * Step iterator backwards. If at end then place it at last valid * position in tree (cf. rbegin()) */ - BTreeIteratorBase & - operator--(); + BTreeIteratorBase & operator--(); ~BTreeIteratorBase(); BTreeIteratorBase(const BTreeIteratorBase &other); @@ -311,9 +234,7 @@ protected: * from this iterator position to end of subtree. */ template <typename FunctionType> - void - foreach_key_range_start(uint32_t level, FunctionType func) const - { + void foreach_key_range_start(uint32_t level, FunctionType func) const { if (level > 0u) { --level; foreach_key_range_start(level, func); @@ -332,9 +253,7 @@ protected: * subtree before this iterator position). */ template <typename FunctionType> - void - foreach_key_range_end(uint32_t level, FunctionType func) const - { + void foreach_key_range_end(uint32_t level, FunctionType func) const { if (level > 0u) { --level; auto &store = _allocator->getNodeStore(); @@ -348,8 +267,7 @@ protected: } public: - bool - operator==(const BTreeIteratorBase & rhs) const { + bool operator==(const BTreeIteratorBase & rhs) const { if (_leaf.getIdx() != rhs._leaf.getIdx()) { return false; } @@ -367,83 +285,55 @@ public: return true; } - bool - operator!=(const BTreeIteratorBase & rhs) const - { - return !operator==(rhs); - } + bool operator!=(const BTreeIteratorBase & rhs) const { return !operator==(rhs); } /** * Swap iterator with the other. * * @param rhs Other iterator. */ - void - swap(BTreeIteratorBase & rhs); + void swap(BTreeIteratorBase & rhs); /** * Get key at current iterator location. */ - const KeyType & - getKey() const - { - return _leaf.getKey(); - } + const KeyType & getKey() const { return _leaf.getKey(); } /** * Get data at current iterator location. */ - const DataType & - getData() const - { - return _leaf.getData(); - } + const DataType & getData() const { return _leaf.getData(); } /** * Check if iterator is at a valid element, i.e. not at end. */ - bool - valid() const - { - return _leaf.valid(); - } + bool valid() const { return _leaf.valid(); } /** * Return the number of elements in the tree. */ - size_t - size() const; + size_t size() const; /** * Return the current position in the tree. */ - size_t - position() const - { - return position(_pathSize); - } + size_t position() const { return position(_pathSize); } /** * Return the distance between two positions in the tree. */ - ssize_t - operator-(const BTreeIteratorBase &rhs) const; + ssize_t operator-(const BTreeIteratorBase &rhs) const; /** * Return if the tree has data or not (e.g. keys and data or only keys). */ - static bool - hasData() - { - return LeafNodeType::hasData(); - } + static bool hasData() { return LeafNodeType::hasData(); } /** * Move the iterator directly to end. Used by findHelper method in BTree. */ - void - setupEnd(); + void setupEnd(); /** * Setup iterator to be empty and not be associated with any tree. @@ -453,50 +343,41 @@ public: /** * Move iterator to beyond last element in the current tree. */ - void - end() __attribute__((noinline)); + void end() __attribute__((noinline)); /** * Move iterator to beyond last element in the given tree. * * @param rootRef Reference to root of tree. */ - void - end(BTreeNode::Ref rootRef); + void end(BTreeNode::Ref rootRef); /** * Move iterator to first element in the current tree. */ - void - begin(); + void begin(); /** * Move iterator to first element in the given tree. * * @param rootRef Reference to root of tree. */ - void - begin(BTreeNode::Ref rootRef); + void begin(BTreeNode::Ref rootRef); /** * Move iterator to last element in the current tree. */ - void - rbegin(); + void rbegin(); /* * Get aggregated values for the current tree. */ - const AggrT & - getAggregated() const; + const AggrT & getAggregated() const; - bool - identical(const BTreeIteratorBase &rhs) const; + bool identical(const BTreeIteratorBase &rhs) const; template <typename FunctionType> - void - foreach_key(FunctionType func) const - { + void foreach_key(FunctionType func) const { if (_pathSize > 0) { _path[_pathSize - 1].getNode()-> foreach_key(_allocator->getNodeStore(), func); @@ -511,9 +392,7 @@ public: * range [this iterator, end_itr)). */ template <typename FunctionType> - void - foreach_key_range(const BTreeIteratorBase &end_itr, FunctionType func) const - { + void foreach_key_range(const BTreeIteratorBase &end_itr, FunctionType func) const { if (!valid()) { return; } @@ -584,9 +463,7 @@ class BTreeConstIterator : public BTreeIteratorBase<KeyT, DataT, AggrT, TraitsT::PATH_SIZE> { protected: - using ParentType = BTreeIteratorBase<KeyT, - DataT, - AggrT, + using ParentType = BTreeIteratorBase<KeyT, DataT, AggrT, TraitsT::INTERNAL_SLOTS, TraitsT::LEAF_SLOTS, TraitsT::PATH_SIZE>; @@ -645,17 +522,12 @@ public: /** * Default constructor. Iterator is not associated with a tree. */ - BTreeConstIterator() - : ParentType() - { - } + BTreeConstIterator() : ParentType() { } /** * Step iterator forwards. If at end then leave it at end. */ - BTreeConstIterator & - operator++() - { + BTreeConstIterator & operator++() { ParentType::operator++(); return *this; } @@ -664,9 +536,7 @@ public: * Step iterator backwards. If at end then place it at last valid * position in tree (cf. rbegin()) */ - BTreeConstIterator & - operator--() - { + BTreeConstIterator & operator--() { ParentType::operator--(); return *this; } @@ -679,8 +549,7 @@ public: * @param key Key to search for * @param comp Comparator for the tree ordering. */ - void - lower_bound(const KeyType & key, CompareT comp = CompareT()); + void lower_bound(const KeyType & key, CompareT comp = CompareT()); /** * Position iterator at first position with a key that is greater @@ -689,9 +558,7 @@ public: * @param key Key to search for * @param comp Comparator for the tree ordering. */ - void - lower_bound(BTreeNode::Ref rootRef, - const KeyType & key, CompareT comp = CompareT()); + void lower_bound(BTreeNode::Ref rootRef, const KeyType & key, CompareT comp = CompareT()); /** * Step iterator forwards until it is at a position with a key @@ -704,8 +571,7 @@ public: * @param key Key to search for * @param comp Comparator for the tree ordering. */ - void - seek(const KeyType &key, CompareT comp = CompareT()); + void seek(const KeyType &key, CompareT comp = CompareT()); /** * Step iterator forwards until it is at a position with a key @@ -717,8 +583,7 @@ public: * @param key Key to search for * @param comp Comparator for the tree ordering. */ - void - binarySeek(const KeyType &key, CompareT comp = CompareT()); + void binarySeek(const KeyType &key, CompareT comp = CompareT()); /** * Step iterator forwards until it is at a position with a key @@ -730,8 +595,7 @@ public: * @param key Key to search for * @param comp Comparator for the tree ordering. */ - void - linearSeek(const KeyType &key, CompareT comp = CompareT()); + void linearSeek(const KeyType &key, CompareT comp = CompareT()); /** * Step iterator forwards until it is at a position with a key @@ -744,8 +608,7 @@ public: * @param key Key to search for * @param comp Comparator for the tree ordering. */ - void - seekPast(const KeyType &key, CompareT comp = CompareT()); + void seekPast(const KeyType &key, CompareT comp = CompareT()); /** * Step iterator forwards until it is at a position with a key @@ -757,8 +620,7 @@ public: * @param key Key to search for * @param comp Comparator for the tree ordering. */ - void - binarySeekPast(const KeyType &key, CompareT comp = CompareT()); + void binarySeekPast(const KeyType &key, CompareT comp = CompareT()); /** * Step iterator forwards until it is at a position with a key @@ -770,8 +632,7 @@ public: * @param key Key to search for * @param comp Comparator for the tree ordering. */ - void - linearSeekPast(const KeyType &key, CompareT comp = CompareT()); + void linearSeekPast(const KeyType &key, CompareT comp = CompareT()); /** * Validate the iterator as a valid iterator or positioned at @@ -781,8 +642,7 @@ public: * @param rootRef Reference to root of tree to operate on * @param comp Comparator for the tree ordering. */ - void - validate(BTreeNode::Ref rootRef, CompareT comp = CompareT()); + void validate(BTreeNode::Ref rootRef, CompareT comp = CompareT()); }; @@ -795,15 +655,10 @@ template <typename KeyT, typename AggrT = NoAggregated, typename CompareT = std::less<KeyT>, typename TraitsT = BTreeDefaultTraits> -class BTreeIterator : public BTreeConstIterator<KeyT, DataT, AggrT, - CompareT, TraitsT> +class BTreeIterator : public BTreeConstIterator<KeyT, DataT, AggrT, CompareT, TraitsT> { public: - using ParentType = BTreeConstIterator<KeyT, - DataT, - AggrT, - CompareT, - TraitsT>; + using ParentType = BTreeConstIterator<KeyT, DataT, AggrT, CompareT, TraitsT>; using NodeAllocatorType = typename ParentType::NodeAllocatorType; using InternalNodeType = typename ParentType::InternalNodeType; using LeafNodeType = typename ParentType::LeafNodeType; @@ -844,40 +699,27 @@ public: { } - BTreeIterator() - : ParentType() - { - } + BTreeIterator() : ParentType() { } - BTreeIterator & - operator++() - { + BTreeIterator & operator++() { ParentType::operator++(); return *this; } - BTreeIterator & - operator--() - { + BTreeIterator & operator--() { ParentType::operator--(); return *this; } - NodeAllocatorType & - getAllocator() const - { + NodeAllocatorType & getAllocator() const { return const_cast<NodeAllocatorType &>(*_allocator); } - BTreeNode::Ref - moveFirstLeafNode(BTreeNode::Ref rootRef); + BTreeNode::Ref moveFirstLeafNode(BTreeNode::Ref rootRef); - void - moveNextLeafNode(); + void moveNextLeafNode(); - void - writeData(const DataType &data) - { + void writeData(const DataType &data) { _leaf.getWNode()->writeData(_leaf.getIdx(), data); } @@ -889,8 +731,7 @@ public: * The new key must have the same semantic meaning as the old key. * Typically used when compacting data store containing keys. */ - void - writeKey(const KeyType &key); + void writeKey(const KeyType &key); /** * Updata data at the current iterator position. The tree should @@ -900,71 +741,33 @@ public: * @param aggrCalc Calculator for updating aggregated information. */ template <class AggrCalcT> - void - updateData(const DataType &data, const AggrCalcT &aggrCalc); + void updateData(const DataType &data, const AggrCalcT &aggrCalc); /** * Thaw a path from the root node down the the current leaf node in * the current tree, allowing for updates to be performed without * disturbing the frozen version of the tree. */ - BTreeNode::Ref - thaw(BTreeNode::Ref rootRef); + BTreeNode::Ref thaw(BTreeNode::Ref rootRef); private: /* Insert into empty tree */ template <class AggrCalcT> - BTreeNode::Ref - insertFirst(const KeyType &key, const DataType &data, - const AggrCalcT &aggrCalc); - - LeafNodeType * - getLeafNode() const - { - return _leaf.getWNode(); - } - - bool - setLeafNodeIdx(uint32_t idx, const LeafNodeType *splitLeafNode); - - void - setLeafNodeIdx(uint32_t idx) - { - _leaf.setIdx(idx); - } - - uint32_t - getLeafNodeIdx() const - { - return _leaf.getIdx(); - } - - uint32_t - getPathSize() const - { - return _pathSize; - } - - PathElement & - getPath(uint32_t pidx) - { - return _path[pidx]; - } + BTreeNode::Ref insertFirst(const KeyType &key, const DataType &data, const AggrCalcT &aggrCalc); + LeafNodeType * getLeafNode() const { return _leaf.getWNode(); } + bool setLeafNodeIdx(uint32_t idx, const LeafNodeType *splitLeafNode); + void setLeafNodeIdx(uint32_t idx) { _leaf.setIdx(idx); } + uint32_t getLeafNodeIdx() const { return _leaf.getIdx(); } + uint32_t getPathSize() const { return _pathSize; } + PathElement & getPath(uint32_t pidx) { return _path[pidx]; } template <class AggrCalcT> - BTreeNode::Ref - addLevel(BTreeNode::Ref rootRef, BTreeNode::Ref splitNodeRef, - bool inRightSplit, const AggrCalcT &aggrCalc); + BTreeNode::Ref addLevel(BTreeNode::Ref rootRef, BTreeNode::Ref splitNodeRef, bool inRightSplit, const AggrCalcT &aggrCalc); - BTreeNode::Ref - removeLevel(BTreeNode::Ref rootRef, InternalNodeType *rootNode); + BTreeNode::Ref removeLevel(BTreeNode::Ref rootRef, InternalNodeType *rootNode); + void removeLast(BTreeNode::Ref rootRef); - void - removeLast(BTreeNode::Ref rootRef); - - void - adjustSteal(uint32_t level, bool leftVictimKilled, uint32_t stolen) - { + void adjustSteal(uint32_t level, bool leftVictimKilled, uint32_t stolen) { assert(_pathSize > level); if (leftVictimKilled) { _path[level].adjustLeftVictimKilled(); diff --git a/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h b/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h index b537602c703..c7c635b4471 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h +++ b/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h @@ -34,10 +34,6 @@ public: using DataStoreBase = datastore::DataStoreBase; private: - BTreeNodeAllocator(const BTreeNodeAllocator &rhs); - - BTreeNodeAllocator & operator=(const BTreeNodeAllocator &rhs); - NodeStore _nodeStore; using RefVector = vespalib::Array<BTreeNode::Ref>; @@ -53,6 +49,8 @@ private: RefVector _leafHoldUntilFreeze; public: + BTreeNodeAllocator(const BTreeNodeAllocator &rhs) = delete; + BTreeNodeAllocator & operator=(const BTreeNodeAllocator &rhs) = delete; BTreeNodeAllocator(); ~BTreeNodeAllocator(); diff --git a/vespalib/src/vespa/vespalib/geo/zcurve.cpp b/vespalib/src/vespa/vespalib/geo/zcurve.cpp index c207f966704..d04a04fda0a 100644 --- a/vespalib/src/vespa/vespalib/geo/zcurve.cpp +++ b/vespalib/src/vespa/vespalib/geo/zcurve.cpp @@ -10,15 +10,38 @@ namespace vespalib::geo { namespace { + /** + * An area defined by its upper left and lower right corners. The + * z-coordinates between these corners act as a spacial + * over-estimation of the actual area. These areas may never cross + * signed borders, since that would break the whole concept of + * hierarchical spatial partitioning. + **/ +struct Area { + const ZCurve::Point min; + const ZCurve::Point max; + Area(const Area &rhs) = default; + Area(int32_t min_x, int32_t min_y, + int32_t max_x, int32_t max_y) + : min(min_x, min_y), max(max_x, max_y) + { + assert((min_x <= max_x) && ((min_x < 0) == (max_x < 0))); + assert((min_y <= max_y) && ((min_y < 0) == (max_y < 0))); + } + Area &operator=(Area &&rhs) { new ((void*)this) Area(rhs); return *this; } + int64_t size() const { return (static_cast<int64_t>(max.x) - min.x + 1) * (static_cast<int64_t>(max.y) - min.y + 1); } + int64_t estimate() const { return (max.z - min.z + 1); } + int64_t error() const { return estimate() - size(); } +}; + class ZAreaQueue { private: struct MaxAreaErrorCmp { - bool operator()(const ZCurve::Area &a, const ZCurve::Area &b) const { + bool operator()(const Area &a, const Area &b) const { return (a.error() > b.error()); } }; - using Area = ZCurve::Area; using Range = ZCurve::Range; using RangeVector = ZCurve::RangeVector; using Queue = PriorityQueue<Area, MaxAreaErrorCmp, LeftArrayHeap>; @@ -61,7 +84,6 @@ public: class ZAreaSplitter { private: - using Area = ZCurve::Area; using RangeVector = ZCurve::RangeVector; ZAreaQueue _queue; diff --git a/vespalib/src/vespa/vespalib/geo/zcurve.h b/vespalib/src/vespa/vespalib/geo/zcurve.h index 2f92b3a019b..c5fbdc08dce 100644 --- a/vespalib/src/vespa/vespalib/geo/zcurve.h +++ b/vespalib/src/vespa/vespalib/geo/zcurve.h @@ -3,7 +3,6 @@ #pragma once #include <cstdint> -#include <cassert> #include <vector> namespace vespalib::geo { @@ -163,30 +162,6 @@ public: Point(int32_t x_, int32_t y_) : x(x_), y(y_), z(encode(x_, y_)) {} }; - /** - * An area defined by its upper left and lower right corners. The - * z-coordinates between these corners act as a spacial - * over-estimation of the actual area. These areas may never cross - * signed borders, since that would break the whole concept of - * hierarchical spatial partitioning. - **/ - struct Area { - const Point min; - const Point max; - Area(const Area &rhs) = default; - Area(int32_t min_x, int32_t min_y, - int32_t max_x, int32_t max_y) - : min(min_x, min_y), max(max_x, max_y) - { - assert((min_x <= max_x) && ((min_x < 0) == (max_x < 0))); - assert((min_y <= max_y) && ((min_y < 0) == (max_y < 0))); - } - Area &operator=(Area &&rhs) { new ((void*)this) Area(rhs); return *this; } - int64_t size() const { return (static_cast<int64_t>(max.x) - min.x + 1) * (static_cast<int64_t>(max.y) - min.y + 1); } - int64_t estimate() const { return (max.z - min.z + 1); } - int64_t error() const { return estimate() - size(); } - }; - class Range { private: @@ -212,11 +187,9 @@ public: static RangeVector find_ranges(int min_x, int min_y, int max_x, int max_y); - static int64_t - encodeSlow(int32_t x, int32_t y); + static int64_t encodeSlow(int32_t x, int32_t y); - static void - decodeSlow(int64_t enc, int32_t *xp, int32_t *yp); + static void decodeSlow(int64_t enc, int32_t *xp, int32_t *yp); }; } diff --git a/vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp b/vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp index 4b6b82697f7..534177e480a 100644 --- a/vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp +++ b/vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "simple_metrics_manager.h" #include "simple_tick.h" +#include <cassert> #include <vespa/log/log.h> LOG_SETUP(".vespalib.metrics.simple_metrics_manager"); diff --git a/vespalib/src/vespa/vespalib/metrics/stable_store.h b/vespalib/src/vespa/vespalib/metrics/stable_store.h index f249fd7729e..d456150ab7e 100644 --- a/vespalib/src/vespa/vespalib/metrics/stable_store.h +++ b/vespalib/src/vespa/vespalib/metrics/stable_store.h @@ -4,7 +4,6 @@ #include <memory> #include <vector> -#include <assert.h> namespace vespalib { @@ -54,8 +53,8 @@ private: StableStore(size_t sz, UP &&more, std::vector<T> &&mine); - size_t _size; - UP _more; + size_t _size; + UP _more; std::vector<T> _mine; }; diff --git a/vespalib/src/vespa/vespalib/util/fiddle.h b/vespalib/src/vespa/vespalib/util/fiddle.h index f4d2ac33695..b6799d9c778 100644 --- a/vespalib/src/vespa/vespalib/util/fiddle.h +++ b/vespalib/src/vespa/vespalib/util/fiddle.h @@ -4,8 +4,7 @@ #include <cassert> -namespace vespalib { -namespace bits { +namespace vespalib::bits { //----------------------------------------------------------------------------- @@ -79,6 +78,5 @@ uint32_t split_range(uint32_t min, uint32_t max, //----------------------------------------------------------------------------- -} // namespace bits -} // namespace vespalib +} diff --git a/vespalib/src/vespa/vespalib/util/latch.h b/vespalib/src/vespa/vespalib/util/latch.h index 3ae49aeb11f..9110b898372 100644 --- a/vespalib/src/vespa/vespalib/util/latch.h +++ b/vespalib/src/vespa/vespalib/util/latch.h @@ -4,7 +4,6 @@ #include <mutex> #include <condition_variable> -#include <cassert> namespace vespalib { |