From 83bf690cfd0e638bc597d33a583276115567bbac Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Wed, 31 May 2017 10:10:03 +0000 Subject: Move ZcBuf to separate files. --- .../src/vespa/searchlib/diskindex/CMakeLists.txt | 1 + searchlib/src/vespa/searchlib/diskindex/zcbuf.cpp | 60 ++++++++++++++++ searchlib/src/vespa/searchlib/diskindex/zcbuf.h | 80 ++++++++++++++++++++++ .../src/vespa/searchlib/diskindex/zcposting.cpp | 39 ----------- .../src/vespa/searchlib/diskindex/zcposting.h | 77 +-------------------- 5 files changed, 142 insertions(+), 115 deletions(-) create mode 100644 searchlib/src/vespa/searchlib/diskindex/zcbuf.cpp create mode 100644 searchlib/src/vespa/searchlib/diskindex/zcbuf.h (limited to 'searchlib') diff --git a/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt b/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt index d9b7237c065..35e8ef6ca11 100644 --- a/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt @@ -19,6 +19,7 @@ vespa_add_library(searchlib_diskindex OBJECT pagedict4file.cpp pagedict4randread.cpp wordnummapper.cpp + zcbuf.cpp zcposocc.cpp zcposocciterators.cpp zcposoccrandread.cpp diff --git a/searchlib/src/vespa/searchlib/diskindex/zcbuf.cpp b/searchlib/src/vespa/searchlib/diskindex/zcbuf.cpp new file mode 100644 index 00000000000..937bef411db --- /dev/null +++ b/searchlib/src/vespa/searchlib/diskindex/zcbuf.cpp @@ -0,0 +1,60 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "zcbuf.h" +#include +#include + +namespace search::diskindex { + +ZcBuf::ZcBuf() + : _valI(NULL), + _valE(NULL), + _mallocStart(NULL), + _mallocSize(0) +{ +} + +ZcBuf::~ZcBuf() +{ + free(_mallocStart); +} + +void +ZcBuf::clearReserve(size_t reserveSize) +{ + if (reserveSize + zcSlack() > _mallocSize) { + size_t newSize = _mallocSize * 2; + if (newSize < 16) + newSize = 16; + while (newSize < reserveSize + zcSlack()) + newSize *= 2; + uint8_t *newBuf = static_cast(malloc(newSize)); + free(_mallocStart); + _mallocStart = newBuf; + _mallocSize = newSize; + } + _valE = _mallocStart + _mallocSize - zcSlack(); + _valI = _mallocStart; +} + + +void +ZcBuf::expand() +{ + size_t newSize = _mallocSize * 2; + size_t oldSize = size(); + if (newSize < 16) + newSize = 16; + + uint8_t *newBuf = static_cast(malloc(newSize)); + + if (oldSize > 0) + memcpy(newBuf, _mallocStart, oldSize); + free(_mallocStart); + _mallocStart = newBuf; + _mallocSize = newSize; + _valI = _mallocStart + oldSize; + _valE = _mallocStart + newSize - zcSlack(); +} + +} // namespace search::diskindex diff --git a/searchlib/src/vespa/searchlib/diskindex/zcbuf.h b/searchlib/src/vespa/searchlib/diskindex/zcbuf.h new file mode 100644 index 00000000000..e73b55c2333 --- /dev/null +++ b/searchlib/src/vespa/searchlib/diskindex/zcbuf.h @@ -0,0 +1,80 @@ +// Copyright 2017 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include +#include + +namespace search::diskindex { + +class ZcBuf +{ +public: + uint8_t *_valI; + uint8_t *_valE; + uint8_t *_mallocStart; + size_t _mallocSize; + + ZcBuf(); + ~ZcBuf(); + + static size_t zcSlack() { return 4; } + void clearReserve(size_t reserveSize); + void clear() { _valI = _mallocStart; } + size_t capacity() const { return _valE - _mallocStart; } + size_t size() const { return _valI - _mallocStart; } + size_t pos() const { return _valI - _mallocStart; } + void expand(); + + void maybeExpand() { + if (__builtin_expect(_valI >= _valE, false)) + expand(); + } + + void encode(uint32_t num) { + for (;;) { + if (num < (1 << 7)) { + *_valI++ = num; + break; + } + *_valI++ = (num & ((1 << 7) - 1)) | (1 << 7); + num >>= 7; + } + maybeExpand(); + } + + uint32_t decode() { + uint32_t res; + uint8_t *valI = _valI; + if (__builtin_expect(valI[0] < (1 << 7), true)) { + res = valI[0]; + valI += 1; + } else if (__builtin_expect(valI[1] < (1 << 7), true)) { + res = (valI[0] & ((1 << 7) - 1)) + + (valI[1] << 7); + valI += 2; + } else if (__builtin_expect(valI[2] < (1 << 7), true)) { + res = (valI[0] & ((1 << 7) - 1)) + + ((valI[1] & ((1 << 7) - 1)) << 7) + + (valI[2] << 14); + valI += 3; + } else if (__builtin_expect(valI[3] < (1 << 7), true)) { + res = (valI[0] & ((1 << 7) - 1)) + + ((valI[1] & ((1 << 7) - 1)) << 7) + + ((valI[2] & ((1 << 7) - 1)) << 14) + + (valI[3] << 21); + valI += 4; + } else { + res = (valI[0] & ((1 << 7) - 1)) + + ((valI[1] & ((1 << 7) - 1)) << 7) + + ((valI[2] & ((1 << 7) - 1)) << 14) + + ((valI[3] & ((1 << 7) - 1)) << 21) + + (valI[4] << 28); + valI += 5; + } + _valI = valI; + return res; + } +}; + +} // namespace search::diskindex diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp index f9523ab07f7..cc193b69ca7 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp @@ -37,45 +37,6 @@ using vespalib::nbostream; using vespalib::getLastErrorString; -void -ZcBuf::clearReserve(size_t reserveSize) -{ - if (reserveSize + zcSlack() > _mallocSize) { - size_t newSize = _mallocSize * 2; - if (newSize < 16) - newSize = 16; - while (newSize < reserveSize + zcSlack()) - newSize *= 2; - uint8_t *newBuf = static_cast(malloc(newSize)); - free(_mallocStart); - _mallocStart = newBuf; - _mallocSize = newSize; - } - _valE = _mallocStart + _mallocSize - zcSlack(); - _valI = _mallocStart; -} - - -void -ZcBuf::expand() -{ - size_t newSize = _mallocSize * 2; - size_t oldSize = size(); - if (newSize < 16) - newSize = 16; - - uint8_t *newBuf = static_cast(malloc(newSize)); - - if (oldSize > 0) - memcpy(newBuf, _mallocStart, oldSize); - free(_mallocStart); - _mallocStart = newBuf; - _mallocSize = newSize; - _valI = _mallocStart + oldSize; - _valE = _mallocStart + newSize - zcSlack(); -} - - Zc4PostingSeqRead:: Zc4PostingSeqRead(PostingListCountFileSeqRead *countFile) : PostingListFileSeqRead(), diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposting.h b/searchlib/src/vespa/searchlib/diskindex/zcposting.h index 8ec0f2e2697..1568302cd2e 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposting.h +++ b/searchlib/src/vespa/searchlib/diskindex/zcposting.h @@ -4,6 +4,7 @@ #include #include +#include "zcbuf.h" namespace search { @@ -16,82 +17,6 @@ class PostingListCountFileSeqWrite; namespace diskindex { -class ZcBuf -{ -public: - uint8_t *_valI; - uint8_t *_valE; - uint8_t *_mallocStart; - size_t _mallocSize; - - ZcBuf() - : _valI(NULL), - _valE(NULL), - _mallocStart(NULL), - _mallocSize(0) - {} - - ~ZcBuf() { free(_mallocStart); } - - static size_t zcSlack() { return 4; } - void clearReserve(size_t reserveSize); - void clear() { _valI = _mallocStart; } - size_t capacity() const { return _valE - _mallocStart; } - size_t size() const { return _valI - _mallocStart; } - size_t pos() const { return _valI - _mallocStart; } - void expand(); - - void maybeExpand() { - if (__builtin_expect(_valI >= _valE, false)) - expand(); - } - - void encode(uint32_t num) { - for (;;) { - if (num < (1 << 7)) { - *_valI++ = num; - break; - } - *_valI++ = (num & ((1 << 7) - 1)) | (1 << 7); - num >>= 7; - } - maybeExpand(); - } - - uint32_t decode() { - uint32_t res; - uint8_t *valI = _valI; - if (__builtin_expect(valI[0] < (1 << 7), true)) { - res = valI[0]; - valI += 1; - } else if (__builtin_expect(valI[1] < (1 << 7), true)) { - res = (valI[0] & ((1 << 7) - 1)) + - (valI[1] << 7); - valI += 2; - } else if (__builtin_expect(valI[2] < (1 << 7), true)) { - res = (valI[0] & ((1 << 7) - 1)) + - ((valI[1] & ((1 << 7) - 1)) << 7) + - (valI[2] << 14); - valI += 3; - } else if (__builtin_expect(valI[3] < (1 << 7), true)) { - res = (valI[0] & ((1 << 7) - 1)) + - ((valI[1] & ((1 << 7) - 1)) << 7) + - ((valI[2] & ((1 << 7) - 1)) << 14) + - (valI[3] << 21); - valI += 4; - } else { - res = (valI[0] & ((1 << 7) - 1)) + - ((valI[1] & ((1 << 7) - 1)) << 7) + - ((valI[2] & ((1 << 7) - 1)) << 14) + - ((valI[3] & ((1 << 7) - 1)) << 21) + - (valI[4] << 28); - valI += 5; - } - _valI = valI; - return res; - } -}; - class Zc4PostingSeqRead : public index::PostingListFileSeqRead { Zc4PostingSeqRead(const Zc4PostingSeqRead &); -- cgit v1.2.3