diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2017-01-30 22:27:50 +0100 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2017-01-31 00:26:12 +0100 |
commit | 4ddb29f10d694c1028d573326023bea128691e3c (patch) | |
tree | 7ee147408eb605a8fb91315b3806a44801c5dfa6 | |
parent | c55b4b0788358c3221a7f74439bc41a61515afc7 (diff) |
Targeted include.
24 files changed, 339 insertions, 793 deletions
diff --git a/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp b/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp index b2d80c3e717..285582ea7ea 100644 --- a/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp +++ b/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp @@ -14,6 +14,7 @@ #include <vespa/searchlib/attribute/enumstore.hpp> #include <vespa/searchlib/attribute/attributevector.hpp> #include <vespa/vespalib/util/compress.h> +#include <vespa/fastos/file.h> #include <iostream> #include <vespa/log/log.h> LOG_SETUP("postinglistattribute_test"); @@ -21,7 +22,7 @@ LOG_SETUP("postinglistattribute_test"); using std::shared_ptr; bool -FastOS_UNIX_File::Sync(void) +FastOS_UNIX_File::Sync() { return true; } diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index a1fa038f6d2..a8fd9226638 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -9,6 +9,7 @@ #include <vespa/eval/tensor/default_tensor.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/data/fileheader.h> +#include <vespa/fastos/file.h> #include <vespa/log/log.h> LOG_SETUP("tensorattribute_test"); diff --git a/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp b/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp index d74b7b09c4e..2abd74d603f 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributefilewriter.cpp @@ -1,25 +1,22 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.attribute.attributefilewriter"); - #include "attributefilewriter.h" -#include <vespa/vespalib/data/databuffer.h> +#include "attributefilebufferwriter.h" #include <vespa/vespalib/data/fileheader.h> #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/searchlib/common/tunefileinfo.h> -#include "attributefilebufferwriter.h" +#include <vespa/fastos/file.h> + +#include <vespa/log/log.h> +LOG_SETUP(".searchlib.attribute.attributefilewriter"); using search::common::FileHeaderContext; using vespalib::getLastErrorString; -namespace search -{ +namespace search { -namespace -{ +namespace { const uint32_t headerAlign = 4096; const uint32_t MIN_ALIGNMENT = 4096; @@ -101,31 +98,29 @@ AttributeFileWriter(const TuneFileAttributes &tuneFileAttributes, const FileHeaderContext &fileHeaderContext, const IAttributeSaveTarget::Config &cfg, const vespalib::string &desc) - : _tuneFileAttributes(tuneFileAttributes), + : _file(new FastOS_File()), + _tuneFileAttributes(tuneFileAttributes), _fileHeaderContext(fileHeaderContext), _cfg(cfg), _desc(desc), _fileBitSize(0) -{ -} +{ } -AttributeFileWriter::~AttributeFileWriter() -{ -} +AttributeFileWriter::~AttributeFileWriter() { } bool AttributeFileWriter::open(const vespalib::string &fileName) { if (_tuneFileAttributes._write.getWantSyncWrites()) { - _file.EnableSyncWrites(); + _file->EnableSyncWrites(); } if (_tuneFileAttributes._write.getWantDirectIO()) { - _file.EnableDirectIO(); + _file->EnableDirectIO(); } - _file.OpenWriteOnlyTruncate(fileName.c_str()); - if (!_file.IsOpened()) { + _file->OpenWriteOnlyTruncate(fileName.c_str()); + if (!_file->IsOpened()) { LOG(error, "Could not open attribute vector '%s' for writing: %s", fileName.c_str(), getLastErrorString().c_str()); return false; @@ -139,9 +134,9 @@ void AttributeFileWriter::writeHeader() { vespalib::FileHeader header(headerAlign); - _fileHeaderContext.addTags(header, _file.GetFileName()); + _fileHeaderContext.addTags(header, _file->GetFileName()); addTags(header); - size_t headerLen = header.writeFile(_file); + size_t headerLen = header.writeFile(*_file); assert((headerLen % MIN_ALIGNMENT) == 0); _fileBitSize = headerLen * 8; } @@ -186,7 +181,7 @@ AttributeFileWriter::writeBuf(Buffer buf) { size_t bufLen = buf->getDataLen(); // TODO: pad to DirectIO boundary when burning bridges - writeDirectIOAligned(_file, buf->getData(), bufLen); + writeDirectIOAligned(*_file, buf->getData(), bufLen); _fileBitSize += bufLen * 8; } @@ -194,10 +189,10 @@ AttributeFileWriter::writeBuf(Buffer buf) void AttributeFileWriter::close() { - if (_file.IsOpened()) { - _file.Sync(); - _file.Close(); - updateHeader(_file.GetFileName(), _fileBitSize); + if (_file->IsOpened()) { + _file->Sync(); + _file->Close(); + updateHeader(_file->GetFileName(), _fileBitSize); } } diff --git a/searchlib/src/vespa/searchlib/attribute/attributefilewriter.h b/searchlib/src/vespa/searchlib/attribute/attributefilewriter.h index fa0fd22b837..5693b0be9d2 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributefilewriter.h +++ b/searchlib/src/vespa/searchlib/attribute/attributefilewriter.h @@ -5,22 +5,13 @@ #include "iattributesavetarget.h" #include "iattributefilewriter.h" -namespace vespalib -{ - -class GenericHeader; - -} +class FastOS_FileInterface; -namespace search -{ - -namespace common -{ +namespace vespalib { class GenericHeader; } -class FileHeaderContext; +namespace search { -} +namespace common { class FileHeaderContext; } class TuneFileAttributes; @@ -30,7 +21,7 @@ class TuneFileAttributes; */ class AttributeFileWriter : public IAttributeFileWriter { - FastOS_File _file; + std::unique_ptr<FastOS_FileInterface> _file; const TuneFileAttributes &_tuneFileAttributes; const search::common::FileHeaderContext &_fileHeaderContext; const IAttributeSaveTarget::Config &_cfg; @@ -42,8 +33,7 @@ class AttributeFileWriter : public IAttributeFileWriter void writeHeader(); public: AttributeFileWriter(const TuneFileAttributes &tuneFileAttributes, - const search::common::FileHeaderContext & - fileHeaderContext, + const search::common::FileHeaderContext & fileHeaderContext, const IAttributeSaveTarget::Config &cfg, const vespalib::string &desc); ~AttributeFileWriter(); diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp index 7ead25c3b6a..49be44c33b0 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp @@ -1,21 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".compression"); #include "compression.h" #include <vespa/vespalib/objects/nbostream.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/vespalib/data/fileheader.h> -namespace search -{ +namespace search { -namespace bitcompression -{ +namespace bitcompression { using vespalib::nbostream; @@ -142,8 +135,7 @@ DecodeContext64Base::checkPointRead(nbostream &in) } // namespace bitcompression -namespace -{ +namespace { vespalib::string noFeatures = "NoFeatures"; diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.h b/searchlib/src/vespa/searchlib/bitcompression/compression.h index 6a8e1ce6790..08b9547f740 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/compression.h +++ b/searchlib/src/vespa/searchlib/bitcompression/compression.h @@ -1,42 +1,26 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once #include <vespa/searchlib/util/comprfile.h> #include <vespa/searchlib/index/postinglistparams.h> #include <vespa/vespalib/stllike/string.h> -#include <map> +#include <cassert> -namespace vespalib -{ +namespace vespalib { class nbostream; class GenericHeader; } -namespace search -{ - -namespace index -{ - -class DocIdAndFeatures; - -} // namespace index - -namespace fef -{ +namespace search { -class TermFieldMatchDataArray; + namespace index { class DocIdAndFeatures; } -} // namespace fef +namespace fef { class TermFieldMatchDataArray; } - -namespace bitcompression -{ +namespace bitcompression { class Position { public: @@ -102,7 +86,7 @@ public: static uint64_t _intMask64[65]; static uint64_t _intMask64le[65]; - CodingTables(void); + CodingTables(); }; #define UC64_DECODECONTEXT(prefix) \ @@ -744,15 +728,14 @@ public: // plus sizeof uint64_t. Then shifted left by 3 to represent bits. uint64_t _fileWriteBias; - EncodeContext64Base(void) + EncodeContext64Base() : search::ComprFileEncodeContext(), _valI(NULL), _valE(NULL), _cacheInt(0), _cacheFree(64), _fileWriteBias(64) - { - } + { } EncodeContext64Base(const EncodeContext64Base &other) : search::ComprFileEncodeContext(other), @@ -761,13 +744,9 @@ public: _cacheInt(other._cacheInt), _cacheFree(other._cacheFree), _fileWriteBias(other._fileWriteBias) - { - } + { } - virtual - ~EncodeContext64Base(void) - { - } + ~EncodeContext64Base() { } EncodeContext64Base & operator=(const EncodeContext64Base &rhs) @@ -784,29 +763,22 @@ public: /** * Get number of used units (e.g. _valI - start) */ - virtual int - getUsedUnits(void *start) - { + int getUsedUnits(void *start) override { return _valI - static_cast<uint64_t *>(start); } /** * Get normal full buffer size (e.g. _valE - start) */ - virtual int - getNormalMaxUnits(void *start) - { + int getNormalMaxUnits(void *start) override { return _valE - static_cast<uint64_t *>(start); } /** * Adjust buffer after write (e.g. _valI, _fileWriteBias) */ - virtual void - afterWrite(search::ComprBuffer &cbuf, - uint32_t remainingUnits, - uint64_t bufferStartFilePos) - { + void + afterWrite(search::ComprBuffer &cbuf, uint32_t remainingUnits, uint64_t bufferStartFilePos) override { _valI = static_cast<uint64_t *>(cbuf._comprBuf) + remainingUnits; _fileWriteBias = (bufferStartFilePos - reinterpret_cast<unsigned long>(cbuf._comprBuf) + @@ -817,9 +789,7 @@ public: /** * Adjust buffer size to align end of buffer. */ - virtual void - adjustBufSize(search::ComprBuffer &cbuf) - { + void adjustBufSize(search::ComprBuffer &cbuf) override { uint64_t fileWriteOffset = (_fileWriteBias + ((reinterpret_cast<unsigned long>(cbuf._comprBuf) - @@ -830,15 +800,11 @@ public: cbuf._comprBufSize); } - virtual uint32_t - getUnitByteSize(void) const - { + uint32_t getUnitByteSize() const override { return sizeof(uint64_t); } - void - setupWrite(search::ComprBuffer &cbuf) - { + void setupWrite(search::ComprBuffer &cbuf) { _valI = static_cast<uint64_t *>(cbuf._comprBuf); _fileWriteBias = @@ -850,9 +816,7 @@ public: _cacheFree = 64; } - void - reload(const EncodeContext64Base &other) - { + void reload(const EncodeContext64Base &other) { _valI = other._valI; _valE = other._valE; _cacheInt = other._cacheInt; @@ -860,38 +824,26 @@ public: _fileWriteBias = other._fileWriteBias; } - void - pushBack(EncodeContext64Base &other) const - { + void pushBack(EncodeContext64Base &other) const { other._valI = _valI; other._cacheInt = _cacheInt; other._cacheFree = _cacheFree; } - virtual void - checkPointWrite(vespalib::nbostream &out); - - virtual void - checkPointRead(vespalib::nbostream &in); + void checkPointWrite(vespalib::nbostream &out) override; + void checkPointRead(vespalib::nbostream &in) override; - uint64_t - getWriteOffset(void) const - { - return _fileWriteBias + - (reinterpret_cast<unsigned long>(_valI) << 3) - _cacheFree; + uint64_t getWriteOffset() const { + return _fileWriteBias + (reinterpret_cast<unsigned long>(_valI) << 3) - _cacheFree; } - void - defineWriteOffset(uint64_t writeOffset) - { + void defineWriteOffset(uint64_t writeOffset) { _fileWriteBias = writeOffset - (reinterpret_cast<unsigned long>(_valI) << 3) + _cacheFree; } - virtual uint64_t - getBitPosV(void) const - { + uint64_t getBitPosV() const override { return getWriteOffset(); } @@ -901,11 +853,8 @@ public: * to both decode macros (making them slower) and encoding method (making * it slower). */ - static uint64_t - maxExpGolombVal(uint32_t kValue) - { - return static_cast<uint64_t> - (- (UINT64_C(1) << kValue) - 1); + static uint64_t maxExpGolombVal(uint32_t kValue) { + return static_cast<uint64_t>(- (UINT64_C(1) << kValue) - 1); } /* @@ -914,9 +863,7 @@ public: * * maxBits must be larger than kValue */ - static uint64_t - maxExpGolombVal(uint32_t kValue, uint32_t maxBits) - { + static uint64_t maxExpGolombVal(uint32_t kValue, uint32_t maxBits) { if ((maxBits + kValue + 1) / 2 > 64) { return static_cast<uint64_t>(-1); } @@ -953,7 +900,7 @@ public: * the _cacheInt. Padding of trailing 0-bits is automatically added. */ void - flush(void) + flush() { if (_cacheFree < 64) { *_valI++ = bswap(_cacheInt); @@ -1865,14 +1812,14 @@ public: writeHeader(const vespalib::GenericHeader &header); void - writeComprBufferIfNeeded(void) + writeComprBufferIfNeeded() { if (_valI >= _valE) _writeContext->writeComprBuffer(false); } void - writeComprBuffer(void) + writeComprBuffer() { _writeContext->writeComprBuffer(true); } @@ -1897,23 +1844,19 @@ public: writeHeader(vespalib::GenericHeader &header, const vespalib::string &prefix) const; - virtual const vespalib::string & - getIdentifier(void) const; + virtual const vespalib::string &getIdentifier() const; - virtual void - writeFeatures(const DocIdAndFeatures &features); + virtual void writeFeatures(const DocIdAndFeatures &features); /* * Set parameters. */ - virtual void - setParams(const PostingListParams ¶ms); + virtual void setParams(const PostingListParams ¶ms); /* * Get current parameters. */ - virtual void - getParams(PostingListParams ¶ms) const; + virtual void getParams(PostingListParams ¶ms) const; }; typedef FeatureEncodeContext<true> FeatureEncodeContextBE; diff --git a/searchlib/src/vespa/searchlib/common/bitvector.cpp b/searchlib/src/vespa/searchlib/common/bitvector.cpp index c7c0c4cb9be..bc90bba17b7 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/bitvector.cpp @@ -1,6 +1,4 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1998-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #include "bitvector.h" #include "allocatedbitvector.h" @@ -9,6 +7,7 @@ #include <vespa/vespalib/hwaccelrated/iaccelrated.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/fastos/file.h> using vespalib::make_string; using vespalib::IllegalArgumentException; diff --git a/searchlib/src/vespa/searchlib/common/bitvector.h b/searchlib/src/vespa/searchlib/common/bitvector.h index bb9d5768042..c7e2fa7d26c 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.h +++ b/searchlib/src/vespa/searchlib/common/bitvector.h @@ -7,7 +7,6 @@ #include <memory> #include <vespa/vespalib/util/alloc.h> #include <vespa/vespalib/util/generationholder.h> -#include <vespa/fastos/file.h> #include <bits/stl_algo.h> #include <bits/stl_function.h> #include <vespa/fastos/dynamiclibrary.h> @@ -16,6 +15,8 @@ namespace vespalib { class nbostream; } +class FastOS_FileInterface; + namespace search { class BitWord { diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.cpp index 799b02dd071..a4a82ff9490 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.cpp @@ -1,16 +1,15 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".diskindex.bitvectordictionary"); #include "bitvectordictionary.h" #include <vespa/vespalib/data/fileheader.h> +#include <vespa/fastos/file.h> -namespace search -{ +#include <vespa/log/log.h> +LOG_SETUP(".diskindex.bitvectordictionary"); -namespace diskindex -{ +namespace search { + +namespace diskindex { BitVectorDictionary::BitVectorDictionary() @@ -19,8 +18,7 @@ BitVectorDictionary::BitVectorDictionary() _vectorSize(), _datFile(), _datHeaderLen(0u) -{ -} +{ } BitVectorDictionary::~BitVectorDictionary() diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.h b/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.h index 75b88de1a75..1fa33bf2a4d 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.h +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.h @@ -1,11 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "bitvectorkeyscope.h" #include <vespa/searchlib/common/bitvector.h> #include <vespa/searchlib/index/bitvectorkeys.h> #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/vespalib/stllike/string.h> -#include "bitvectorkeyscope.h" namespace search { @@ -19,22 +19,18 @@ namespace diskindex { class BitVectorDictionary { private: - BitVectorDictionary(const BitVectorDictionary &rhs); - - BitVectorDictionary & - operator=(const BitVectorDictionary &rhs); - typedef search::index::BitVectorWordSingleKey WordSingleKey; - uint32_t _docIdLimit; - std::vector<WordSingleKey> _entries; - size_t _vectorSize; - std::unique_ptr<FastOS_File> _datFile; - uint32_t _datHeaderLen; + uint32_t _docIdLimit; + std::vector<WordSingleKey> _entries; + size_t _vectorSize; + std::unique_ptr<FastOS_FileInterface> _datFile; + uint32_t _datHeaderLen; public: typedef std::shared_ptr<BitVectorDictionary> SP; - + BitVectorDictionary(const BitVectorDictionary &rhs) = delete; + BitVectorDictionary &operator=(const BitVectorDictionary &rhs) = delete; BitVectorDictionary(); ~BitVectorDictionary(); @@ -59,20 +55,11 @@ public: * @param wordNum the word number to lookup a bit vector for. * @return the loaded bit vector or NULL if not found. **/ - BitVector::UP - lookup(uint64_t wordNum); + BitVector::UP lookup(uint64_t wordNum); - uint32_t - getDocIdLimit() const - { - return _docIdLimit; - } + uint32_t getDocIdLimit() const { return _docIdLimit; } - const std::vector<WordSingleKey> & - getEntries() const - { - return _entries; - } + const std::vector<WordSingleKey> & getEntries() const { return _entries; } }; } // namespace diskindex diff --git a/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.cpp b/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.cpp index f1261c83a51..e43e10f0709 100644 --- a/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.cpp @@ -1,42 +1,36 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".diskindex.pagedict4randread"); -#include <vespa/searchlib/util/filekit.h> #include "pagedict4randread.h" #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/data/fileheader.h> +#include <vespa/fastos/file.h> +#include <vespa/log/log.h> +LOG_SETUP(".diskindex.pagedict4randread"); -namespace -{ +namespace { vespalib::string myPId("PageDict4P.1"); vespalib::string mySPId("PageDict4SP.1"); vespalib::string mySSId("PageDict4SS.1"); -vespalib::string emptyId; -vespalib::string emptyStr; } using vespalib::getLastErrorString; -namespace search -{ +namespace search { -namespace diskindex -{ +namespace diskindex { PageDict4RandRead::PageDict4RandRead(void) : DictionaryFileRandRead(), - _ssReader(NULL), + _ssReader(), _ssd(), _ssReadContext(_ssd), - _ssfile(), - _spfile(), - _pfile(), + _ssfile(new FastOS_File()), + _spfile(new FastOS_File()), + _pfile(new FastOS_File()), _ssFileBitSize(0u), _spFileBitSize(0u), _pFileBitSize(0u), @@ -48,10 +42,7 @@ PageDict4RandRead::PageDict4RandRead(void) } -PageDict4RandRead::~PageDict4RandRead(void) -{ - delete _ssReader; -} +PageDict4RandRead::~PageDict4RandRead() { } void @@ -60,7 +51,7 @@ PageDict4RandRead::readSSHeader() DC &ssd = _ssd; vespalib::FileHeader header; - uint32_t headerLen = ssd.readHeader(header, _ssfile.getSize()); + uint32_t headerLen = ssd.readHeader(header, _ssfile->getSize()); assert(header.hasTag("frozen")); assert(header.hasTag("fileBitSize")); assert(header.hasTag("format.0")); @@ -95,14 +86,14 @@ PageDict4RandRead::readSPHeader(void) ComprFileReadContext rc(d); d.setReadContext(&rc); - rc.setFile(&_spfile); - rc.setFileSize(_spfile.GetSize()); + rc.setFile(_spfile.get()); + rc.setFileSize(_spfile->GetSize()); rc.allocComprBuf(512, 32768u); d.emptyBuffer(0); rc.readComprBuffer(); vespalib::FileHeader header; - uint32_t headerLen = d.readHeader(header, _spfile.getSize()); + uint32_t headerLen = d.readHeader(header, _spfile->getSize()); assert(header.hasTag("frozen")); assert(header.hasTag("fileBitSize")); assert(header.hasTag("format.0")); @@ -128,14 +119,14 @@ PageDict4RandRead::readPHeader(void) ComprFileReadContext rc(d); d.setReadContext(&rc); - rc.setFile(&_pfile); - rc.setFileSize(_pfile.GetSize()); + rc.setFile(_pfile.get()); + rc.setFileSize(_pfile->GetSize()); rc.allocComprBuf(512, 32768u); d.emptyBuffer(0); rc.readComprBuffer(); vespalib::FileHeader header; - uint32_t headerLen = d.readHeader(header, _pfile.getSize()); + uint32_t headerLen = d.readHeader(header, _pfile->getSize()); assert(header.hasTag("frozen")); assert(header.hasTag("fileBitSize")); assert(header.hasTag("format.0")); @@ -177,8 +168,7 @@ PageDict4RandRead::lookup(const vespalib::stringref &word, } else { SPLookupRes spRes; size_t pageSize = PageDict4PageParams::getPageByteSize(); - const char *spData = static_cast<const char *> - (_spfile.MemoryMapPtr(0)); + const char *spData = static_cast<const char *>(_spfile->MemoryMapPtr(0)); spRes.lookup(*_ssReader, spData + pageSize * ssRes._sparsePageNum, word, @@ -190,7 +180,7 @@ PageDict4RandRead::lookup(const vespalib::stringref &word, PLookupRes pRes; const char *pData = static_cast<const char *> - (_pfile.MemoryMapPtr(0)); + (_pfile->MemoryMapPtr(0)); pRes.lookup(*_ssReader, pData + pageSize * spRes._pageNum, word, @@ -221,37 +211,32 @@ PageDict4RandRead::open(const vespalib::string &name, if (tuneFileRead.getWantMemoryMap() || true) { int mmapFlags(tuneFileRead.getMemoryMapFlags()); - _ssfile.enableMemoryMap(mmapFlags); - _spfile.enableMemoryMap(mmapFlags); - _pfile.enableMemoryMap(mmapFlags); + _ssfile->enableMemoryMap(mmapFlags); + _spfile->enableMemoryMap(mmapFlags); + _pfile->enableMemoryMap(mmapFlags); } else if (tuneFileRead.getWantDirectIO()) { - _ssfile.EnableDirectIO(); - _spfile.EnableDirectIO(); - _pfile.EnableDirectIO(); + _ssfile->EnableDirectIO(); + _spfile->EnableDirectIO(); + _pfile->EnableDirectIO(); } - if (!_ssfile.OpenReadOnly(ssname.c_str())) { - LOG(error, "could not open %s: %s", - _ssfile.GetFileName(), getLastErrorString().c_str()); + if (!_ssfile->OpenReadOnly(ssname.c_str())) { + LOG(error, "could not open %s: %s", _ssfile->GetFileName(), getLastErrorString().c_str()); return false; } - if (!_spfile.OpenReadOnly(spname.c_str())) { - LOG(error, "could not open %s: %s", - _spfile.GetFileName(), getLastErrorString().c_str()); + if (!_spfile->OpenReadOnly(spname.c_str())) { + LOG(error, "could not open %s: %s", _spfile->GetFileName(), getLastErrorString().c_str()); return false; } - if (!_pfile.OpenReadOnly(pname.c_str())) { - LOG(error, "could not open %s: %s", - _pfile.GetFileName(), getLastErrorString().c_str()); + if (!_pfile->OpenReadOnly(pname.c_str())) { + LOG(error, "could not open %s: %s", _pfile->GetFileName(), getLastErrorString().c_str()); return false; } - uint64_t fileSize = _ssfile.GetSize(); - _ssReadContext.setFile(&_ssfile); + uint64_t fileSize = _ssfile->GetSize(); + _ssReadContext.setFile(_ssfile.get()); _ssReadContext.setFileSize(fileSize); - _ssReadContext.allocComprBuf((fileSize + sizeof(uint64_t) - 1) / - sizeof(uint64_t), - 32768u); + _ssReadContext.allocComprBuf((fileSize + sizeof(uint64_t) - 1) / sizeof(uint64_t), 32768u); _ssd.emptyBuffer(0); _ssReadContext.readComprBuffer(); assert(_ssReadContext.getBufferEndFilePos() >= fileSize); @@ -260,13 +245,8 @@ PageDict4RandRead::open(const vespalib::string &name, readSPHeader(); readPHeader(); - _ssReader = new SSReader(_ssReadContext, - _ssHeaderLen, - _ssFileBitSize, - _spHeaderLen, - _spFileBitSize, - _pHeaderLen, - _pFileBitSize); + _ssReader = std::make_unique<SSReader>(_ssReadContext, _ssHeaderLen, _ssFileBitSize, _spHeaderLen, + _spFileBitSize, _pHeaderLen, _pFileBitSize); _ssReader->setup(_ssd); return true; @@ -276,14 +256,13 @@ PageDict4RandRead::open(const vespalib::string &name, bool PageDict4RandRead::close(void) { - delete _ssReader; - _ssReader = NULL; + _ssReader.reset(); _ssReadContext.dropComprBuf(); _ssReadContext.setFile(NULL); - _ssfile.Close(); - _spfile.Close(); - _pfile.Close(); + _ssfile->Close(); + _spfile->Close(); + _pfile->Close(); return true; } diff --git a/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.h b/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.h index fba9dfd483b..d0d8b4b6209 100644 --- a/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.h +++ b/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.h @@ -7,11 +7,9 @@ #include <vespa/searchlib/bitcompression/countcompression.h> #include <vespa/searchlib/bitcompression/pagedict4.h> -namespace search -{ +namespace search { -namespace diskindex -{ +namespace diskindex { class PageDict4RandRead : public index::DictionaryFileRandRead { @@ -26,13 +24,13 @@ class PageDict4RandRead : public index::DictionaryFileRandRead typedef index::PostingListCounts PostingListCounts; typedef index::PostingListOffsetAndCounts PostingListOffsetAndCounts; - SSReader *_ssReader; + std::unique_ptr<SSReader> _ssReader; DC _ssd; ComprFileReadContext _ssReadContext; - FastOS_File _ssfile; - FastOS_File _spfile; - FastOS_File _pfile; + std::unique_ptr<FastOS_FileInterface> _ssfile; + std::unique_ptr<FastOS_FileInterface> _spfile; + std::unique_ptr<FastOS_FileInterface> _pfile; uint64_t _ssFileBitSize; uint64_t _spFileBitSize; @@ -41,45 +39,22 @@ class PageDict4RandRead : public index::DictionaryFileRandRead uint32_t _spHeaderLen; uint32_t _pHeaderLen; - void - readSSHeader(); - - void - readSPHeader(void); - - void - readPHeader(void); - + void readSSHeader(); + void readSPHeader(); + void readPHeader(); public: - PageDict4RandRead(void); - - virtual - ~PageDict4RandRead(void); + PageDict4RandRead(); + ~PageDict4RandRead(); - virtual bool - lookup(const vespalib::stringref &word, - uint64_t &wordNum, - PostingListOffsetAndCounts &offsetAndCounts); + bool lookup(const vespalib::stringref &word, uint64_t &wordNum, + PostingListOffsetAndCounts &offsetAndCounts) override; - /** - * Open dictionary file for random read. - */ - virtual bool open(const vespalib::string &name, - const TuneFileRandRead &tuneFileRead); + bool open(const vespalib::string &name, const TuneFileRandRead &tuneFileRead) override; - /** - * Close dictionary file. - */ - virtual bool close(void); - - virtual uint64_t - getNumWordIds(void) const; + bool close() override; + uint64_t getNumWordIds() const override; }; - } // namespace diskindex } // namespace search - - - diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.cpp index 26ed327ec52..56c96b751b8 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.cpp @@ -1,13 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".diskindex.zcposoccrandread"); -#include <vespa/searchlib/common/bitvector.h> #include "zcposoccrandread.h" #include "zcposocciterators.h" #include <vespa/vespalib/data/fileheader.h> #include <vespa/searchlib/queryeval/emptysearch.h> +#include <vespa/fastos/file.h> + +#include <vespa/log/log.h> +LOG_SETUP(".diskindex.zcposoccrandread"); using search::bitcompression::EG2PosOccEncodeContext; using search::bitcompression::EGPosOccEncodeContext; @@ -21,24 +21,21 @@ using search::index::PostingListCounts; using search::index::PostingListHandle; using search::ComprFileReadContext; -namespace -{ +namespace { vespalib::string myId4("Zc.4"); vespalib::string myId5("Zc.5"); } -namespace search -{ +namespace search { -namespace diskindex -{ +namespace diskindex { using vespalib::getLastErrorString; -ZcPosOccRandRead::ZcPosOccRandRead(void) - : _file(), +ZcPosOccRandRead::ZcPosOccRandRead() + : _file(new FastOS_File()), _fileSize(0), _minChunkDocs(1 << 30), _minSkipDocs(64), @@ -48,13 +45,12 @@ ZcPosOccRandRead::ZcPosOccRandRead(void) _headerBitSize(0), _fieldsParams(), _dynamicK(true) -{ -} +{ } -ZcPosOccRandRead::~ZcPosOccRandRead(void) +ZcPosOccRandRead::~ZcPosOccRandRead() { - if (_file.IsOpened()) + if (_file->IsOpened()) close(); } @@ -128,7 +124,7 @@ ZcPosOccRandRead::readPostingList(const PostingListCounts &counts, // Align start at 64-bit boundary startOffset -= (startOffset & 7); - void *mapPtr = _file.MemoryMapPtr(startOffset); + void *mapPtr = _file->MemoryMapPtr(startOffset); if (mapPtr != NULL) { handle._mem = mapPtr; handle._allocMem = NULL; @@ -143,8 +139,7 @@ ZcPosOccRandRead::readPostingList(const PostingListCounts &counts, size_t padBefore; size_t padAfter; size_t padExtraAfter; // Decode prefetch space - _file.DirectIOPadding(startOffset, vectorLen, - padBefore, padAfter); + _file->DirectIOPadding(startOffset, vectorLen, padBefore, padAfter); padExtraAfter = 0; if (padAfter < 16) padExtraAfter = 16 - padAfter; @@ -153,18 +148,16 @@ ZcPosOccRandRead::readPostingList(const PostingListCounts &counts, void *mallocStart = NULL; void *alignedBuffer = NULL; if (mallocLen > 0) { - alignedBuffer = _file.AllocateDirectIOBuffer(mallocLen, - mallocStart); + alignedBuffer = _file->AllocateDirectIOBuffer(mallocLen, mallocStart); assert(mallocStart != NULL); assert(endOffset + padAfter + padExtraAfter <= _fileSize); - _file.ReadBuf(alignedBuffer, - padBefore + vectorLen + padAfter, - startOffset - padBefore); + _file->ReadBuf(alignedBuffer, + padBefore + vectorLen + padAfter, + startOffset - padBefore); } // Zero decode prefetch memory to avoid uninitialized reads if (padExtraAfter > 0) { - memset(reinterpret_cast<char *>(alignedBuffer) + - padBefore + vectorLen + padAfter, + memset(reinterpret_cast<char *>(alignedBuffer) + padBefore + vectorLen + padAfter, '\0', padExtraAfter); } @@ -181,16 +174,16 @@ ZcPosOccRandRead:: open(const vespalib::string &name, const TuneFileRandRead &tuneFileRead) { if (tuneFileRead.getWantMemoryMap()) { - _file.enableMemoryMap(tuneFileRead.getMemoryMapFlags()); + _file->enableMemoryMap(tuneFileRead.getMemoryMapFlags()); } else if (tuneFileRead.getWantDirectIO()) - _file.EnableDirectIO(); - bool res = _file.OpenReadOnly(name.c_str()); + _file->EnableDirectIO(); + bool res = _file->OpenReadOnly(name.c_str()); if (!res) { LOG(error, "could not open %s: %s", - _file.GetFileName(), getLastErrorString().c_str()); + _file->GetFileName(), getLastErrorString().c_str()); return false; } - _fileSize = _file.GetSize(); + _fileSize = _file->GetSize(); readHeader(); return true; @@ -198,28 +191,28 @@ open(const vespalib::string &name, const TuneFileRandRead &tuneFileRead) bool -ZcPosOccRandRead::close(void) +ZcPosOccRandRead::close() { - _file.Close(); + _file->Close(); return true; } void -ZcPosOccRandRead::readHeader(void) +ZcPosOccRandRead::readHeader() { EGPosOccDecodeContext<true> d(&_fieldsParams); ComprFileReadContext drc(d); - drc.setFile(&_file); - drc.setFileSize(_file.GetSize()); + drc.setFile(_file.get()); + drc.setFileSize(_file->GetSize()); drc.allocComprBuf(512, 32768u); d.emptyBuffer(0); drc.readComprBuffer(); d.setReadContext(&drc); vespalib::FileHeader header; - d.readHeader(header, _file.getSize()); + d.readHeader(header, _file->getSize()); uint32_t headerLen = header.getSize(); assert(header.hasTag("frozen")); assert(header.hasTag("fileBitSize")); @@ -249,14 +242,14 @@ ZcPosOccRandRead::readHeader(void) const vespalib::string & -ZcPosOccRandRead::getIdentifier(void) +ZcPosOccRandRead::getIdentifier() { return myId5; } const vespalib::string & -ZcPosOccRandRead::getSubIdentifier(void) +ZcPosOccRandRead::getSubIdentifier() { PosOccFieldsParams fieldsParams; EGPosOccDecodeContext<true> d(&fieldsParams); @@ -265,7 +258,7 @@ ZcPosOccRandRead::getSubIdentifier(void) Zc4PosOccRandRead:: -Zc4PosOccRandRead(void) +Zc4PosOccRandRead() : ZcPosOccRandRead() { _dynamicK = false; @@ -316,22 +309,21 @@ createIterator(const PostingListCounts &counts, } } - void -Zc4PosOccRandRead::readHeader(void) +Zc4PosOccRandRead::readHeader() { EG2PosOccDecodeContext<true> d(&_fieldsParams); ComprFileReadContext drc(d); - drc.setFile(&_file); - drc.setFileSize(_file.GetSize()); + drc.setFile(_file.get()); + drc.setFileSize(_file->GetSize()); drc.allocComprBuf(512, 32768u); d.emptyBuffer(0); drc.readComprBuffer(); d.setReadContext(&drc); vespalib::FileHeader header; - d.readHeader(header, _file.getSize()); + d.readHeader(header, _file->getSize()); uint32_t headerLen = header.getSize(); assert(header.hasTag("frozen")); assert(header.hasTag("fileBitSize")); @@ -359,23 +351,20 @@ Zc4PosOccRandRead::readHeader(void) _headerBitSize = d.getReadOffset(); } - const vespalib::string & -Zc4PosOccRandRead::getIdentifier(void) +Zc4PosOccRandRead::getIdentifier() { return myId4; } - const vespalib::string & -Zc4PosOccRandRead::getSubIdentifier(void) +Zc4PosOccRandRead::getSubIdentifier() { PosOccFieldsParams fieldsParams; EG2PosOccDecodeContext<true> d(&fieldsParams); return d.getIdentifier(); } - } // namespace diskindex } // namespace search diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.h b/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.h index c86d9a2cd13..bab8b05fb4d 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.h +++ b/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.h @@ -7,16 +7,14 @@ #include <vespa/searchlib/bitcompression/posocccompression.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> -namespace search -{ +namespace search { -namespace diskindex -{ +namespace diskindex { class ZcPosOccRandRead : public index::PostingListFileRandRead { protected: - FastOS_File _file; + std::unique_ptr<FastOS_FileInterface> _file; uint64_t _fileSize; uint32_t _minChunkDocs; // # of documents needed for chunking @@ -31,10 +29,8 @@ protected: public: - ZcPosOccRandRead(void); - - virtual - ~ZcPosOccRandRead(void); + ZcPosOccRandRead(); + ~ZcPosOccRandRead(); typedef index::PostingListCounts PostingListCounts; typedef index::PostingListHandle PostingListHandle; @@ -43,41 +39,26 @@ public: * Create iterator for single word. Semantic lifetime of counts and * handle must exceed lifetime of iterator. */ - virtual search::queryeval::SearchIterator * + search::queryeval::SearchIterator * createIterator(const PostingListCounts &counts, const PostingListHandle &handle, const search::fef::TermFieldMatchDataArray &matchData, - bool usebitVector) const; + bool usebitVector) const override; /** * Read (possibly partial) posting list into handle. */ - virtual void + void readPostingList(const PostingListCounts &counts, uint32_t firstSegment, uint32_t numSegments, - PostingListHandle &handle); - - /** - * Open posting list file for random read. - */ - virtual bool - open(const vespalib::string &name, const TuneFileRandRead &tuneFileRead); + PostingListHandle &handle) override; - /** - * Close posting list file. - */ - virtual bool - close(void); - - virtual void - readHeader(void); - - static const vespalib::string & - getIdentifier(void); - - static const vespalib::string & - getSubIdentifier(void); + bool open(const vespalib::string &name, const TuneFileRandRead &tuneFileRead) override; + bool close() override; + virtual void readHeader(); + static const vespalib::string &getIdentifier(); + static const vespalib::string &getSubIdentifier(); }; class Zc4PosOccRandRead : public ZcPosOccRandRead @@ -89,20 +70,16 @@ public: * Create iterator for single word. Semantic lifetime of counts and * handle must exceed lifetime of iterator. */ - virtual search::queryeval::SearchIterator * + search::queryeval::SearchIterator * createIterator(const PostingListCounts &counts, const PostingListHandle &handle, const search::fef::TermFieldMatchDataArray &matchData, - bool usebitVector) const; - - virtual void - readHeader(void); + bool usebitVector) const override; - static const vespalib::string & - getIdentifier(void); + void readHeader() override; - static const vespalib::string & - getSubIdentifier(void); + static const vespalib::string &getIdentifier(); + static const vespalib::string &getSubIdentifier(); }; diff --git a/searchlib/src/vespa/searchlib/index/dictionaryfile.h b/searchlib/src/vespa/searchlib/index/dictionaryfile.h index db9524d129c..b89fefce354 100644 --- a/searchlib/src/vespa/searchlib/index/dictionaryfile.h +++ b/searchlib/src/vespa/searchlib/index/dictionaryfile.h @@ -5,9 +5,6 @@ #include "postinglisthandle.h" #include "postinglistcountfile.h" #include <vespa/searchlib/common/tunefileinfo.h> -#include <map> -#include <vector> -#include <string> class FastOS_FileInterface; @@ -24,7 +21,6 @@ class DictionaryFileSeqRead : public PostingListCountFileSeqRead { public: DictionaryFileSeqRead() { } - virtual ~DictionaryFileSeqRead(); /** @@ -53,8 +49,7 @@ class DictionaryFileSeqWrite : public PostingListCountFileSeqWrite protected: public: DictionaryFileSeqWrite() { } - - virtual~DictionaryFileSeqWrite(); + virtual ~DictionaryFileSeqWrite(); /** * Write word and counts. Only nonzero counts should be supplied. @@ -73,7 +68,6 @@ protected: bool _memoryMapped; public: DictionaryFileRandRead(); - virtual ~DictionaryFileRandRead(); virtual bool lookup(const vespalib::stringref &word, uint64_t &wordNum, @@ -91,7 +85,7 @@ public: bool getMemoryMapped() const { return _memoryMapped; } - virtual uint64_t getNumWordIds(void) const = 0; + virtual uint64_t getNumWordIds() const = 0; protected: void afterOpen(FastOS_FileInterface &file); }; diff --git a/searchlib/src/vespa/searchlib/index/postinglistcounts.h b/searchlib/src/vespa/searchlib/index/postinglistcounts.h index 564955d8c6b..064907c5f1e 100644 --- a/searchlib/src/vespa/searchlib/index/postinglistcounts.h +++ b/searchlib/src/vespa/searchlib/index/postinglistcounts.h @@ -2,19 +2,13 @@ #pragma once #include <vector> +#include <cstdint> -namespace vespalib -{ - -class nbostream; +namespace vespalib { class nbostream; } -} +namespace search { -namespace search -{ - -namespace index -{ +namespace index { /** * Basic class for holding the result of a dictionary lookup result @@ -40,12 +34,11 @@ public: uint32_t _numDocs; // Number of documents in segment uint32_t _lastDoc; // Last document id in segment - Segment(void) + Segment() : _bitLength(0), _numDocs(0), _lastDoc(0) - { - } + { } bool operator==(const Segment &rhs) const @@ -77,29 +70,24 @@ public: */ std::vector<Segment> _segments; - PostingListCounts(void) + PostingListCounts() : _numDocs(0), _bitLength(0), _segments() - { - } + { } void swap(PostingListCounts & rhs) { std::swap(_numDocs, rhs._numDocs); std::swap(_bitLength, rhs._bitLength); std::swap(_segments, rhs._segments); } - void - clear(void) - { + void clear() { _bitLength = 0; _numDocs = 0; _segments.clear(); } - bool - operator==(const PostingListCounts &rhs) const - { + bool operator==(const PostingListCounts &rhs) const { return (_numDocs == rhs._numDocs && _bitLength == rhs._bitLength && _segments == rhs._segments); @@ -122,20 +110,17 @@ public: uint64_t _accNumDocs; // Used by prefix search for now. PostingListCounts _counts; - PostingListOffsetAndCounts(void) + PostingListOffsetAndCounts() : _offset(0), _accNumDocs(0u), _counts() - { - } + { } friend vespalib::nbostream & - operator<<(vespalib::nbostream &out, - const PostingListOffsetAndCounts &offsetAndCounts); + operator<<(vespalib::nbostream &out, const PostingListOffsetAndCounts &offsetAndCounts); friend vespalib::nbostream & - operator>>(vespalib::nbostream &in, - PostingListOffsetAndCounts &offsetAndCounts); + operator>>(vespalib::nbostream &in, PostingListOffsetAndCounts &offsetAndCounts); }; } // namespace index diff --git a/searchlib/src/vespa/searchlib/index/postinglistfile.h b/searchlib/src/vespa/searchlib/index/postinglistfile.h index 95bb3c39a70..bcf5427a386 100644 --- a/searchlib/src/vespa/searchlib/index/postinglistfile.h +++ b/searchlib/src/vespa/searchlib/index/postinglistfile.h @@ -5,8 +5,6 @@ #include <vespa/searchlib/index/postinglisthandle.h> #include <vespa/searchlib/index/postinglistparams.h> #include <vespa/searchlib/common/tunefileinfo.h> -#include <map> -#include <string> class FastOS_FileInterface; diff --git a/searchlib/src/vespa/searchlib/index/postinglisthandle.cpp b/searchlib/src/vespa/searchlib/index/postinglisthandle.cpp index 97a5a6f3da6..86cdaa7f162 100644 --- a/searchlib/src/vespa/searchlib/index/postinglisthandle.cpp +++ b/searchlib/src/vespa/searchlib/index/postinglisthandle.cpp @@ -1,16 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -LOG_SETUP(".index.postinglisthandle"); #include "postinglisthandle.h" #include <vespa/searchlib/index/postinglistfile.h> -namespace search -{ +namespace search { -namespace index -{ +namespace index { search::queryeval::SearchIterator * PostingListHandle::createIterator(const PostingListCounts &counts, diff --git a/searchlib/src/vespa/searchlib/index/postinglisthandle.h b/searchlib/src/vespa/searchlib/index/postinglisthandle.h index 605904e3912..1b896ac8b36 100644 --- a/searchlib/src/vespa/searchlib/index/postinglisthandle.h +++ b/searchlib/src/vespa/searchlib/index/postinglisthandle.h @@ -2,6 +2,7 @@ #pragma once #include <vespa/searchlib/index/postinglistcounts.h> +#include <memory> namespace search { class BitVector; } namespace search { namespace queryeval { class SearchIterator; } } @@ -34,7 +35,7 @@ public: void *_allocMem; // What to free after posting list size_t _allocSize; // Size of allocated memory - PostingListHandle(void) + PostingListHandle() : _file(NULL), _bitOffset(0), _bitLength(0), @@ -44,10 +45,9 @@ public: _mem(NULL), _allocMem(NULL), _allocSize(0) - { - } + { } - ~PostingListHandle(void) + ~PostingListHandle() { if (_allocMem != NULL) free(_allocMem); @@ -69,9 +69,7 @@ public: /** * Drop value portion of handle. */ - void - drop(void) - { + void drop() { _firstSegment = 0; _numSegments = 0; _bitOffsetMem = 0; diff --git a/searchlib/src/vespa/searchlib/util/comprbuffer.cpp b/searchlib/src/vespa/searchlib/util/comprbuffer.cpp index b76cfc2674d..5d5cd74ae3d 100644 --- a/searchlib/src/vespa/searchlib/util/comprbuffer.cpp +++ b/searchlib/src/vespa/searchlib/util/comprbuffer.cpp @@ -1,16 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 2002-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS -#include <vespa/fastos/fastos.h> -#include <vespa/log/log.h> -#include <vespa/searchlib/util/comprbuffer.h> +#include "comprbuffer.h" #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/fastos/file.h> -LOG_SETUP(".comprbuffer"); - -namespace search -{ +namespace search { using vespalib::nbostream; @@ -19,18 +13,17 @@ ComprBuffer::ComprBuffer(uint32_t unitSize) _comprBufSize(0), _unitSize(unitSize), _comprBufMalloc(NULL) -{ -} +{ } -ComprBuffer::~ComprBuffer(void) +ComprBuffer::~ComprBuffer() { dropComprBuf(); } void -ComprBuffer::dropComprBuf(void) +ComprBuffer::dropComprBuf() { free(_comprBufMalloc); _comprBuf = NULL; @@ -44,17 +37,14 @@ ComprBuffer::allocComprBuf(size_t comprBufSize, FastOS_FileInterface *file, bool padBefore) { - comprBufSize = _aligner.setupAlign(comprBufSize, - _unitSize, - file, - preferredFileAlignment); + comprBufSize = _aligner.setupAlign(comprBufSize, _unitSize, file, preferredFileAlignment); _comprBufSize = comprBufSize; _padBefore = padBefore; allocComprBuf(); } void -ComprBuffer::allocComprBuf(void) +ComprBuffer::allocComprBuf() { dropComprBuf(); /* diff --git a/searchlib/src/vespa/searchlib/util/comprbuffer.h b/searchlib/src/vespa/searchlib/util/comprbuffer.h index 9769f60fedb..5d6927834b9 100644 --- a/searchlib/src/vespa/searchlib/util/comprbuffer.h +++ b/searchlib/src/vespa/searchlib/util/comprbuffer.h @@ -11,13 +11,7 @@ namespace search { class ComprBuffer { private: - ComprBuffer(const ComprBuffer &); - - ComprBuffer & - operator=(const ComprBuffer &); - - void - allocComprBuf(void); + void allocComprBuf(void); public: void *_comprBuf; size_t _comprBufSize; @@ -26,71 +20,44 @@ public: void *_comprBufMalloc; FileAlign _aligner; + ComprBuffer(const ComprBuffer &) = delete; + ComprBuffer &operator=(const ComprBuffer &) = delete; ComprBuffer(uint32_t unitSize); + virtual ~ComprBuffer(); - virtual - ~ComprBuffer(void); - - void - dropComprBuf(void); - - void - allocComprBuf(size_t comprBufSize, - size_t preferredFileAlignment, - FastOS_FileInterface *const file, - bool padbefore); - - static size_t - minimumPadding(void) - { - return 8; - } - - uint32_t - getUnitBitSize(void) const - { - return _unitSize * 8; - } + void dropComprBuf(); - bool - getPadBefore(void) const - { - return _padBefore; - } + void allocComprBuf(size_t comprBufSize, size_t preferredFileAlignment, + FastOS_FileInterface *const file, bool padbefore); - bool - getCheckPointResumed(void) const - { - return _aligner.getCheckPointResumed(); - } + static size_t minimumPadding() { return 8; } + uint32_t getUnitBitSize() const { return _unitSize * 8; } + bool getPadBefore() const { return _padBefore; } + bool getCheckPointResumed() const { return _aligner.getCheckPointResumed(); } /* * When encoding to memory instead of file, the compressed buffer must * be able to grow. */ - void - expandComprBuf(uint32_t overflowUnits); + void expandComprBuf(uint32_t overflowUnits); /* * For unit testing only. Reference data owned by rhs, only works as * long as rhs is live and unchanged. */ - void - referenceComprBuf(const ComprBuffer &rhs); + void referenceComprBuf(const ComprBuffer &rhs); /** * Checkpoint write. Used at semi-regular intervals during indexing * to allow for continued indexing after an interrupt. */ - void - checkPointWrite(vespalib::nbostream &out); + void checkPointWrite(vespalib::nbostream &out); /** * Checkpoint read. Used when resuming indexing after an interrupt. * */ - void - checkPointRead(vespalib::nbostream &in); + void checkPointRead(vespalib::nbostream &in); }; } diff --git a/searchlib/src/vespa/searchlib/util/comprfile.h b/searchlib/src/vespa/searchlib/util/comprfile.h index 29c3af9a945..a30583e2c1b 100644 --- a/searchlib/src/vespa/searchlib/util/comprfile.h +++ b/searchlib/src/vespa/searchlib/util/comprfile.h @@ -1,11 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Copyright (C) 1999-2003 Fast Search & Transfer ASA -// Copyright (C) 2003 Overture Services Norway AS #pragma once #include <utility> -#include <vespa/searchlib/util/filealign.h> #include <vespa/searchlib/util/comprbuffer.h> class FastOS_FileInterface; @@ -19,89 +16,63 @@ class ComprFileWriteContext; class ComprFileDecodeContext { public: - virtual - ~ComprFileDecodeContext(void) - { - } + virtual ~ComprFileDecodeContext() { } /** * * Check if the chunk referenced by the decode context was the * last chunk in the file (e.g. _valE > _realValE) */ - virtual bool - lastChunk(void) const = 0; + virtual bool lastChunk() const = 0; /** * Check if we're at the end of the current chunk (e.g. _valI >= _valE) */ - virtual bool - endOfChunk(void) const = 0; + virtual bool endOfChunk() const = 0; /** * Get remaining units in buffer (e.g. _realValE - _valI) */ - virtual int32_t - remainingUnits(void) const = 0; + virtual int32_t remainingUnits() const = 0; /** * Get unit ptr (e.g. _valI) from decode context. */ - virtual const void * - getUnitPtr(void) const = 0; + virtual const void *getUnitPtr() const = 0; /** * Setup unit buffer in decode context after read. */ - virtual void - afterRead(const void *start, - size_t bufferUnits, - uint64_t bufferEndFilePos, - bool isMore) = 0; + virtual void afterRead(const void *start, size_t bufferUnits, uint64_t bufferEndFilePos, bool isMore) = 0; /** * Setup for bitwise reading. */ - virtual void - setupBits(int bitOffset) = 0; - - virtual uint64_t - getBitPos(int bitOffset, - uint64_t bufferEndFilePos) const = 0; - - virtual uint64_t - getBitPosV(void) const = 0; - - virtual - void skipBits(int bits) = 0; - - virtual void - adjUnitPtr(int newRemainingUnits) = 0; - - virtual void - emptyBuffer(uint64_t newBitPosition) = 0; + virtual void setupBits(int bitOffset) = 0; + virtual uint64_t getBitPos(int bitOffset, uint64_t bufferEndFilePos) const = 0; + virtual uint64_t getBitPosV(void) const = 0; + virtual void skipBits(int bits) = 0; + virtual void adjUnitPtr(int newRemainingUnits) = 0; + virtual void emptyBuffer(uint64_t newBitPosition) = 0; /** * Get size of each unit (typically 4 or 8) */ - virtual uint32_t - getUnitByteSize(void) const = 0; + virtual uint32_t getUnitByteSize(void) const = 0; /** * Checkpoint write. Used at semi-regular intervals during indexing * to allow for continued indexing after an interrupt. Caller must * save position. */ - virtual void - checkPointWrite(vespalib::nbostream &out) = 0; + virtual void checkPointWrite(vespalib::nbostream &out) = 0; /** * Checkpoint read. Used when resuming indexing after an interrupt. * Caller must restore position. */ - virtual void - checkPointRead(vespalib::nbostream &in) = 0; + virtual void checkPointRead(vespalib::nbostream &in) = 0; }; class ComprFileReadBase @@ -145,89 +116,35 @@ private: public: ComprFileReadContext(ComprFileDecodeContext &decodeContext); - ComprFileReadContext(uint32_t unitSize); - - ~ComprFileReadContext(void); - - void - readComprBuffer(uint64_t stopOffset, bool readAll); - - void - readComprBuffer(void); - - void - setPosition(uint64_t newPosition, - uint64_t stopOffset, - bool readAll); - - void - setPosition(uint64_t newPosition); - - void - allocComprBuf(unsigned int comprBufSize, - size_t preferredFileAlignment); - - void - setDecodeContext(ComprFileDecodeContext *decodeContext) - { - _decodeContext = decodeContext; - } - - ComprFileDecodeContext * - getDecodeContext(void) const - { - return _decodeContext; - } - - void - setFile(FastOS_FileInterface *file) - { - _file = file; - } - - FastOS_FileInterface * - getFile(void) const - { - return _file; - } + ~ComprFileReadContext(); + + void readComprBuffer(uint64_t stopOffset, bool readAll); + void readComprBuffer(); + void setPosition(uint64_t newPosition, uint64_t stopOffset, bool readAll); + void setPosition(uint64_t newPosition); + void allocComprBuf(unsigned int comprBufSize, size_t preferredFileAlignment); + void setDecodeContext(ComprFileDecodeContext *decodeContext) { _decodeContext = decodeContext; } + ComprFileDecodeContext *getDecodeContext() const { return _decodeContext; } + void setFile(FastOS_FileInterface *file) { _file = file; } + FastOS_FileInterface *getFile() const { return _file; } /** * Get file offset for end of compressed buffer. */ - uint64_t - getBufferEndFilePos(void) const - { - return _fileReadByteOffset; - } + uint64_t getBufferEndFilePos() const { return _fileReadByteOffset; } /** * Set file offset for end of compressed byffer. */ - void - setBufferEndFilePos(uint64_t bufferEndFilePos) - { - _fileReadByteOffset = bufferEndFilePos; - } - - void - setBitOffset(int bitOffset) - { - _bitOffset = bitOffset; - } - - void - setFileSize(uint64_t fileSize) - { - _fileSize = fileSize; - } + void setBufferEndFilePos(uint64_t bufferEndFilePos) { _fileReadByteOffset = bufferEndFilePos; } + void setBitOffset(int bitOffset) { _bitOffset = bitOffset; } + void setFileSize(uint64_t fileSize) { _fileSize = fileSize; } /* * Set stop offset for sequential read. */ - void - setStopOffset(uint64_t stopOffset, bool readAll) - { + void setStopOffset(uint64_t stopOffset, bool readAll) { _stopOffset = stopOffset; _readAll = readAll; } @@ -236,113 +153,83 @@ public: * For unit testing only. Reference data owned by rhs, only works as * long as rhs is live and unchanged. */ - void - referenceReadContext(const ComprFileReadContext &rhs); + void referenceReadContext(const ComprFileReadContext &rhs); /* * For unit testing only. Copy data owned by rhs. */ - void - copyReadContext(const ComprFileReadContext &rhs); + void copyReadContext(const ComprFileReadContext &rhs); /* * For unit testing only. Reference data owned by rhs, only works as * long as rhs is live and unchanged. */ - void - referenceWriteContext(const ComprFileWriteContext &rhs); + void referenceWriteContext(const ComprFileWriteContext &rhs); /* * For unit testing only. Copy data owned by rhs. */ - void - copyWriteContext(const ComprFileWriteContext &rhs); + void copyWriteContext(const ComprFileWriteContext &rhs); /** * Checkpoint write. Used at semi-regular intervals during indexing * to allow for continued indexing after an interrupt. */ - void - checkPointWrite(vespalib::nbostream &out); + void checkPointWrite(vespalib::nbostream &out); /** * Checkpoint read. Used when resuming indexing after an interrupt. */ - void - checkPointRead(vespalib::nbostream &in); - - bool - getCheckPointOffsetValid(void) const - { - return _checkPointOffsetValid; - } - - uint64_t - getCheckPointOffset(void) const - { - return _checkPointOffset; - } + void checkPointRead(vespalib::nbostream &in); + bool getCheckPointOffsetValid() const { return _checkPointOffsetValid; } + uint64_t getCheckPointOffset() const { return _checkPointOffset; } }; class ComprFileEncodeContext { public: - virtual - ~ComprFileEncodeContext(void) - { - } + virtual ~ComprFileEncodeContext() { } /** * Get number of used units (e.g. _valI - start) */ - virtual int - getUsedUnits(void *start) = 0; + virtual int getUsedUnits(void *start) = 0; /** * Get normal full buffer size (e.g. _valE - start) */ - virtual int - getNormalMaxUnits(void *start) = 0; + virtual int getNormalMaxUnits(void *start) = 0; /** * Adjust buffer after write (e.g. _valI, _fileWriteBias) */ - virtual void - afterWrite(ComprBuffer &cbuf, - uint32_t remainingUnits, - uint64_t bufferStartFilePos) = 0; - + virtual void afterWrite(ComprBuffer &cbuf, uint32_t remainingUnits, uint64_t bufferStartFilePos) = 0; /** * Adjust buffer size to align end of buffer. */ - virtual void - adjustBufSize(ComprBuffer &cbuf) = 0; + virtual void adjustBufSize(ComprBuffer &cbuf) = 0; /** * Get size of each unit (typically 4 or 8) */ - virtual uint32_t - getUnitByteSize(void) const = 0; + virtual uint32_t getUnitByteSize() const = 0; /** * Checkpoint write. Used at semi-regular intervals during indexing * to allow for continued indexing after an interrupt. Caller must * save position, although partial unit is saved. */ - virtual void - checkPointWrite(vespalib::nbostream &out) = 0; + virtual void checkPointWrite(vespalib::nbostream &out) = 0; /** * Checkpoint read. Used when resuming indexing after an interrupt. * Caller must restore positon, although partial unit is restored. */ - virtual void - checkPointRead(vespalib::nbostream &in) = 0; + virtual void checkPointRead(vespalib::nbostream &in) = 0; - virtual uint64_t - getBitPosV(void) const = 0; + virtual uint64_t getBitPosV(void) const = 0; }; class ComprFileWriteBase @@ -355,7 +242,7 @@ public: bool flushSlack); protected: - virtual ~ComprFileWriteBase(void) { } + virtual ~ComprFileWriteBase() { } }; @@ -368,84 +255,43 @@ private: public: ComprFileWriteContext(ComprFileEncodeContext &encodeContext); - ComprFileWriteContext(uint32_t unitSize); + ~ComprFileWriteContext(); - ~ComprFileWriteContext(void); - - void - writeComprBuffer(bool flushSlack); - - void - allocComprBuf(unsigned int comprBufSize, - size_t preferredFileAlignment); - - void - allocComprBuf(void); - - void - setEncodeContext(ComprFileEncodeContext *encodeContext) - { - _encodeContext = encodeContext; - } - - ComprFileEncodeContext * - getEncodeContext(void) const - { - return _encodeContext; - } - - void - setFile(FastOS_FileInterface *file) - { - _file = file; - } - - FastOS_FileInterface * - getFile(void) const - { - return _file; - } + void writeComprBuffer(bool flushSlack); + void allocComprBuf(unsigned int comprBufSize, size_t preferredFileAlignment); + void allocComprBuf(void); + void setEncodeContext(ComprFileEncodeContext *encodeContext) { _encodeContext = encodeContext; } + ComprFileEncodeContext *getEncodeContext(void) const { return _encodeContext; } + void setFile(FastOS_FileInterface *file) { _file = file; } + FastOS_FileInterface *getFile(void) const { return _file; } /** * Get file offset for start of compressed buffer. */ - uint64_t - getBufferStartFilePos(void) const - { - return _fileWriteByteOffset; - } + uint64_t getBufferStartFilePos(void) const { return _fileWriteByteOffset; } /** * Set file offset for start of compressed byffer. */ - void - setBufferStartFilePos(uint64_t bufferStartFilePos) - { - _fileWriteByteOffset = bufferStartFilePos; - } + void setBufferStartFilePos(uint64_t bufferStartFilePos) { _fileWriteByteOffset = bufferStartFilePos; } /** * Grab compressed buffer from write context. This is only legal when * no file is attached. */ - std::pair<void *, size_t> - grabComprBuffer(void *&comprBufMalloc); + std::pair<void *, size_t> grabComprBuffer(void *&comprBufMalloc); /** * Checkpoint write. Used at semi-regular intervals during indexing * to allow for continued indexing after an interrupt. */ - void - checkPointWrite(vespalib::nbostream &out); + void checkPointWrite(vespalib::nbostream &out); /** * Checkpoint read. Used when resuming indexing after an interrupt. */ - void - checkPointRead(vespalib::nbostream &in); + void checkPointRead(vespalib::nbostream &in); }; - } - diff --git a/searchlib/src/vespa/searchlib/util/filealign.cpp b/searchlib/src/vespa/searchlib/util/filealign.cpp index e3a7b85f0f0..105e63b7251 100644 --- a/searchlib/src/vespa/searchlib/util/filealign.cpp +++ b/searchlib/src/vespa/searchlib/util/filealign.cpp @@ -1,11 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/searchlib/util/filealign.h> +#include "filealign.h" #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/fastos/file.h> -namespace search -{ +namespace search { using vespalib::nbostream; @@ -35,7 +34,7 @@ getMinBlocking(size_t elementsize, size_t alignment) } -FileAlign::FileAlign(void) +FileAlign::FileAlign() : _directIOFileAlign(1), _preferredFileAlign(1), _minDirectIOSize(1), @@ -44,13 +43,10 @@ FileAlign::FileAlign(void) _directIOMemAlign(1), _directio(false), _checkPointResumed(false) -{ -} +{ } -FileAlign::~FileAlign(void) -{ -} +FileAlign::~FileAlign() { } size_t @@ -92,10 +88,7 @@ FileAlign::setupAlign(size_t elements, size_t transferMaximum; if (file != NULL) { - _directio = - file->GetDirectIORestrictions(memoryAlignment, - transferGranularity, - transferMaximum); + _directio = file->GetDirectIORestrictions(memoryAlignment, transferGranularity, transferMaximum); } else _directio = false; if (_directio) { diff --git a/searchlib/src/vespa/searchlib/util/filealign.h b/searchlib/src/vespa/searchlib/util/filealign.h index 23586d02eea..7647ba668dd 100644 --- a/searchlib/src/vespa/searchlib/util/filealign.h +++ b/searchlib/src/vespa/searchlib/util/filealign.h @@ -2,6 +2,9 @@ #pragma once +#include <cstddef> +#include <cstdint> + class FastOS_FileInterface; namespace vespalib { class nbostream; } @@ -22,9 +25,8 @@ private: public: - FileAlign(void); - - ~FileAlign(void); + FileAlign(); + ~FileAlign(); /** * Adjust number of bytes for IO (read or write), reducing @@ -36,8 +38,7 @@ public: * * @return adjusted number of bytes for IO */ - size_t - adjustSize(int64_t offset, size_t size); + size_t adjustSize(int64_t offset, size_t size); /** * Adjust number of elements for IO (read or write), reducing @@ -49,8 +50,7 @@ public: * * @return adjusted number of elements for IO */ - size_t - adjustElements(int64_t eoffset, size_t esize); + size_t adjustElements(int64_t eoffset, size_t esize); /** * Setup alignment @@ -62,73 +62,26 @@ public: * * @return adjusted number of elements in buffer */ - size_t - setupAlign(size_t elements, - size_t elemSize, - FastOS_FileInterface *file, - size_t preferredFileAlignment); - - bool - getDirectIO(void) const - { - return _directio; - } - - bool - getCheckPointResumed(void) const - { - return _checkPointResumed; - } - - size_t - getDirectIOFileAlign(void) const - { - return _directIOFileAlign; - } - - size_t - getDirectIOMemAlign(void) const - { - return _directIOMemAlign; - } - - size_t - getMinDirectIOSize(void) const - { - return _minDirectIOSize; - } - - size_t - getMinAlignedSize(void) const - { - return _minAlignedSize; - } - - size_t - getPreferredFileAlign(void) const - { - return _preferredFileAlign; - } - - size_t - getElemSize(void) const - { - return _elemSize; - } + size_t setupAlign(size_t elements, size_t elemSize, FastOS_FileInterface *file, size_t preferredFileAlignment); + bool getDirectIO() const { return _directio; } + bool getCheckPointResumed() const { return _checkPointResumed; } + size_t getDirectIOFileAlign() const { return _directIOFileAlign; } + size_t getDirectIOMemAlign() const { return _directIOMemAlign; } + size_t getMinDirectIOSize() const { return _minDirectIOSize; } + size_t getMinAlignedSize() const { return _minAlignedSize; } + size_t getPreferredFileAlign() const { return _preferredFileAlign; } + size_t getElemSize() const { return _elemSize; } /** * Checkpoint write. Used at semi-regular intervals during indexing * to allow for continued indexing after an interrupt. */ - void - checkPointWrite(vespalib::nbostream &out); + void checkPointWrite(vespalib::nbostream &out); /** * Checkpoint read. Used when resuming indexing after an interrupt. */ - void - checkPointRead(vespalib::nbostream &in); + void checkPointRead(vespalib::nbostream &in); }; } - |