diff options
author | Tor Egge <Tor.Egge@broadpark.no> | 2020-10-16 13:23:25 +0200 |
---|---|---|
committer | Tor Egge <Tor.Egge@broadpark.no> | 2020-10-16 13:51:26 +0200 |
commit | 74328b3e224d5a439f409a9c864cf4c8c3ef2b5f (patch) | |
tree | 8237a1e9c3e035418f000994435b7d309761d6c3 /searchlib | |
parent | 621fae9dd249a0e5e713cf037f0d215adcbdae13 (diff) |
Remove search::StateFile and search::StateBuf.
Diffstat (limited to 'searchlib')
17 files changed, 0 insertions, 1364 deletions
diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index bc2c59d448b..32253fcd2ed 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -229,8 +229,6 @@ vespa_define_module( src/tests/util/bufferwriter src/tests/util/searchable_stats src/tests/util/slime_output_raw_buf_adapter - src/tests/util/statebuf - src/tests/util/statefile src/tests/vespa-fileheader-inspect ) diff --git a/searchlib/src/tests/util/statebuf/.gitignore b/searchlib/src/tests/util/statebuf/.gitignore deleted file mode 100644 index 270347c1d6b..00000000000 --- a/searchlib/src/tests/util/statebuf/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchlib_statebuf_test_app diff --git a/searchlib/src/tests/util/statebuf/CMakeLists.txt b/searchlib/src/tests/util/statebuf/CMakeLists.txt deleted file mode 100644 index 7a7e65429eb..00000000000 --- a/searchlib/src/tests/util/statebuf/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_statebuf_test_app TEST - SOURCES - statebuf_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_statebuf_test_app COMMAND searchlib_statebuf_test_app) diff --git a/searchlib/src/tests/util/statebuf/statebuf_test.cpp b/searchlib/src/tests/util/statebuf/statebuf_test.cpp deleted file mode 100644 index 0b651a8c13c..00000000000 --- a/searchlib/src/tests/util/statebuf/statebuf_test.cpp +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/stllike/string.h> -#include <vespa/searchlib/util/statebuf.h> - -#include <vespa/log/log.h> -LOG_SETUP("statebuf_test"); - -namespace search { - -class Fixture : public StateBuf -{ - char _buf[1024]; - -public: - Fixture() - : StateBuf(_buf, sizeof(_buf)) - { - } -}; - -TEST_F("single character can be appended to stream", Fixture) -{ - f << 'H' << 'e' << 'l' << 'l' << 'o'; - EXPECT_EQUAL("Hello", f.str()); -} - - -TEST_F("strings can be appended to stream", Fixture) -{ - f << "Hello world"; - EXPECT_EQUAL("Hello world", f.str()); -} - -TEST_F("keys can be appended to stream", Fixture) -{ - (f.appendKey("foo") << "fooval").appendKey("bar") << "barval"; - EXPECT_EQUAL("foo=fooval bar=barval", f.str()); -} - - -TEST_F("positive integers can be appended to stream", Fixture) -{ - f << (1ull << 63) << " " << 42l << " " << 21 << " " << 0; - EXPECT_EQUAL("9223372036854775808 42 21 0", f.str()); -} - -TEST_F("negative integers can be appended to stream", Fixture) -{ - f << (1ll << 63) << " " << -42l << " " << -21; - EXPECT_EQUAL("-9223372036854775808 -42 -21", f.str()); -} - -TEST_F("struct timespec can be appended to stream", Fixture) -{ - std::chrono::nanoseconds ts(15*1000000000l + 256); - f << ts; - EXPECT_EQUAL("15.000000256", f.str()); -} - -TEST_F("timestamp can be appended to stream", Fixture) -{ - std::chrono::nanoseconds ts(16*1000000000l + 257); - f.appendTimestamp(ts); - EXPECT_EQUAL("ts=16.000000257", f.str()); -} - - -TEST_F("hexadecimal numbers can be appended to stream", Fixture) -{ - (f.appendHex(0xdeadbeefcafebabeul) << " ").appendHex(0x123456789abcdef0ul); - EXPECT_EQUAL("0xdeadbeefcafebabe 0x123456789abcdef0", f.str()); - -} - -TEST_F("pointer address can be appended to stream", Fixture) -{ - f.appendAddr(nullptr); - f.appendAddr(reinterpret_cast<void *>(0x12345ul)); - EXPECT_EQUAL("addr=0x0000000000000000 addr=0x0000000000012345", f.str()); -} - - -TEST_F("base and size methods can be called on stream", Fixture) -{ - f << "Hello world\n"; - std::string s(f.base(), f.base() + f.size()); - EXPECT_EQUAL("Hello world\n", s); -} - -} - - -TEST_MAIN() -{ - TEST_RUN_ALL(); -} diff --git a/searchlib/src/tests/util/statefile/.gitignore b/searchlib/src/tests/util/statefile/.gitignore deleted file mode 100644 index 504b7431a7a..00000000000 --- a/searchlib/src/tests/util/statefile/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchlib_statefile_test_app diff --git a/searchlib/src/tests/util/statefile/CMakeLists.txt b/searchlib/src/tests/util/statefile/CMakeLists.txt deleted file mode 100644 index 272e6d72b36..00000000000 --- a/searchlib/src/tests/util/statefile/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_statefile_test_app TEST - SOURCES - statefile_test.cpp - DEPENDS - searchlib_test - searchlib -) -vespa_add_test(NAME searchlib_statefile_test_app COMMAND searchlib_statefile_test_app) diff --git a/searchlib/src/tests/util/statefile/statefile_test.cpp b/searchlib/src/tests/util/statefile/statefile_test.cpp deleted file mode 100644 index a9f087c773b..00000000000 --- a/searchlib/src/tests/util/statefile/statefile_test.cpp +++ /dev/null @@ -1,296 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/stllike/string.h> -#include <vespa/searchlib/util/statefile.h> -#include <atomic> -#include <iostream> -#include <fstream> -#include <string> -#include <vespa/searchlib/test/statefile.h> -#include <sys/stat.h> -#include <fcntl.h> -#include <unistd.h> - -#include <vespa/log/log.h> -LOG_SETUP("statefile_test"); - -using namespace search::test::statefile; - -namespace search { - -namespace { - -bool -hasFile(const char *name) -{ - return access(name, R_OK | W_OK) == 0; -} - - -void -addState(StateFile &sf, const char *buf) -{ - size_t bufLen = strlen(buf); - sf.addState(buf, bufLen, false); -} - -void -addSignalState(StateFile &sf, const char *buf) -{ - size_t bufLen = strlen(buf); - sf.addState(buf, bufLen, true); -} - - -bool -assertHistory(std::vector<vespalib::string> &exp, - std::vector<vespalib::string> &act) -{ - if (!EXPECT_EQUAL(exp.size(), act.size())) { - return false; - } - for (size_t i = 0; i < exp.size(); ++i) { - if (!EXPECT_EQUAL(exp[i], act[i])) { - return false; - } - } - return true; -} - - -int64_t -getSize(const char *name) -{ - struct stat stbuf; - if (stat(name, &stbuf) != 0) - return 0; - return stbuf.st_size; -} - - -void -setSize(const char *name, int64_t newSize) -{ - int truncRes = truncate(name, newSize); - (void) truncRes; - assert(truncRes == 0); -} - - -} - - -TEST("Test lock free atomic int used by async signal safe lock primitive") -{ - std::atomic<int> f; - ASSERT_TRUE(f.is_lock_free()); -} - - -TEST("Test that statefile can be created") -{ - StateFile::erase("state"); - EXPECT_FALSE(hasFile("state")); - EXPECT_FALSE(hasFile("state.history")); - StateFile sf("state"); - EXPECT_TRUE(hasFile("state")); - EXPECT_TRUE(hasFile("state.history")); - EXPECT_EQUAL(0, sf.getGen()); - StateFile::erase("state"); - EXPECT_FALSE(hasFile("state")); - EXPECT_FALSE(hasFile("state.history")); - StateFile::erase("state"); - EXPECT_FALSE(hasFile("state")); - EXPECT_FALSE(hasFile("state.history")); -} - - -TEST("Test that statefile can add event") -{ - StateFile::erase("state"); - StateFile sf("state"); - - addState(sf, "Hello world\n"); - vespalib::string check = readState(sf); - EXPECT_EQUAL("Hello world\n", check); - EXPECT_EQUAL(1, sf.getGen()); -} - -TEST("Test that history is appended to") -{ - StateFile::erase("state"); - StateFile sf("state"); - - addState(sf, "Hello world\n"); - addState(sf, "Foo bar\n"); - vespalib::string check = readState(sf); - EXPECT_EQUAL("Foo bar\n", check); - EXPECT_EQUAL(2, sf.getGen()); - { - std::vector<vespalib::string> exp({ "Hello world\n", "Foo bar\n" }); - std::vector<vespalib::string> act(readHistory("state.history")); - TEST_DO(assertHistory(exp, act)); - } -} - - -TEST("Test that truncated history is truncated at event boundary") -{ - StateFile::erase("state"); - int64_t histSize = 1; - { - StateFile sf("state"); - addState(sf, "Hello world\n"); - addState(sf, "Foo bar\n"); - EXPECT_EQUAL(2, sf.getGen()); - histSize = getSize("state.history"); - EXPECT_EQUAL(20, histSize); - addState(sf, "zap\n"); - EXPECT_EQUAL(3, sf.getGen()); - } - // Lose 2 last events in history - setSize("state.history", histSize - 1); - // Last event is restored to history from main state file - StateFile sf("state"); - vespalib::string check = readState(sf); - EXPECT_EQUAL("zap\n", check); - EXPECT_EQUAL(0, sf.getGen()); - { - std::vector<vespalib::string> exp({ "Hello world\n", "zap\n" }); - std::vector<vespalib::string> act(readHistory("state.history")); - TEST_DO(assertHistory(exp, act)); - } -} - - -TEST("Test that async signal safe path adds event") -{ - StateFile::erase("state"); - StateFile sf("state"); - - addSignalState(sf, "Hello world\n"); - addSignalState(sf, "Foo bar\n"); - vespalib::string check = readState(sf); - EXPECT_EQUAL("Foo bar\n", check); - EXPECT_EQUAL(2, sf.getGen()); - { - std::vector<vespalib::string> exp({ "Hello world\n", "Foo bar\n" }); - std::vector<vespalib::string> act(readHistory("state.history")); - TEST_DO(assertHistory(exp, act)); - } -} - - -TEST("Test that state file can be restored from history") -{ - StateFile::erase("state"); - { - StateFile sf("state"); - addState(sf, "Hello world\n"); - addState(sf, "Foo bar\n"); - EXPECT_EQUAL(2, sf.getGen()); - } - // Lose event in main state file - setSize("state", 0); - EXPECT_EQUAL(0, getSize("state")); - // Last event is restored to history from main state file - StateFile sf("state"); - EXPECT_NOT_EQUAL(0, getSize("state")); - vespalib::string check = readState(sf); - EXPECT_EQUAL("Foo bar\n", check); - { - std::vector<vespalib::string> exp({ "Hello world\n", "Foo bar\n" }); - std::vector<vespalib::string> act(readHistory("state.history")); - TEST_DO(assertHistory(exp, act)); - } -} - - -TEST("Test that different entry is added to history") -{ - StateFile::erase("state"); - { - StateFile sf("state"); - addState(sf, "Hello world\n"); - EXPECT_EQUAL(1, sf.getGen()); - } - // Write changed entry to main state file - { - std::ofstream of("state"); - of << "zap\n"; - } - // Add changed event to history - StateFile sf("state"); - EXPECT_NOT_EQUAL(0, getSize("state")); - vespalib::string check = readState(sf); - EXPECT_EQUAL("zap\n", check); - { - std::vector<vespalib::string> exp({ "Hello world\n", "zap\n" }); - std::vector<vespalib::string> act(readHistory("state.history")); - TEST_DO(assertHistory(exp, act)); - } -} - - -TEST("Test that state history stops at NUL byte") -{ - StateFile::erase("state"); - { - StateFile sf("state"); - addState(sf, "Hello world\n"); - addState(sf, "Foo bar\n"); - EXPECT_EQUAL(2, sf.getGen()); - } - // Corrupt history state file - { - char buf[1]; - buf[0] = '\0'; - std::ofstream of("state.history"); - of.write(&buf[0], 1); - } - StateFile sf("state"); - vespalib::string check = readState(sf); - EXPECT_EQUAL("Foo bar\n", check); - { - std::vector<vespalib::string> exp({ "Foo bar\n" }); - std::vector<vespalib::string> act(readHistory("state.history")); - TEST_DO(assertHistory(exp, act)); - } - -} - -TEST("Test that main state stops at NUL byte") -{ - StateFile::erase("state"); - { - StateFile sf("state"); - addState(sf, "Hello world\n"); - addState(sf, "Foo bar\n"); - EXPECT_EQUAL(2, sf.getGen()); - } - // Corrupt history state file - { - char buf[10]; - strcpy(buf, "zap"); - std::ofstream of("state"); - of.write(&buf[0], strlen(buf) + 1); - } - StateFile sf("state"); - vespalib::string check = readState(sf); - EXPECT_EQUAL("Foo bar\n", check); - { - std::vector<vespalib::string> exp({ "Hello world\n", "Foo bar\n" }); - std::vector<vespalib::string> act(readHistory("state.history")); - TEST_DO(assertHistory(exp, act)); - } - -} - -} - -TEST_MAIN() -{ - TEST_RUN_ALL(); - search::StateFile::erase("state"); -} diff --git a/searchlib/src/vespa/searchlib/test/CMakeLists.txt b/searchlib/src/vespa/searchlib/test/CMakeLists.txt index bfa162a7e37..adaaafeda64 100644 --- a/searchlib/src/vespa/searchlib/test/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/test/CMakeLists.txt @@ -8,8 +8,6 @@ vespa_add_library(searchlib_test mock_attribute_context.cpp mock_attribute_manager.cpp searchiteratorverifier.cpp - statefile.cpp - statestring.cpp $<TARGET_OBJECTS:searchlib_test_fakedata> $<TARGET_OBJECTS:searchlib_searchlib_test_diskindex> $<TARGET_OBJECTS:searchlib_test_gtest_migration> diff --git a/searchlib/src/vespa/searchlib/test/statefile.cpp b/searchlib/src/vespa/searchlib/test/statefile.cpp deleted file mode 100644 index f6b1fe39de0..00000000000 --- a/searchlib/src/vespa/searchlib/test/statefile.cpp +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "statefile.h" -#include <vespa/searchlib/util/statefile.h> -#include <iostream> -#include <fstream> -#include <string> - -namespace search::test::statefile { - -vespalib::string -readState(StateFile &sf) -{ - std::vector<char> buf; - sf.readState(buf); - return vespalib::string(buf.begin(), buf.end()); -} - -std::vector<vespalib::string> -readHistory(const char *name) -{ - std::vector<vespalib::string> res; - std::ifstream is(name); - std::string line; - while (!is.eof()) { - std::getline(is, line); - if (is.eof() && line.empty()) { - break; - } - res.push_back(line + "\n"); - } - return res; -} - -} diff --git a/searchlib/src/vespa/searchlib/test/statefile.h b/searchlib/src/vespa/searchlib/test/statefile.h deleted file mode 100644 index 55746f6fe07..00000000000 --- a/searchlib/src/vespa/searchlib/test/statefile.h +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/stllike/string.h> -#include <vector> - -namespace search { - -class StateFile; - -namespace test::statefile { - -vespalib::string readState(StateFile &sf); -std::vector<vespalib::string> readHistory(const char *name); - -} -} diff --git a/searchlib/src/vespa/searchlib/test/statestring.cpp b/searchlib/src/vespa/searchlib/test/statestring.cpp deleted file mode 100644 index 480e6bbc275..00000000000 --- a/searchlib/src/vespa/searchlib/test/statestring.cpp +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "statestring.h" -#include <sstream> - -namespace search::test::statestring { - -bool -testStartPos(vespalib::string &s, size_t pos) -{ - return ((pos >= s.size()) || (pos == 0) || (s[pos - 1] == ' ')); -} - -size_t -findStartPos(vespalib::string &s, const vespalib::string &key) -{ - size_t pos = 0; - while (pos < s.size()) { - pos = s.find(key, pos); - if (testStartPos(s, pos)) { - break; - } - ++pos; - } - return pos; -} - -size_t -scanBreakPos(vespalib::string &s, size_t pos) -{ - while (pos < s.size() && s[pos] != ' ' && s[pos] != '\n') { - ++pos; - } - return pos; -} - -void -normalizeTimestamp(vespalib::string &s) -{ - size_t pos = findStartPos(s, "ts="); - if (pos < s.size()) { - size_t npos = scanBreakPos(s, pos + 3); - s.replace(pos, npos - pos, "ts=0.0"); - return; - } -} - -void -normalizeAddr(vespalib::string &s, void *addr) -{ - size_t pos = findStartPos(s, "addr="); - if (pos < s.size()) { - size_t npos = scanBreakPos(s, pos + 5); - std::ostringstream os; - os << "addr=0x"; - os.width(16); - os.fill('0'); - os << std::hex << reinterpret_cast<unsigned long>(addr); - s.replace(pos, npos - pos, os.str()); - return; - } -} - -void -normalizeTimestamps(std::vector<vespalib::string> &sv) -{ - for (auto &s : sv) { - normalizeTimestamp(s); - } -} - -void -normalizeAddrs(std::vector<vespalib::string> &sv, void *addr) -{ - for (auto &s : sv) { - normalizeAddr(s, addr); - } -} - -} diff --git a/searchlib/src/vespa/searchlib/test/statestring.h b/searchlib/src/vespa/searchlib/test/statestring.h deleted file mode 100644 index 4a6c5d25073..00000000000 --- a/searchlib/src/vespa/searchlib/test/statestring.h +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/stllike/string.h> -#include <vector> - -namespace search::test::statestring { - -void normalizeTimestamp(vespalib::string &s); -void normalizeAddr(vespalib::string &s, void *addr); -void normalizeTimestamps(std::vector<vespalib::string> &sv); -void normalizeAddrs(std::vector<vespalib::string> &sv, void *addr); - -} diff --git a/searchlib/src/vespa/searchlib/util/CMakeLists.txt b/searchlib/src/vespa/searchlib/util/CMakeLists.txt index 26fa8f6c041..320a6480202 100644 --- a/searchlib/src/vespa/searchlib/util/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/util/CMakeLists.txt @@ -16,8 +16,6 @@ vespa_add_library(searchlib_util OBJECT rawbuf.cpp slime_output_raw_buf_adapter.cpp state_explorer_utils.cpp - statebuf.cpp - statefile.cpp stringenum.cpp url.cpp DEPENDS diff --git a/searchlib/src/vespa/searchlib/util/statebuf.cpp b/searchlib/src/vespa/searchlib/util/statebuf.cpp deleted file mode 100644 index d97dccebaf0..00000000000 --- a/searchlib/src/vespa/searchlib/util/statebuf.cpp +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "statebuf.h" - -#include <vespa/log/log.h> - -LOG_SETUP(".searchlib.util.statebuf"); - - -static const char *hexx = "0123456789abcdef"; - -namespace search { - -void -StateBuf::overflow() noexcept -{ - LOG_ABORT("should not be reached"); -} - - - -StateBuf::StateBuf(void *buf, size_t bufLen) noexcept - : _start(static_cast<char *>(buf)), - _cur(static_cast<char *>(buf)), - _end(static_cast<char *>(buf) + bufLen) -{ -} - - -StateBuf & -StateBuf::operator<<(const char *s) noexcept -{ - for (const char *p = s; *p != '\0'; ++p) { - *this << *p; - } - return *this; -} - - -StateBuf & -StateBuf::appendKey(const char *s) noexcept -{ - if (_cur != _start) { - *this << ' '; - } - *this << s << '='; - return *this; -} - - -StateBuf & -StateBuf::operator<<(unsigned long long val) noexcept -{ - char buf[22]; - char *p = buf; - for (; val != 0; ++p) { - *p = '0' + (val % 10); - val /= 10; - } - if (p == buf) { - *this << '0'; - } - while (p != buf) { - --p; - *this << *p; - } - return *this; -} - - -StateBuf & -StateBuf::operator<<(long long val) noexcept -{ - if (val < 0) { - *this << '-' << static_cast<unsigned long long>(- val); - } else { - *this << static_cast<unsigned long long>(val); - } - return *this; -} - - -StateBuf & -StateBuf::operator<<(unsigned long val) noexcept -{ - *this << static_cast<unsigned long long>(val); - return *this; -} - - -StateBuf & -StateBuf::operator<<(long val) noexcept -{ - *this << static_cast<long long>(val); - return *this; -} - - -StateBuf & -StateBuf::operator<<(unsigned int val) noexcept -{ - *this << static_cast<unsigned long long>(val); - return *this; -} - - -StateBuf & -StateBuf::operator<<(int val) noexcept -{ - *this << static_cast<long long>(val); - return *this; -} - - -StateBuf & -StateBuf::appendDecFraction(unsigned long val, unsigned int width) noexcept -{ - char buf[22]; - if (width > sizeof(buf)) { - LOG_ABORT("should not be reached"); - } - char *p = buf; - char *pe = buf + width; - for (; p != pe; ++p) { - *p = '0' + (val % 10); - val /= 10; - } - while (p != buf) { - --p; - *this << *p; - } - return *this; -} - -StateBuf & -StateBuf::appendHex(unsigned long val) noexcept -{ - *this << "0x"; - for (int shft = 64; shft != 0;) { - shft -= 4; - *this << hexx[(val >> shft) & 15]; - } - return *this; -} - - -StateBuf & -StateBuf::operator<<(std::chrono::nanoseconds ns) noexcept -{ - std::chrono::seconds sec = std::chrono::duration_cast<std::chrono::seconds>(ns); - std::chrono::nanoseconds remainder = (ns - sec); - (*this << static_cast<unsigned long>(sec.count()) << '.'). - appendDecFraction(static_cast<unsigned long>(remainder.count()), 9); - return *this; -} - - -StateBuf & -StateBuf::appendTimestamp(std::chrono::nanoseconds ns) noexcept -{ - appendKey("ts") << ns; - return *this; -} - - -StateBuf & -StateBuf::appendTimestamp() noexcept -{ - appendTimestamp(std::chrono::system_clock::now().time_since_epoch()); - return *this; -} - - -StateBuf & -StateBuf::appendAddr(void *addr) noexcept -{ - appendKey("addr"); - appendHex(reinterpret_cast<unsigned long>(addr)); - return *this; -} - - -std::string -StateBuf::str() const -{ - return std::string(_start, _cur); -} - -} diff --git a/searchlib/src/vespa/searchlib/util/statebuf.h b/searchlib/src/vespa/searchlib/util/statebuf.h deleted file mode 100644 index b16783cee64..00000000000 --- a/searchlib/src/vespa/searchlib/util/statebuf.h +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <string> -#include <chrono> - -namespace search { - -/** - * Class used to serialize application state in a mostly safe manner. - * - * Only async signal safe methods can be called, except for unit test - * helper methods (str). - * - */ -class StateBuf -{ - char *_start; - char *_cur; - char *_end; - - static void overflow() noexcept __attribute__((__noinline__, __noreturn__)); - -public: - StateBuf(void *buf, size_t bufLen) noexcept; - - StateBuf & - operator<<(char c) noexcept __attribute__((__always_inline__)) - { - if (__builtin_expect(_cur != _end, true)) { - *_cur++ = c; - return *this; - } - overflow(); - } - - - StateBuf & operator<<(const char *s) noexcept; - StateBuf & appendKey(const char *s) noexcept; - StateBuf & operator<<(std::chrono::nanoseconds ns) noexcept; - StateBuf & appendTimestamp(std::chrono::nanoseconds ns) noexcept; - StateBuf & appendTimestamp() noexcept; - StateBuf & appendAddr(void *addr) noexcept; - StateBuf & operator<<(unsigned long long val) noexcept; - StateBuf & operator<<(long long val) noexcept; - StateBuf & operator<<(unsigned long val) noexcept; - StateBuf & operator<<(long val) noexcept; - StateBuf & operator<<(unsigned int val) noexcept; - StateBuf & operator<<(int val) noexcept; - StateBuf & appendDecFraction(unsigned long val, unsigned int width) noexcept; - StateBuf & appendHex(unsigned long val) noexcept; - size_t size() const noexcept { return _cur - _start; } - const char * base() const noexcept { return _start; } - - /* - * Unit test helper methods. - */ - std::string str() const; -}; - -} diff --git a/searchlib/src/vespa/searchlib/util/statefile.cpp b/searchlib/src/vespa/searchlib/util/statefile.cpp deleted file mode 100644 index cac8acb9c68..00000000000 --- a/searchlib/src/vespa/searchlib/util/statefile.cpp +++ /dev/null @@ -1,463 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "statefile.h" -#include <vespa/vespalib/util/stringfmt.h> -#include <system_error> -#include <mutex> -#include <unistd.h> -#include <fcntl.h> -#include <sys/stat.h> - -#include <vespa/log/log.h> -LOG_SETUP(".searchlib.util.statefile"); - -using Mutex = std::mutex; -using Guard = std::lock_guard<Mutex>; - -namespace search { - -namespace { - -Mutex stateMutex; - -/* - * Assumes that std::atomic implementation is lock free, which it is - * for gcc 4.9.2. Usage is not async signal safe unless the - * implementation is lock free. - */ -std::atomic<int> nestingCount; - -int -myopen(const char *name) noexcept -{ - int fd = open(name, O_CREAT | O_CLOEXEC | O_SYNC | O_RDWR, 0644); - if (fd < 0) { - std::error_code ec(errno, std::system_category()); - fprintf(stderr, - "Could not open %s: %s\n", name, ec.message().c_str()); - LOG_ABORT("should not be reached"); - } - return fd; -} - - -void -myfstat(const char *name, int fd, struct stat &stbuf) noexcept -{ - int fsres = fstat(fd, &stbuf); - if (fsres != 0) { - std::error_code ec(errno, std::system_category()); - fprintf(stderr, "Could not fstat %s: %s\n", name, ec.message().c_str()); - LOG_ABORT("should not be reached"); - } -} - - -void -mypread(const char *name, int fd, void *buf, size_t bufLen, int64_t offset) noexcept -{ - ssize_t rres = pread(fd, buf, bufLen, offset); - if (static_cast<size_t>(rres) != bufLen) { - if (rres >= 0) { - fprintf(stderr, - "Could not read %zu bytes from %s offset %" PRId64 - ": short read (%zd)\n", - bufLen, name, offset, rres); - } else { - std::error_code ec(errno, std::system_category()); - fprintf(stderr, - "Could not read %zu bytes from %s offset %" PRId64 ": %s\n", - bufLen, name, offset, ec.message().c_str()); - } - LOG_ABORT("should not be reached"); - } -} - - -void -mypwrite(const char *name, int fd, const void *buf, size_t bufLen, - int64_t offset) noexcept -{ - ssize_t wres = pwrite(fd, buf, bufLen, offset); - if (static_cast<size_t>(wres) != bufLen) { - if (wres >= 0) { - fprintf(stderr,"Could not write %zu bytes to %s offset %" PRId64 - ": short write (%zd)\n", - bufLen, name, offset, wres); - } else { - std::error_code ec(errno, std::system_category()); - fprintf(stderr, - "Could not write %zu bytes to %s offset %" PRId64 ": %s\n", - bufLen, name, offset, ec.message().c_str()); - } - LOG_ABORT("should not be reached"); - } -} - - -void -myclose(const char *name, int fd) noexcept -{ - int closeres = close(fd); - if (closeres != 0) { - std::error_code ec(errno, std::system_category()); - fprintf(stderr, "Could not close %s: %s\n", - name, ec.message().c_str()); - LOG_ABORT("should not be reached"); - } -} - - -void -myfsync(const char *name, int fd) noexcept -{ - int fsyncres = fsync(fd); - if (fsyncres != 0) { - std::error_code ec(errno, std::system_category()); - fprintf(stderr, "Could not fsync %s: %s\n", - name, ec.message().c_str()); - LOG_ABORT("should not be reached"); - } -} - - -void -myunlink(const char *name) noexcept -{ - int unlinkres = unlink(name); - if (unlinkres != 0 && errno != ENOENT) { - std::error_code ec(errno, std::system_category()); - fprintf(stderr, "Could not unlink %s: %s\n", - name, ec.message().c_str()); - LOG_ABORT("should not be reached"); - } -} - - -/* - * Write string to standard error using only async signal safe methods. - */ -void -mystderr(const char *msg) noexcept -{ - const char *p = msg; - while (*p != '\0') { - ++p; - } - [[maybe_unused]] auto writeRes = write(STDERR_FILENO, msg, static_cast<size_t>(p - msg)); -} - - -/* - * Get async signal safe spinlock. - */ -void -getLock() noexcept -{ - int expzero = 0; - while (!nestingCount.compare_exchange_weak(expzero, 1)) { - expzero = 0; - sleep(1); - } -} - - -/* - * Release async signal safe spinlock - */ -void -releaseLock() noexcept -{ - nestingCount = 0; -} - -class SpinGuard -{ -public: - SpinGuard() noexcept - { - getLock(); - } - - ~SpinGuard() noexcept - { - releaseLock(); - } -}; - -} - - -StateFile::StateFile(const std::string &name) - : _name(nullptr), - _historyName(nullptr), - _gen(0) -{ - _name = strdup(name.c_str()); - std::string historyName = name + ".history"; - _historyName = strdup(historyName.c_str()); - zeroPad(); - fixupHistory(); -} - - -StateFile::~StateFile() -{ - free(_name); - free(_historyName); -} - - -void -StateFile::erase(const std::string &name) -{ - std::string historyName = name + ".history"; - myunlink(name.c_str()); - myunlink(historyName.c_str()); -} - - -void -StateFile::readRawState(std::vector<char> &buf) -{ - struct stat stbuf; - Guard guard(stateMutex); // Serialize states - SpinGuard spinGuard; - int fd = myopen(_name); - myfstat(_name, fd, stbuf); - buf.resize(stbuf.st_size); - mypread(_name, fd, &buf[0], buf.size(), 0); - myclose(_name, fd); -} - - -void -StateFile::trimState(std::vector<char> &buf) -{ - auto newBufEnd = buf.cbegin(); - auto bufEnd = buf.cend(); - for (auto p = buf.cbegin(); p != bufEnd; ++p) { - if (*p == '\n') { // End of state string - newBufEnd = p + 1; - break; // stop scanning after first state - } - if (*p == '\0') { // padding encountered, stop scanning for end - break; - } - } - size_t newStateSize = newBufEnd - buf.cbegin(); - buf.resize(newStateSize); -} - - -void -StateFile::readState(std::vector<char> &buf) -{ - readRawState(buf); - trimState(buf); -} - - -void -StateFile::trimHistory(std::vector<char> &history, const char *name, int hfd, - std::vector<char> &lastHistoryState) -{ - auto historyEnd = history.cend(); - auto prevHistoryEnd = history.cbegin(); - auto newHistoryEnd = history.cbegin(); - for (auto p = history.cbegin(); p != historyEnd; ++p) { - if (*p == '\n') { // End of state string - prevHistoryEnd = newHistoryEnd; - newHistoryEnd = p + 1; - } - if (*p == '\0') { // corruption, stop scanning for end - break; - } - } - std::vector<char> historyEntry(prevHistoryEnd, newHistoryEnd); - size_t newHistSize = newHistoryEnd - history.cbegin(); - if (newHistSize != history.size()) { - int ftruncres = ftruncate(hfd, newHistSize); - if (ftruncres != 0) { - std::error_code ec(errno, std::system_category()); - fprintf(stderr, "Could not truncate %s: %s\n", - name, ec.message().c_str()); - LOG_ABORT("should not be reached"); - } - history.resize(newHistSize); - } - historyEntry.swap(lastHistoryState); -} - -/* - * Fixup history after failed append, e.g. trucated write caused partial - * last state. - */ -void -StateFile::fixupHistory() -{ - struct stat sthbuf; - int hfd = myopen(_historyName); - myfstat(_historyName, hfd, sthbuf); - std::vector<char> history(sthbuf.st_size); - mypread(_historyName, hfd, &history[0], history.size(), 0); - std::vector<char> lastHistory; - trimHistory(history, _historyName, hfd, lastHistory); - std::vector<char> buf; - readState(buf); - if (!buf.empty() && buf != lastHistory) { - mypwrite(_historyName, hfd, &buf[0], buf.size(), history.size()); - myfsync(_historyName, hfd); - } - myclose(_historyName, hfd); - if (buf.empty() && !lastHistory.empty()) { - // Restore state in main state file from last state in history. - int fd = myopen(_name); - mypwrite(_name, fd, &lastHistory[0], lastHistory.size(), 0); - myfsync(_name, fd); - myclose(_name, fd); - } -} - - -void -StateFile::zeroPad() -{ - struct stat stbuf; - int minSize = 4096; - int fd = myopen(_name); - myfstat(_name, fd, stbuf); - std::vector<char> buf(minSize); - if (stbuf.st_size < minSize) { - int padSize = minSize - stbuf.st_size; - mypwrite(_name, fd, &buf[0], padSize, stbuf.st_size); - myfsync(_name, fd); - } - myclose(_name, fd); -} - - -void -StateFile::checkState(const char *buf, size_t bufLen) noexcept -{ - const char *pe = buf + bufLen; - for (const char *p = buf; p < pe; ++p) { - if (*p == '\n') { - if (p != buf + bufLen - 1) { - mystderr("statefile state corrupted: early newline\n"); - LOG_ABORT("should not be reached"); - } - return; - } - if (*p == '\0') { - mystderr("statefile state corrupted: nul byte found\n"); - LOG_ABORT("should not be reached"); - } - } - mystderr("statefile state corrupted: missing newline at end\n"); - LOG_ABORT("should not be reached"); -} - - -void -StateFile::internalAddSignalState(const char *buf, size_t bufLen, - const char *name, - int appendFlag, - const char *openerr, - const char *writeerr, - const char *fsyncerr, - const char *closeerr) noexcept -{ - // Write to main state file, overwriting previous state - int fd = open(name, O_CREAT | O_CLOEXEC | O_SYNC | O_RDWR | appendFlag, - 0644); - if (fd < 0) { - mystderr(openerr); - LOG_ABORT("should not be reached"); - } - ssize_t wres = write(fd, buf, bufLen); - if (static_cast<size_t>(wres) != bufLen) { - mystderr(writeerr); - LOG_ABORT("should not be reached"); - } - int fsyncres = fsync(fd); - if (fsyncres != 0) { - mystderr(fsyncerr); - LOG_ABORT("should not be reached"); - } - int closeres = close(fd); - if (closeres != 0) { - mystderr(closeerr); - LOG_ABORT("should not be reached"); - } -} - -/* - * Write state string to file. State string contains one newline, at the end. - * - * Async signal safe functions used: - * open(), write(), fsync(), close() - * - * Is in signal handler, thus cannot throw exception. - */ -void -StateFile::addSignalState(const char *buf, size_t bufLen) noexcept -{ - checkState(buf, bufLen); - SpinGuard spinGuard; - // Write to main state file, overwriting previous state - internalAddSignalState(buf, bufLen, _name, 0, - "Could not open statefile for read/write\n", - "Error writing to statefile\n", - "Error syncing statefile\n", - "Error closing statefile\n"); - // Write to state file history, appending - internalAddSignalState(buf, bufLen, _historyName, O_APPEND, - "Could not open statefile history for read/write\n", - "Error writing to statefile history\n", - "Error syncing statefile history\n", - "Error closing statefile history\n"); - ++_gen; -} - -/* - * Write state string to file. State string contains one newline, at the end. - */ -void -StateFile::addState(const char *buf, size_t bufLen, bool signal) -{ - if (signal) { - // In signal context, degraded error reporting on state file failures - addSignalState(buf, bufLen); - return; - } - checkState(buf, bufLen); - Guard guard(stateMutex); // Serialize states - SpinGuard spinGuard; - { - // Write to main state file, overwriting previous state - int fd = myopen(_name); - mypwrite(_name, fd, buf, bufLen, 0); - myfsync(_name, fd); - myclose(_name, fd); - } - { - // Write to state file history, appending - int hfd = myopen(_historyName); - struct stat sthbuf; - myfstat(_historyName, hfd, sthbuf); - mypwrite(_historyName, hfd, buf, bufLen, sthbuf.st_size); - myfsync(_historyName, hfd); - myclose(_historyName, hfd); - } - ++_gen; -} - - -int -StateFile::getGen() const -{ - return _gen; -} - -} diff --git a/searchlib/src/vespa/searchlib/util/statefile.h b/searchlib/src/vespa/searchlib/util/statefile.h deleted file mode 100644 index be56990f7e5..00000000000 --- a/searchlib/src/vespa/searchlib/util/statefile.h +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <string> -#include <atomic> -#include <vector> - -namespace search { - -/* - * Class used to store application state in a mostly safe manner. - * - * It maintaines two files, one file with zero-padding at end to store - * last state, and another file with history of states. - * - * State files can not be shared between processes, file locking is not - * async signal safe. - * - * Standalone implementation (doesn't use fastos or vespalib) to - * ensure that we don't trigger callback hooks in fastos. - * - */ -class StateFile -{ - char *_name; - char *_historyName; - std::atomic<int> _gen; - - /* - * Zero pad file, to ensure that a later write won't run out of space. - */ - void zeroPad(); - - /* - * Read state file to buffer in raw form, including padding. - */ - void readRawState(std::vector<char> &buf); - - /* - * Trim padding and everything after state (i.e. stop at first newline). - */ - static void trimState(std::vector<char> &buf); - - /* - * Trim partial state from end of history. - */ - static void trimHistory(std::vector<char> &history, const char *historyName, int hfd, - std::vector<char> &lastHistoryState); - - /* - * Fixup history: trim partial state from end and append current state - * in state file to history if different from last state in history. - * If main state file doesn't have a state but history has a state then - * restore main state from history. - */ - void fixupHistory(); - - /* - * Check that state doesn't contain nul bytes or early newline and - * that it is terminated by a newline at end. - */ - void checkState(const char *buf, size_t bufLen) noexcept; - - void internalAddSignalState(const char *buf, size_t bufLen, const char *name, int appendFlag, - const char *openerr, const char *writeerr, const char *fsyncerr, - const char *closeerr) noexcept; - - void addSignalState(const char *buf, size_t bufLen) noexcept; -public: - StateFile(const std::string &name); - ~StateFile(); - - void addState(const char *buf, size_t bufLen, bool signal); - - static void erase(const std::string &name); - - /* - * Read state file to buffer and trim it down to a state. - */ - void readState(std::vector<char> &buf); - - /* - * Get current state generation (bumped whenever new state is written). - */ - int getGen() const; -}; - -} |