diff options
| author | Henning Baldersheim <balder@yahoo-inc.com> | 2022-05-18 11:05:54 +0000 |
|---|---|---|
| committer | Henning Baldersheim <balder@yahoo-inc.com> | 2022-05-18 11:05:54 +0000 |
| commit | 39443ba7ffe7966fb06555ef832f4eff3756c076 (patch) | |
| tree | 5e0a2fd6ab79aa6be435551ea307be9750e69227 /staging_vespalib | |
| parent | 36df8bd3d9fd4ee60aadd04af89199a8bc504e68 (diff) | |
Move state_server, metrivs and some all executors from staging_vespalib too vespalib.
Diffstat (limited to 'staging_vespalib')
134 files changed, 0 insertions, 9357 deletions
diff --git a/staging_vespalib/CMakeLists.txt b/staging_vespalib/CMakeLists.txt index 04ad5f85b74..6d65c5046df 100644 --- a/staging_vespalib/CMakeLists.txt +++ b/staging_vespalib/CMakeLists.txt @@ -17,13 +17,10 @@ vespa_define_module( src/tests/clock src/tests/crc ${STAGING_VESPALIB_DIRECTIO_TESTDIR} - src/tests/encoding/base64 - src/tests/fileheader src/tests/floatingpointtype src/tests/growablebytebuffer src/tests/json src/tests/memorydatastore - src/tests/metrics src/tests/objectdump src/tests/objects src/tests/objectselection @@ -31,19 +28,12 @@ vespa_define_module( src/tests/programoptions src/tests/rusage src/tests/shutdownguard - src/tests/state_server src/tests/stllike - src/tests/sequencedtaskexecutor - src/tests/singleexecutor ${STAGING_VESPALIB_PROCESS_MEMORY_STATS_TESTDIR} src/tests/xmlserializable LIBS src/vespa/vespalib - src/vespa/vespalib/data - src/vespa/vespalib/encoding - src/vespa/vespalib/metrics - src/vespa/vespalib/net src/vespa/vespalib/objects src/vespa/vespalib/stllike src/vespa/vespalib/util diff --git a/staging_vespalib/src/tests/encoding/.gitignore b/staging_vespalib/src/tests/encoding/.gitignore deleted file mode 100644 index a3e9c375723..00000000000 --- a/staging_vespalib/src/tests/encoding/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -.depend -Makefile -*_test diff --git a/staging_vespalib/src/tests/encoding/base64/.gitignore b/staging_vespalib/src/tests/encoding/base64/.gitignore deleted file mode 100644 index 04f2994a28f..00000000000 --- a/staging_vespalib/src/tests/encoding/base64/.gitignore +++ /dev/null @@ -1 +0,0 @@ -staging_vespalib_base64_test_app diff --git a/staging_vespalib/src/tests/encoding/base64/CMakeLists.txt b/staging_vespalib/src/tests/encoding/base64/CMakeLists.txt deleted file mode 100644 index 2c88ab1dc43..00000000000 --- a/staging_vespalib/src/tests/encoding/base64/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_base64_test_app TEST - SOURCES - base64_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_base64_test_app COMMAND staging_vespalib_base64_test_app) diff --git a/staging_vespalib/src/tests/encoding/base64/base64_test.cpp b/staging_vespalib/src/tests/encoding/base64/base64_test.cpp deleted file mode 100644 index 295aad7ffdd..00000000000 --- a/staging_vespalib/src/tests/encoding/base64/base64_test.cpp +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/encoding/base64.h> -#include <vector> - -using namespace vespalib; - -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("base64_test"); - - // Basic test without padding - std::string source = "No need to pad this string."; - std::string encoded = Base64::encode(source); - std::string expected = "Tm8gbmVlZCB0byBwYWQgdGhpcyBzdHJpbmcu"; - std::string decoded = Base64::decode(encoded); - - EXPECT_EQUAL(expected, encoded); - EXPECT_EQUAL(source, decoded); - - EXPECT_TRUE(static_cast<uint32_t>( - Base64::getMaximumEncodeLength(source.size())) >= encoded.size()); - EXPECT_TRUE(static_cast<uint32_t>( - Base64::getMaximumDecodeLength(encoded.size())) >= source.size()); - - // Basic string that needs padding - source = "This string will need to be padded."; - encoded = Base64::encode(source); - expected = "VGhpcyBzdHJpbmcgd2lsbCBuZWVkIHRvIGJlIHBhZGRlZC4="; - decoded = Base64::decode(encoded); - - EXPECT_EQUAL(expected, encoded); - EXPECT_EQUAL(source, decoded); - - EXPECT_TRUE(static_cast<uint32_t>( - Base64::getMaximumEncodeLength(source.size())) >= encoded.size()); - EXPECT_TRUE(static_cast<uint32_t>( - Base64::getMaximumDecodeLength(encoded.size())) >= source.size()); - - // Check that max sizes are good for whatever input sizes - source = ""; - for (uint32_t i=0; i<100; ++i) { - source += "a"; - // Code will assert if -1 is returned from either - // getMaximumEncodeLength() or getMaximumDecodeLength(). - encoded = Base64::encode(source); - decoded = Base64::decode(encoded); - EXPECT_EQUAL(source, decoded); - } - - // Check that -1 is returned on too little space when encoding - source = "Checking that -1 is returned when not enough space to encode"; - std::vector<char> buffer(100, '\0'); - uint32_t minSizeNeeded = 81; - for (uint32_t i=0; i<minSizeNeeded; ++i) { - EXPECT_EQUAL(-1, Base64::encode(source.c_str(), source.size(), - &buffer[0], i)); - } - EXPECT_EQUAL(80, Base64::encode(source.c_str(), source.size(), - &buffer[0], minSizeNeeded)); - EXPECT_EQUAL(Base64::encode(source), std::string(&buffer[0], 80)); - EXPECT_TRUE(minSizeNeeded <= static_cast<uint32_t>( - Base64::getMaximumEncodeLength(source.size()))); - - EXPECT_TRUE(buffer[80] == '\0'); - - // Check that -1 is returned on too little space when decoding - encoded = Base64::encode(source); - minSizeNeeded = 60; - for (uint32_t i=0; i<minSizeNeeded; ++i) { - EXPECT_EQUAL(-1, Base64::decode(encoded.c_str(), encoded.size(), - &buffer[0], i)); - } - EXPECT_EQUAL(60, Base64::decode(encoded.c_str(), encoded.size(), - &buffer[0], minSizeNeeded)); - EXPECT_EQUAL(source, std::string(&buffer[0], 60)); - - TEST_DONE(); -} diff --git a/staging_vespalib/src/tests/fileheader/.gitignore b/staging_vespalib/src/tests/fileheader/.gitignore deleted file mode 100644 index 909655ba711..00000000000 --- a/staging_vespalib/src/tests/fileheader/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -*.So -.depend* -Makefile -fileheader.tmp -fileheader_test -staging_vespalib_fileheader_test_app diff --git a/staging_vespalib/src/tests/fileheader/CMakeLists.txt b/staging_vespalib/src/tests/fileheader/CMakeLists.txt deleted file mode 100644 index 0312dddb06a..00000000000 --- a/staging_vespalib/src/tests/fileheader/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_fileheader_test_app TEST - SOURCES - fileheader_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_fileheader_test_app COMMAND staging_vespalib_fileheader_test_app) diff --git a/staging_vespalib/src/tests/fileheader/fileheader.dat b/staging_vespalib/src/tests/fileheader/fileheader.dat Binary files differdeleted file mode 100644 index 90660f64b98..00000000000 --- a/staging_vespalib/src/tests/fileheader/fileheader.dat +++ /dev/null diff --git a/staging_vespalib/src/tests/fileheader/fileheader_test.cpp b/staging_vespalib/src/tests/fileheader/fileheader_test.cpp deleted file mode 100644 index 21e374e4f62..00000000000 --- a/staging_vespalib/src/tests/fileheader/fileheader_test.cpp +++ /dev/null @@ -1,694 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/data/fileheader.h> -#include <vespa/vespalib/data/databuffer.h> -#include <vespa/fastos/file.h> - -using namespace vespalib; - -class Test : public vespalib::TestApp { -private: - void testTag(); - void testTagErrors(); - void testTagIteration(); - void testGenericHeader(); - void testBufferReader(); - void testBufferWriter(); - void testBufferAccess(); - void testFileReader(); - void testFileWriter(); - void testFileHeader(); - void testFileAlign(); - void testFileSize(); - void testReadErrors(); - bool testReadError(DataBuffer &buf, const std::string &expected); - void testWriteErrors(); - void testRewriteErrors(); - void testLayout(); - - void testReadSize(bool mapped); - void testReadSizeErrors(bool mapped); - bool testReadSizeError(DataBuffer &buf, const std::string &expected, bool mapped); - -public: - int Main() override { - TEST_INIT("fileheader_test"); - - testTag(); TEST_FLUSH(); - testTagErrors(); TEST_FLUSH(); - testTagIteration(); TEST_FLUSH(); - testGenericHeader(); TEST_FLUSH(); - testBufferReader(); TEST_FLUSH(); - testBufferWriter(); TEST_FLUSH(); - testBufferAccess(); TEST_FLUSH(); - testFileReader(); TEST_FLUSH(); - testFileWriter(); TEST_FLUSH(); - testFileHeader(); TEST_FLUSH(); - testFileAlign(); TEST_FLUSH(); - testFileSize(); TEST_FLUSH(); - testReadErrors(); TEST_FLUSH(); - testWriteErrors(); TEST_FLUSH(); - testRewriteErrors(); TEST_FLUSH(); - testLayout(); TEST_FLUSH(); - testReadSize(false); TEST_FLUSH(); - testReadSizeErrors(false); TEST_FLUSH(); - testReadSize(true); TEST_FLUSH(); - testReadSizeErrors(true); TEST_FLUSH(); - - TEST_DONE(); - } -}; - -TEST_APPHOOK(Test); - -void -Test::testTag() -{ - { - std::vector<GenericHeader::Tag> tags; - tags.push_back(GenericHeader::Tag("foo", 6.9)); - tags.push_back(GenericHeader::Tag("foo", 6.9f)); - for (std::vector<GenericHeader::Tag>::iterator it = tags.begin(); - it != tags.end(); ++it) - { - GenericHeader::Tag tag = *it; - for (uint32_t i = 0; i < 2; ++i) { - EXPECT_EQUAL(GenericHeader::Tag::TYPE_FLOAT, tag.getType()); - EXPECT_EQUAL("foo", tag.getName()); - EXPECT_TRUE(tag.asString().empty()); - EXPECT_APPROX(6.9, tag.asFloat(), 1E-6); - EXPECT_EQUAL(0, tag.asInteger()); - - uint32_t len = tag.getSize(); - DataBuffer buf(len); - EXPECT_EQUAL(len, tag.write(buf)); - - GenericHeader::Tag tmp; - EXPECT_EQUAL(len, tmp.read(buf)); - tag = tmp; - } - } - } - { - std::vector<GenericHeader::Tag> tags; - tags.push_back(GenericHeader::Tag("foo", (int8_t)69)); - tags.push_back(GenericHeader::Tag("foo", (uint8_t)69)); - tags.push_back(GenericHeader::Tag("foo", (int16_t)69)); - tags.push_back(GenericHeader::Tag("foo", (uint16_t)69)); - tags.push_back(GenericHeader::Tag("foo", (int32_t)69)); - tags.push_back(GenericHeader::Tag("foo", (uint32_t)69)); - tags.push_back(GenericHeader::Tag("foo", (int64_t)69)); - for (std::vector<GenericHeader::Tag>::iterator it = tags.begin(); - it != tags.end(); ++it) - { - GenericHeader::Tag tag = *it; - for (uint32_t i = 0; i < 2; ++i) { - EXPECT_EQUAL(GenericHeader::Tag::TYPE_INTEGER, tag.getType()); - EXPECT_EQUAL("foo", tag.getName()); - EXPECT_TRUE(tag.asString().empty()); - EXPECT_EQUAL(0.0, tag.asFloat()); - EXPECT_EQUAL(69l, tag.asInteger()); - - uint32_t len = tag.getSize(); - DataBuffer buf(len); - EXPECT_EQUAL(len, tag.write(buf)); - - GenericHeader::Tag tmp; - EXPECT_EQUAL(len, tmp.read(buf)); - tag = tmp; - } - } - } - { - GenericHeader::Tag tag("foo", "bar"); - for (uint32_t i = 0; i < 2; ++i) { - EXPECT_EQUAL(GenericHeader::Tag::TYPE_STRING, tag.getType()); - EXPECT_EQUAL("foo", tag.getName()); - EXPECT_EQUAL("bar", tag.asString()); - EXPECT_EQUAL(0.0, tag.asFloat()); - EXPECT_EQUAL(0, tag.asInteger()); - - uint32_t len = tag.getSize(); - DataBuffer buf(len); - EXPECT_EQUAL(len, tag.write(buf)); - - GenericHeader::Tag tmp; - EXPECT_EQUAL(len, tmp.read(buf)); - tag = tmp; - } - } - { - GenericHeader::Tag trueTag("foo", true); - GenericHeader::Tag falseTag("foo", false); - EXPECT_EQUAL(GenericHeader::Tag::TYPE_INTEGER, trueTag.getType()); - EXPECT_EQUAL(GenericHeader::Tag::TYPE_INTEGER, falseTag.getType()); - EXPECT_EQUAL(1, trueTag.asInteger()); - EXPECT_EQUAL(0, falseTag.asInteger()); - EXPECT_TRUE(trueTag.asBool()); - EXPECT_FALSE(falseTag.asBool()); - } -} - -void -Test::testTagErrors() -{ - DataBuffer buf(1024); - buf.writeBytes("foo", 3); - buf.writeInt8(0); - buf.writeInt8((uint8_t)GenericHeader::Tag::TYPE_EMPTY); - - GenericHeader::Tag tag("bar", 6.9); - try { - tag.read(buf); - EXPECT_TRUE(false); - } catch (IllegalHeaderException &e) { - EXPECT_EQUAL("Can not deserialize empty tag.", e.getMessage()); - } - EXPECT_EQUAL("bar", tag.getName()); - EXPECT_EQUAL(GenericHeader::Tag::TYPE_FLOAT, tag.getType()); - EXPECT_EQUAL(6.9, tag.asFloat()); -} - -void -Test::testTagIteration() -{ - GenericHeader header; - header.putTag(GenericHeader::Tag("foo", 6.9)); - header.putTag(GenericHeader::Tag("bar", 6699)); - header.putTag(GenericHeader::Tag("baz", "666999")); - - EXPECT_EQUAL(3u, header.getNumTags()); - EXPECT_EQUAL("bar", header.getTag(0).getName()); - EXPECT_EQUAL("baz", header.getTag(1).getName()); - EXPECT_EQUAL("foo", header.getTag(2).getName()); -} - -void -Test::testGenericHeader() -{ - GenericHeader header; - EXPECT_TRUE(header.isEmpty()); - EXPECT_EQUAL(0u, header.getNumTags()); - EXPECT_TRUE(!header.hasTag("foo")); - EXPECT_TRUE(header.getTag("foo").isEmpty()); - EXPECT_TRUE(!header.hasTag("bar")); - EXPECT_TRUE(header.getTag("bar").isEmpty()); - EXPECT_TRUE(!header.hasTag("baz")); - EXPECT_TRUE(header.getTag("baz").isEmpty()); - - header.putTag(GenericHeader::Tag("foo", 6.9)); - EXPECT_TRUE(!header.isEmpty()); - EXPECT_EQUAL(1u, header.getNumTags()); - EXPECT_TRUE(header.hasTag("foo")); - EXPECT_EQUAL(6.9, header.getTag("foo").asFloat()); - EXPECT_TRUE(!header.hasTag("bar")); - EXPECT_TRUE(header.getTag("bar").isEmpty()); - EXPECT_TRUE(!header.hasTag("baz")); - EXPECT_TRUE(header.getTag("baz").isEmpty()); - - header.putTag(GenericHeader::Tag("bar", 6699)); - EXPECT_TRUE(!header.isEmpty()); - EXPECT_EQUAL(2u, header.getNumTags()); - EXPECT_TRUE(header.hasTag("foo")); - EXPECT_EQUAL(6.9, header.getTag("foo").asFloat()); - EXPECT_TRUE(header.hasTag("bar")); - EXPECT_EQUAL(6699, header.getTag("bar").asInteger()); - EXPECT_TRUE(!header.hasTag("baz")); - EXPECT_TRUE(header.getTag("baz").isEmpty()); - - header.putTag(GenericHeader::Tag("baz", "666999")); - EXPECT_TRUE(header.hasTag("foo")); - EXPECT_EQUAL(6.9, header.getTag("foo").asFloat()); - EXPECT_TRUE(header.hasTag("bar")); - EXPECT_EQUAL(6699, header.getTag("bar").asInteger()); - EXPECT_TRUE(header.hasTag("baz")); - EXPECT_EQUAL("666999", header.getTag("baz").asString()); - - header.removeTag("bar"); - EXPECT_TRUE(header.hasTag("foo")); - EXPECT_EQUAL(6.9, header.getTag("foo").asFloat()); - EXPECT_TRUE(!header.hasTag("bar")); - EXPECT_TRUE(header.getTag("bar").isEmpty()); - EXPECT_TRUE(header.hasTag("baz")); - EXPECT_EQUAL("666999", header.getTag("baz").asString()); - - header.removeTag("foo"); - EXPECT_TRUE(!header.hasTag("foo")); - EXPECT_TRUE(header.getTag("foo").isEmpty()); - EXPECT_TRUE(!header.hasTag("bar")); - EXPECT_TRUE(header.getTag("bar").isEmpty()); - EXPECT_TRUE(header.hasTag("baz")); - EXPECT_EQUAL("666999", header.getTag("baz").asString()); - - header.removeTag("baz"); - EXPECT_TRUE(!header.hasTag("foo")); - EXPECT_TRUE(header.getTag("foo").isEmpty()); - EXPECT_TRUE(!header.hasTag("bar")); - EXPECT_TRUE(header.getTag("bar").isEmpty()); - EXPECT_TRUE(!header.hasTag("baz")); - EXPECT_TRUE(header.getTag("baz").isEmpty()); -} - -void -Test::testBufferReader() -{ - DataBuffer src(256); - for (uint32_t i = 0; i < 256; ++i) { - src.writeInt8((uint8_t)i); - } - - GenericHeader::BufferReader reader(src); - - char dst[7]; - uint32_t sum = 0; - while (sum < 256) { - uint32_t len = (uint32_t)reader.getData(dst, 7); - for (uint32_t i = 0; i < len; ++i) { - EXPECT_EQUAL(sum + i, (uint8_t)dst[i]); - } - sum += len; - } - EXPECT_EQUAL(256u, sum); -} - -void -Test::testBufferWriter() -{ - DataBuffer dst(256); - GenericHeader::BufferWriter writer(dst); - - uint32_t sum = 0; - while(sum < 256) { - char src[7]; - for (uint32_t i = 0; i < 7; ++i) { - src[i] = (uint8_t)(sum + i); - } - uint32_t len = std::min(7u, 256 - sum); - EXPECT_EQUAL(len, (uint32_t)writer.putData(src, len)); - sum += len; - } - EXPECT_EQUAL(256u, sum); - - // flip dst - for (uint32_t i = 0; i < 256; ++i) { - uint8_t b = dst.readInt8(); - EXPECT_EQUAL(i, (uint32_t)b); - } -} - -void -Test::testBufferAccess() -{ - DataBuffer buf; - uint32_t len; - { - GenericHeader header; - header.putTag(GenericHeader::Tag("foo", 6.9)); - header.putTag(GenericHeader::Tag("bar", 6699)); - header.putTag(GenericHeader::Tag("baz", "666999")); - - int64_t bval = 0x1234567890abcdefLL; - header.putTag(GenericHeader::Tag("big", bval)); - - len = header.getSize(); - buf.ensureFree(len); - GenericHeader::BufferWriter writer(buf); - EXPECT_EQUAL(len, header.write(writer)); - } - { - GenericHeader header; - GenericHeader::BufferReader reader(buf); - EXPECT_EQUAL(len, header.read(reader)); - - EXPECT_TRUE(header.hasTag("foo")); - EXPECT_EQUAL(6.9, header.getTag("foo").asFloat()); - EXPECT_TRUE(header.hasTag("bar")); - EXPECT_EQUAL(6699, header.getTag("bar").asInteger()); - EXPECT_TRUE(header.hasTag("baz")); - EXPECT_EQUAL("666999", header.getTag("baz").asString()); - EXPECT_TRUE(header.hasTag("big")); - EXPECT_EQUAL(0x1234567890abcdefLL, header.getTag("big").asInteger()); - } -} - -void -Test::testFileReader() -{ - { - FastOS_File file; - ASSERT_TRUE(file.OpenWriteOnlyTruncate("fileheader.tmp")); - - uint8_t buf[256]; - for (uint32_t i = 0; i < 256; ++i) { - buf[i] = (uint8_t)i; - } - EXPECT_EQUAL(256, file.Write2(buf, 256)); - } - { - FastOS_File file; - ASSERT_TRUE(file.OpenReadOnly("fileheader.tmp")); - FileHeader::FileReader reader(file); - - char buf[7]; - uint32_t sum = 0; - while(sum < 256) { - uint32_t len = (uint32_t)reader.getData(buf, 7); - for (uint32_t i = 0; i < len; ++i) { - EXPECT_EQUAL(sum + i, (uint8_t)buf[i]); - } - sum += len; - } - EXPECT_EQUAL(256u, sum); - - ASSERT_TRUE(file.Close()); - file.Delete(); - } -} - -void -Test::testFileWriter() -{ - { - FastOS_File file; - ASSERT_TRUE(file.OpenWriteOnlyTruncate("fileheader.tmp")); - FileHeader::FileWriter writer(file); - - uint32_t sum = 0; - while(sum < 256) { - char src[7]; - for (uint32_t i = 0; i < 7; ++i) { - src[i] = (uint8_t)(sum + i); - } - uint32_t len = std::min(7u, 256 - sum); - EXPECT_EQUAL(len, (uint32_t)writer.putData(src, len)); - sum += len; - } - EXPECT_EQUAL(256u, sum); - } - { - FastOS_File file; - ASSERT_TRUE(file.OpenReadOnly("fileheader.tmp")); - - uint8_t buf[256]; - EXPECT_EQUAL(256, file.Read(buf, 256)); - for (uint32_t i = 0; i < 256; ++i) { - EXPECT_EQUAL(i, (uint32_t)buf[i]); - } - - ASSERT_TRUE(file.Close()); - file.Delete(); - } -} - -void -Test::testFileHeader() -{ - uint32_t len = 0; - { - FileHeader header; - header.putTag(FileHeader::Tag("foo", 6.9)); - header.putTag(FileHeader::Tag("bar", 6699)); - header.putTag(FileHeader::Tag("baz", "666999")); - - FastOS_File file; - ASSERT_TRUE(file.OpenWriteOnlyTruncate("fileheader.tmp")); - len = header.writeFile(file); - EXPECT_EQUAL(len, header.getSize()); - } - { - FastOS_File file; - ASSERT_TRUE(file.OpenReadWrite("fileheader.tmp")); - - FileHeader header; - EXPECT_EQUAL(len, header.readFile(file)); - EXPECT_EQUAL(len, header.getSize()); - - EXPECT_TRUE(header.hasTag("foo")); - EXPECT_EQUAL(6.9, header.getTag("foo").asFloat()); - EXPECT_TRUE(header.hasTag("bar")); - EXPECT_EQUAL(6699, header.getTag("bar").asInteger()); - EXPECT_TRUE(header.hasTag("baz")); - EXPECT_EQUAL("666999", header.getTag("baz").asString()); - - header.putTag(FileHeader::Tag("foo", 9.6)); - header.putTag(FileHeader::Tag("bar", 9966)); - header.putTag(FileHeader::Tag("baz", "999666")); - EXPECT_EQUAL(len, header.getSize()); - EXPECT_EQUAL(len, header.rewriteFile(file)); - } - { - FileHeader header; - - FastOS_File file; - ASSERT_TRUE(file.OpenReadOnly("fileheader.tmp")); - EXPECT_EQUAL(len, header.readFile(file)); - EXPECT_EQUAL(len, header.getSize()); - ASSERT_TRUE(file.Close()); - file.Delete(); - - EXPECT_TRUE(header.hasTag("foo")); - EXPECT_EQUAL(9.6, header.getTag("foo").asFloat()); - EXPECT_TRUE(header.hasTag("bar")); - EXPECT_EQUAL(9966, header.getTag("bar").asInteger()); - EXPECT_TRUE(header.hasTag("baz")); - EXPECT_EQUAL("999666", header.getTag("baz").asString()); - } -} - -void -Test::testFileAlign() -{ - for (uint32_t alignTo = 1; alignTo < 16; ++alignTo) { - FileHeader header(alignTo); - header.putTag(FileHeader::Tag("foo", "bar")); - EXPECT_EQUAL(0u, header.getSize() % alignTo); - } -} - -void -Test::testFileSize() -{ - for (uint32_t minSize = 0; minSize < 512; ++minSize) { - FileHeader header(1u, minSize); - header.putTag(FileHeader::Tag("foo", "bar")); - EXPECT_TRUE(header.getSize() >= minSize); - } -} - -void -Test::testReadErrors() -{ - { - DataBuffer buf; - EXPECT_TRUE(testReadError(buf, "Failed to read header info.")); - } - { - DataBuffer buf; - buf.writeInt32(0xDEADBEAF); - buf.writeInt32(8); - EXPECT_TRUE(testReadError(buf, "Failed to verify magic bits.")); - } - { - DataBuffer buf; - buf.writeInt32(GenericHeader::MAGIC); - buf.writeInt32(8); - EXPECT_TRUE(testReadError(buf, "Failed to verify header size.")); - } - { - DataBuffer buf; - buf.writeInt32(GenericHeader::MAGIC); - buf.writeInt32(16); - buf.writeInt32(-1); - buf.writeInt32(0); - EXPECT_TRUE(testReadError(buf, "Failed to verify header version.")); - } - { - DataBuffer buf; - buf.writeInt32(GenericHeader::MAGIC); - buf.writeInt32(21); - buf.writeInt32(GenericHeader::VERSION); - buf.writeInt32(1); - buf.writeBytes("foo", 3); - buf.writeInt8(0); - buf.writeInt8((uint8_t)GenericHeader::Tag::TYPE_EMPTY); - EXPECT_TRUE(testReadError(buf, "Can not deserialize empty tag.")); - } -} - -bool -Test::testReadError(DataBuffer &buf, const std::string &expected) -{ - GenericHeader header; - header.putTag(GenericHeader::Tag("foo", "bar")); - try { - GenericHeader::BufferReader reader(buf); - header.read(reader); - EXPECT_TRUE(false); - return false; - } catch (IllegalHeaderException &e) { - if (!EXPECT_EQUAL(expected, e.getMessage())) { - return false; - } - } - if (!EXPECT_EQUAL(1u, header.getNumTags())) { - return false; - } - if (!EXPECT_EQUAL("bar", header.getTag("foo").asString())) { - return false; - } - return true; -} - -void -Test::testWriteErrors() -{ - GenericHeader header; - header.putTag(GenericHeader::Tag("foo", 69)); - - DataBuffer buf; - buf.ensureFree(4); - buf.moveFreeToData(buf.getFreeLen() - 4); - EXPECT_TRUE(header.getSize() > buf.getFreeLen()); - try { - GenericHeader::BufferWriter writer(buf); - header.write(writer); - EXPECT_TRUE(false); - } catch (IllegalHeaderException &e) { - EXPECT_EQUAL("Failed to write header.", e.getMessage()); - } - - EXPECT_TRUE(header.hasTag("foo")); - EXPECT_EQUAL(69, header.getTag("foo").asInteger()); -} - -void -Test::testRewriteErrors() -{ - FileHeader header; - header.putTag(FileHeader::Tag("foo", "bar")); - uint32_t len = header.getSize(); - - { - FastOS_File file; - ASSERT_TRUE(file.OpenWriteOnlyTruncate("fileheader.tmp")); - EXPECT_EQUAL(len, header.writeFile(file)); - } - { - FastOS_File file; - ASSERT_TRUE(file.OpenReadWrite("fileheader.tmp")); - header.putTag(FileHeader::Tag("baz", "cox")); - EXPECT_TRUE(len != header.getSize()); - try { - header.rewriteFile(file); - EXPECT_TRUE(false); - } catch (IllegalHeaderException &e) { - EXPECT_EQUAL("Failed to rewrite resized header.", e.getMessage()); - } - } -} - -void -Test::testLayout() -{ - FileHeader header; - { - FastOS_File file; - const std::string fileName = TEST_PATH("fileheader.dat"); - ASSERT_TRUE(file.OpenReadOnly(fileName.c_str())); - uint32_t len = header.readFile(file); - EXPECT_EQUAL(len, header.getSize()); - } - - EXPECT_TRUE(header.hasTag("foo")); - EXPECT_EQUAL(6.9, header.getTag("foo").asFloat()); - EXPECT_TRUE(header.hasTag("bar")); - EXPECT_EQUAL(6699, header.getTag("bar").asInteger()); - EXPECT_TRUE(header.hasTag("baz")); - EXPECT_EQUAL("666999", header.getTag("baz").asString()); -} - - -void -Test::testReadSize(bool mapped) -{ - DataBuffer buf; - buf.writeInt32(GenericHeader::MAGIC); - buf.writeInt32(21); - buf.writeInt32(GenericHeader::VERSION); - buf.writeInt32(1); - uint32_t headerLen; - if (mapped) { - GenericHeader::MMapReader reader(buf.getData(), buf.getDataLen()); - headerLen = FileHeader::readSize(reader); - } else { - GenericHeader::BufferReader reader(buf); - headerLen = FileHeader::readSize(reader); - } - EXPECT_EQUAL(21u, headerLen); -} - - -void -Test::testReadSizeErrors(bool mapped) -{ - { - DataBuffer buf; - EXPECT_TRUE(testReadSizeError(buf, "Failed to read header info.", - mapped)); - } - { - DataBuffer buf; - buf.writeInt32(0xDEADBEAF); - buf.writeInt32(8); - buf.writeInt32(0); - buf.writeInt32(0); - EXPECT_TRUE(testReadSizeError(buf, "Failed to verify magic bits.", - mapped)); - } - { - DataBuffer buf; - buf.writeInt32(GenericHeader::MAGIC); - buf.writeInt32(8); - buf.writeInt32(GenericHeader::VERSION); - buf.writeInt32(0); - EXPECT_TRUE(testReadSizeError(buf, "Failed to verify header size.", - mapped)); - } - { - DataBuffer buf; - buf.writeInt32(GenericHeader::MAGIC); - buf.writeInt32(16); - buf.writeInt32(-1); - buf.writeInt32(0); - EXPECT_TRUE(testReadSizeError(buf, - "Failed to verify header version.", - mapped)); - } -} - - -bool -Test::testReadSizeError(DataBuffer &buf, const std::string &expected, - bool mapped) -{ - uint32_t headerLen = 0u; - try { - if (mapped) { - GenericHeader::MMapReader reader(buf.getData(), buf.getDataLen()); - headerLen = FileHeader::readSize(reader); - } else { - GenericHeader::BufferReader reader(buf); - headerLen = FileHeader::readSize(reader); - } - EXPECT_TRUE(false); - return false; - } catch (IllegalHeaderException &e) { - if (!EXPECT_EQUAL(expected, e.getMessage())) { - return false; - } - } - EXPECT_EQUAL(headerLen, 0u); - return true; -} - diff --git a/staging_vespalib/src/tests/health_server/.gitignore b/staging_vespalib/src/tests/health_server/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/staging_vespalib/src/tests/health_server/.gitignore +++ /dev/null diff --git a/staging_vespalib/src/tests/metrics/CMakeLists.txt b/staging_vespalib/src/tests/metrics/CMakeLists.txt deleted file mode 100644 index ae5462c28ea..00000000000 --- a/staging_vespalib/src/tests/metrics/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_metrics_test_app TEST - SOURCES - simple_metrics_test.cpp - mock_tick.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_metrics_test_app COMMAND staging_vespalib_metrics_test_app) - -vespa_add_executable(staging_vespalib_stablestore_test_app TEST - SOURCES - stable_store_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_stablestore_test_app COMMAND staging_vespalib_stablestore_test_app) diff --git a/staging_vespalib/src/tests/metrics/mock_tick.cpp b/staging_vespalib/src/tests/metrics/mock_tick.cpp deleted file mode 100644 index c16ef25cfe6..00000000000 --- a/staging_vespalib/src/tests/metrics/mock_tick.cpp +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "mock_tick.h" - -namespace vespalib::metrics { - -} // namespace vespalib::metrics diff --git a/staging_vespalib/src/tests/metrics/mock_tick.h b/staging_vespalib/src/tests/metrics/mock_tick.h deleted file mode 100644 index 4d9f6758537..00000000000 --- a/staging_vespalib/src/tests/metrics/mock_tick.h +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <atomic> -#include <condition_variable> -#include <mutex> -#include <vespa/vespalib/metrics/clock.h> -#include <vespa/vespalib/testkit/test_kit.h> - -namespace vespalib::metrics { - -// used to test clients of the Tick interface -// values shared between threads are bounded queues with max size 1 -class MockTick : public Tick { -private: - using Guard = std::unique_lock<std::mutex>; - struct Value { - Value() noexcept : value(0.0), valid(false) {} - TimeStamp value; - bool valid; - }; - - TimeStamp _first_value; - std::mutex _lock; - std::condition_variable _cond; - bool _alive; - Value _prev; - Value _next; - - void push(Value &dst, TimeStamp value) { - Guard guard(_lock); - while (_alive && dst.valid) { - _cond.wait(guard); - } - dst.value = value; - dst.valid = true; - _cond.notify_one(); - } - - TimeStamp pop(Value &src) { - Guard guard(_lock); - while (_alive && !src.valid) { - _cond.wait(guard); - } - src.valid = false; - _cond.notify_one(); - return src.value; - } - - TimeStamp peek(const Value &src) { - Guard guard(_lock); - while (_alive && !src.valid) { - _cond.wait(guard); - } - return src.value; - } - -public: - explicit MockTick(TimeStamp first_value) noexcept - : _first_value(first_value), _lock(), _cond(), _alive(true), _prev(), _next() {} - TimeStamp first() override { return _first_value; } - TimeStamp next(TimeStamp prev) override { - push(_prev, prev); - return pop(_next); - } - TimeStamp give(TimeStamp next_value) { - TimeStamp prev_value = pop(_prev); - push(_next, next_value); - EXPECT_EQUAL(peek(_prev).count(), next_value.count()); - return prev_value; - } - bool alive() const override { return _alive; } - void kill() override { - Guard guard(_lock); - _alive = false; - _cond.notify_all(); - } -}; - -// share the MockTick between the tested and the tester. -class TickProxy : public Tick { -private: - std::shared_ptr<Tick> _tick; -public: - explicit TickProxy(std::shared_ptr<Tick> tick) noexcept : _tick(std::move(tick)) {} - TimeStamp first() override { return _tick->first(); } - TimeStamp next(TimeStamp prev) override { return _tick->next(prev); } - bool alive() const override { return _tick->alive(); } - void kill() override { _tick->kill(); } -}; - -} // namespace vespalib::metrics diff --git a/staging_vespalib/src/tests/metrics/simple_metrics_test.cpp b/staging_vespalib/src/tests/metrics/simple_metrics_test.cpp deleted file mode 100644 index 3006022a43d..00000000000 --- a/staging_vespalib/src/tests/metrics/simple_metrics_test.cpp +++ /dev/null @@ -1,219 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/data/slime/slime.h> -#include <vespa/vespalib/data/slime/json_format.h> -#include <vespa/vespalib/metrics/simple_metrics.h> -#include <vespa/vespalib/metrics/simple_metrics_manager.h> -#include <vespa/vespalib/metrics/stable_store.h> -#include <vespa/vespalib/metrics/json_formatter.h> -#include "mock_tick.h" -#include <stdio.h> -#include <unistd.h> - -using namespace vespalib; -using namespace vespalib::metrics; - -TEST("require that simple metrics gauge merge works") -{ - std::pair<MetricId, Point> id(MetricId(42), Point(17)); - Gauge::Measurement a1(id, 0.0); - Gauge::Measurement b1(id, 7.0); - Gauge::Measurement b2(id, 9.0); - Gauge::Measurement b3(id, 8.0); - Gauge::Measurement c1(id, 10.0); - Gauge::Measurement c2(id, 1.0); - - GaugeAggregator a(a1), b(b1), c(c1); - b.merge(b2); - b.merge(b3); - c.merge(c2); - - EXPECT_EQUAL(a.observedCount, 1u); - EXPECT_EQUAL(a.sumValue, 0.0); - EXPECT_EQUAL(a.minValue, 0.0); - EXPECT_EQUAL(a.maxValue, 0.0); - EXPECT_EQUAL(a.lastValue, 0.0); - - EXPECT_EQUAL(b.observedCount, 3u); - EXPECT_EQUAL(b.sumValue, 24.0); - EXPECT_EQUAL(b.minValue, 7.0); - EXPECT_EQUAL(b.maxValue, 9.0); - EXPECT_EQUAL(b.lastValue, 8.0); - - EXPECT_EQUAL(c.observedCount, 2u); - EXPECT_EQUAL(c.sumValue, 11.0); - EXPECT_EQUAL(c.minValue, 1.0); - EXPECT_EQUAL(c.maxValue, 10.0); - EXPECT_EQUAL(c.lastValue, 1.0); - - a.minValue = 8; - - a.merge(b); - EXPECT_EQUAL(a.observedCount, 4u); - EXPECT_EQUAL(a.sumValue, 24.0); - EXPECT_EQUAL(a.minValue, 7.0); - EXPECT_EQUAL(a.maxValue, 9.0); - EXPECT_EQUAL(a.lastValue, 8.0); - - a.merge(b); - EXPECT_EQUAL(a.observedCount, 7u); - EXPECT_EQUAL(a.sumValue, 48.0); - EXPECT_EQUAL(a.minValue, 7.0); - EXPECT_EQUAL(a.maxValue, 9.0); - EXPECT_EQUAL(a.lastValue, 8.0); - - a.merge(c); - EXPECT_EQUAL(a.observedCount, 9u); - EXPECT_EQUAL(a.sumValue, 59.0); - EXPECT_EQUAL(a.minValue, 1.0); - EXPECT_EQUAL(a.maxValue, 10.0); - EXPECT_EQUAL(a.lastValue, 1.0); -} - -bool compare_json(const vespalib::string &a, const vespalib::string &b) -{ - using vespalib::Memory; - using vespalib::slime::JsonFormat; - - Slime slimeA, slimeB; - if (! JsonFormat::decode(a, slimeA)) { -fprintf(stderr, "bad json a:\n>>>%s\n<<<\n", a.c_str()); - return false; - } - if (! JsonFormat::decode(b, slimeB)) { -fprintf(stderr, "bad json b\n"); - return false; - } - if (!(slimeA == slimeB)) { -fprintf(stderr, "compares unequal:\n[A]\n%s\n[B]\n%s\n", a.c_str(), b.c_str()); - } - return slimeA == slimeB; -} - -void check_json(const vespalib::string &actual) -{ - vespalib::string expect = "{" - " snapshot: { from: 1, to: 4 }," - " values: [ { name: 'foo'," - " values: { count: 17, rate: 4.85714 }" - " }, {" - " name: 'foo'," - " dimensions: { chain: 'default', documenttype: 'music', thread: '0' }," - " values: { count: 4, rate: 1.14286 }" - " }, {" - " name: 'bar'," - " values: { count: 4, rate: 1.14286, average: 42, sum: 168, min: 41, max: 43, last: 42 }" - " }, {" - " name: 'bar'," - " dimensions: { chain: 'vespa', documenttype: 'blogpost', thread: '1' }," - " values: { count: 1, rate: 0.285714, average: 14, sum: 14, min: 14, max: 14, last: 14 }" - " }, {" - " name: 'bar'," - " dimensions: { chain: 'vespa', documenttype: 'blogpost', thread: '2' }," - " values: { count: 1, rate: 0.285714, average: 11, sum: 11, min: 11, max: 11, last: 11 }" - " } ]" - "}"; - EXPECT_TRUE(compare_json(expect, actual)); -} - - -TEST("use simple_metrics_collector") -{ - using namespace vespalib::metrics; - SimpleManagerConfig cf; - cf.sliding_window_seconds = 5; - std::shared_ptr<MockTick> ticker = std::make_shared<MockTick>(TimeStamp(1.0)); - auto manager = SimpleMetricsManager::createForTest(cf, std::make_unique<TickProxy>(ticker)); - - Counter myCounter = manager->counter("foo", "no description"); - myCounter.add(); - myCounter.add(16); - - Gauge myGauge = manager->gauge("bar", "dummy description"); - myGauge.sample(42.0); - myGauge.sample(41.0); - myGauge.sample(43.0); - myGauge.sample(42.0); - - EXPECT_EQUAL(1.0, ticker->give(TimeStamp(2.0)).count()); - - Snapshot snap1 = manager->snapshot(); - EXPECT_EQUAL(1.0, snap1.startTime()); - EXPECT_EQUAL(2.0, snap1.endTime()); - - EXPECT_EQUAL(1u, snap1.counters().size()); - EXPECT_EQUAL("foo", snap1.counters()[0].name()); - EXPECT_EQUAL(17u, snap1.counters()[0].count()); - - EXPECT_EQUAL(1u, snap1.gauges().size()); - EXPECT_EQUAL("bar", snap1.gauges()[0].name()); - EXPECT_EQUAL(4u, snap1.gauges()[0].observedCount()); - EXPECT_EQUAL(41.0, snap1.gauges()[0].minValue()); - EXPECT_EQUAL(43.0, snap1.gauges()[0].maxValue()); - EXPECT_EQUAL(42.0, snap1.gauges()[0].lastValue()); - - Point one = manager->pointBuilder() - .bind("chain", "default") - .bind("documenttype", "music") - .bind("thread", "0").build(); - PointBuilder b2 = manager->pointBuilder(); - b2.bind("chain", "vespa") - .bind("documenttype", "blogpost"); - b2.bind("thread", "1"); - Point two = b2.build(); - EXPECT_EQUAL(one.id(), 1u); - EXPECT_EQUAL(two.id(), 2u); - - Point anotherOne = manager->pointBuilder() - .bind("chain", "default") - .bind("documenttype", "music") - .bind("thread", "0"); - EXPECT_EQUAL(anotherOne.id(), 1u); - - Point three = manager->pointBuilder(two).bind("thread", "2"); - EXPECT_EQUAL(three.id(), 3u); - - myCounter.add(3, one); - myCounter.add(one); - myGauge.sample(14.0, two); - myGauge.sample(11.0, three); - - EXPECT_EQUAL(2.0, ticker->give(TimeStamp(4.5)).count()); - - Snapshot snap2 = manager->snapshot(); - EXPECT_EQUAL(1.0, snap2.startTime()); - EXPECT_EQUAL(4.5, snap2.endTime()); - EXPECT_EQUAL(2u, snap2.counters().size()); - EXPECT_EQUAL(3u, snap2.gauges().size()); - - JsonFormatter fmt2(snap2); - check_json(fmt2.asString()); - - // flush sliding window - for (int i = 5; i <= 10; ++i) { - ticker->give(TimeStamp(i)); - } - Snapshot snap3 = manager->snapshot(); - EXPECT_EQUAL(5.0, snap3.startTime()); - EXPECT_EQUAL(10.0, snap3.endTime()); - EXPECT_EQUAL(2u, snap3.counters().size()); - EXPECT_EQUAL(0u, snap3.counters()[0].count()); - EXPECT_EQUAL(0u, snap3.counters()[1].count()); - EXPECT_EQUAL(3u, snap3.gauges().size()); - EXPECT_EQUAL(0u, snap3.gauges()[0].observedCount()); - EXPECT_EQUAL(0u, snap3.gauges()[1].observedCount()); - EXPECT_EQUAL(0u, snap3.gauges()[2].observedCount()); - - Snapshot snap4 = manager->totalSnapshot(); - EXPECT_EQUAL(1.0, snap4.startTime()); - EXPECT_EQUAL(10.0, snap4.endTime()); - EXPECT_EQUAL(2u, snap4.counters().size()); - EXPECT_NOT_EQUAL(0u, snap4.counters()[0].count()); - EXPECT_NOT_EQUAL(0u, snap4.counters()[1].count()); - EXPECT_EQUAL(3u, snap4.gauges().size()); - EXPECT_NOT_EQUAL(0u, snap4.gauges()[0].observedCount()); - EXPECT_NOT_EQUAL(0u, snap4.gauges()[1].observedCount()); - EXPECT_NOT_EQUAL(0u, snap4.gauges()[2].observedCount()); -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/staging_vespalib/src/tests/metrics/stable_store_test.cpp b/staging_vespalib/src/tests/metrics/stable_store_test.cpp deleted file mode 100644 index cead112069f..00000000000 --- a/staging_vespalib/src/tests/metrics/stable_store_test.cpp +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/metrics/simple_metrics.h> -#include <vespa/vespalib/metrics/simple_metrics_manager.h> -#include <vespa/vespalib/metrics/stable_store.h> -#include <vespa/vespalib/metrics/json_formatter.h> -#include <stdio.h> -#include <unistd.h> - -using namespace vespalib; -using namespace vespalib::metrics; - -struct Foo { - int a; - char *p; - explicit Foo(int v) : a(v), p(nullptr) {} - bool operator==(const Foo &other) const { - return a == other.a; - } -}; - -TEST("require that stable_store works") -{ - vespalib::StableStore<Foo> bunch; - bunch.add(Foo(1)); - bunch.add(Foo(2)); - bunch.add(Foo(3)); - bunch.add(Foo(5)); - bunch.add(Foo(8)); - bunch.add(Foo(13)); - bunch.add(Foo(21)); - bunch.add(Foo(34)); - bunch.add(Foo(55)); - bunch.add(Foo(89)); - - EXPECT_EQUAL(bunch.size(), 10u); - - int sum = 0; - - bunch.for_each([&sum](const Foo& value) { sum += value.a; }); - EXPECT_EQUAL(231, sum); - - std::vector<const Foo *> pointers; - bunch.for_each([&pointers](const Foo& value) - { pointers.push_back(&value); }); - EXPECT_EQUAL(1, pointers[0]->a); - EXPECT_EQUAL(2, pointers[1]->a); - EXPECT_EQUAL(55, pointers[8]->a); - EXPECT_EQUAL(89, pointers[9]->a); - - for (int i = 0; i < 20000; ++i) { - bunch.add(Foo(i)); - } - bunch.for_each([&sum](const Foo& value) { sum -= value.a; }); - EXPECT_EQUAL(-199990000, sum); - - std::vector<const Foo *> after; - bunch.for_each([&after](const Foo& value) - { if (after.size() < 10) after.push_back(&value); }); - - EXPECT_EQUAL(pointers[0], after[0]); - EXPECT_EQUAL(pointers[9], after[9]); -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/.gitignore b/staging_vespalib/src/tests/sequencedtaskexecutor/.gitignore deleted file mode 100644 index 523cfe5e3e1..00000000000 --- a/staging_vespalib/src/tests/sequencedtaskexecutor/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -staging_vespalib_sequencedtaskexecutor_test_app -staging_vespalib_sequencedtaskexecutor_benchmark_app -staging_vespalib_adaptive_sequenced_executor_test_app -staging_vespalib_foregroundtaskexecutor_test_app diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/CMakeLists.txt b/staging_vespalib/src/tests/sequencedtaskexecutor/CMakeLists.txt deleted file mode 100644 index cc851f668da..00000000000 --- a/staging_vespalib/src/tests/sequencedtaskexecutor/CMakeLists.txt +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_sequencedtaskexecutor_benchmark_app TEST - SOURCES - sequencedtaskexecutor_benchmark.cpp - DEPENDS - staging_vespalib -) - -vespa_add_executable(staging_vespalib_sequencedtaskexecutor_test_app TEST - SOURCES - sequencedtaskexecutor_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_sequencedtaskexecutor_test_app COMMAND staging_vespalib_sequencedtaskexecutor_test_app) - -vespa_add_executable(staging_vespalib_adaptive_sequenced_executor_test_app TEST - SOURCES - adaptive_sequenced_executor_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_adaptive_sequenced_executor_test_app COMMAND staging_vespalib_adaptive_sequenced_executor_test_app) - -vespa_add_executable(staging_vespalib_foregroundtaskexecutor_test_app TEST - SOURCES - foregroundtaskexecutor_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_foregroundtaskexecutor_test_app COMMAND staging_vespalib_foregroundtaskexecutor_test_app) diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp b/staging_vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp deleted file mode 100644 index 1a458f86232..00000000000 --- a/staging_vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp +++ /dev/null @@ -1,248 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/util/adaptive_sequenced_executor.h> -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/test/insertion_operators.h> - -#include <condition_variable> -#include <unistd.h> - -#include <vespa/log/log.h> -LOG_SETUP("adaptive_sequenced_executor_test"); - -namespace vespalib { - - -class Fixture -{ -public: - AdaptiveSequencedExecutor _threads; - - Fixture(bool is_max_pending_hard=true) : _threads(2, 2, 0, 1000, is_max_pending_hard) { } -}; - - -class TestObj -{ -public: - std::mutex _m; - std::condition_variable _cv; - int _done; - int _fail; - int _val; - - TestObj() noexcept - : _m(), - _cv(), - _done(0), - _fail(0), - _val(0) - { - } - - void - modify(int oldValue, int newValue) - { - { - std::lock_guard<std::mutex> guard(_m); - if (_val == oldValue) { - _val = newValue; - } else { - ++_fail; - } - ++_done; - _cv.notify_all(); - } - } - - void - wait(int wantDone) - { - std::unique_lock<std::mutex> guard(_m); - _cv.wait(guard, [&] { return this->_done >= wantDone; }); - } -}; - -vespalib::stringref ZERO("0"); - -TEST_F("testExecute", Fixture) { - std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); - EXPECT_EQUAL(0, tv->_val); - f._threads.execute(1, [&]() { tv->modify(0, 42); }); - tv->wait(1); - EXPECT_EQUAL(0, tv->_fail); - EXPECT_EQUAL(42, tv->_val); - f._threads.sync_all(); - EXPECT_EQUAL(0, tv->_fail); - EXPECT_EQUAL(42, tv->_val); -} - - -TEST_F("require that task with same component id are serialized", Fixture) -{ - std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); - EXPECT_EQUAL(0, tv->_val); - f._threads.execute(0, [&]() { usleep(2000); tv->modify(0, 14); }); - f._threads.execute(0, [&]() { tv->modify(14, 42); }); - tv->wait(2); - EXPECT_EQUAL(0, tv->_fail); - EXPECT_EQUAL(42, tv->_val); - f._threads.sync_all(); - EXPECT_EQUAL(0, tv->_fail); - EXPECT_EQUAL(42, tv->_val); -} - -TEST_F("require that task with different component ids are not serialized", Fixture) -{ - int tryCnt = 0; - for (tryCnt = 0; tryCnt < 100; ++tryCnt) { - std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); - EXPECT_EQUAL(0, tv->_val); - f._threads.execute(0, [&]() { usleep(2000); tv->modify(0, 14); }); - f._threads.execute(1, [&]() { tv->modify(14, 42); }); - tv->wait(2); - if (tv->_fail != 1) { - continue; - } - EXPECT_EQUAL(1, tv->_fail); - EXPECT_EQUAL(14, tv->_val); - f._threads.sync_all(); - EXPECT_EQUAL(1, tv->_fail); - EXPECT_EQUAL(14, tv->_val); - break; - } - EXPECT_TRUE(tryCnt < 100); -} - - -TEST_F("require that task with same string component id are serialized", Fixture) -{ - std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); - EXPECT_EQUAL(0, tv->_val); - auto test2 = [&]() { tv->modify(14, 42); }; - f._threads.execute(f._threads.getExecutorIdFromName(ZERO), [&]() { usleep(2000); tv->modify(0, 14); }); - f._threads.execute(f._threads.getExecutorIdFromName(ZERO), test2); - tv->wait(2); - EXPECT_EQUAL(0, tv->_fail); - EXPECT_EQUAL(42, tv->_val); - f._threads.sync_all(); - EXPECT_EQUAL(0, tv->_fail); - EXPECT_EQUAL(42, tv->_val); -} - -namespace { - -int detectSerializeFailure(Fixture &f, vespalib::stringref altComponentId, int tryLimit) -{ - int tryCnt = 0; - for (tryCnt = 0; tryCnt < tryLimit; ++tryCnt) { - std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); - EXPECT_EQUAL(0, tv->_val); - f._threads.execute(f._threads.getExecutorIdFromName(ZERO), [&]() { usleep(2000); tv->modify(0, 14); }); - f._threads.execute(f._threads.getExecutorIdFromName(altComponentId), [&]() { tv->modify(14, 42); }); - tv->wait(2); - if (tv->_fail != 1) { - continue; - } - EXPECT_EQUAL(1, tv->_fail); - EXPECT_EQUAL(14, tv->_val); - f._threads.sync_all(); - EXPECT_EQUAL(1, tv->_fail); - EXPECT_EQUAL(14, tv->_val); - break; - } - return tryCnt; -} - -vespalib::string makeAltComponentId(Fixture &f) -{ - int tryCnt = 0; - char altComponentId[20]; - ISequencedTaskExecutor::ExecutorId executorId0 = f._threads.getExecutorIdFromName(ZERO); - for (tryCnt = 1; tryCnt < 100; ++tryCnt) { - sprintf(altComponentId, "%d", tryCnt); - if (f._threads.getExecutorIdFromName(altComponentId) == executorId0) { - break; - } - } - EXPECT_TRUE(tryCnt < 100); - return altComponentId; -} - -} - -TEST_F("require that task with different string component ids are not serialized", Fixture) -{ - int tryCnt = detectSerializeFailure(f, "2", 100); - EXPECT_TRUE(tryCnt < 100); -} - - -TEST_F("require that task with different string component ids mapping to the same executor id are serialized", - Fixture) -{ - vespalib::string altComponentId = makeAltComponentId(f); - LOG(info, "second string component id is \"%s\"", altComponentId.c_str()); - int tryCnt = detectSerializeFailure(f, altComponentId, 100); - EXPECT_TRUE(tryCnt == 100); -} - - -TEST_F("require that execute works with const lambda", Fixture) -{ - int i = 5; - std::vector<int> res; - const auto lambda = [i, &res]() mutable - { res.push_back(i--); res.push_back(i--); }; - f._threads.execute(0, lambda); - f._threads.execute(0, lambda); - f._threads.sync_all(); - std::vector<int> exp({5, 4, 5, 4}); - EXPECT_EQUAL(exp, res); - EXPECT_EQUAL(5, i); -} - -TEST_F("require that execute works with reference to lambda", Fixture) -{ - int i = 5; - std::vector<int> res; - auto lambda = [i, &res]() mutable - { res.push_back(i--); res.push_back(i--); }; - auto &lambdaref = lambda; - f._threads.execute(0, lambdaref); - f._threads.execute(0, lambdaref); - f._threads.sync_all(); - std::vector<int> exp({5, 4, 5, 4}); - EXPECT_EQUAL(exp, res); - EXPECT_EQUAL(5, i); -} - -TEST_F("require that executeLambda works", Fixture) -{ - int i = 5; - std::vector<int> res; - const auto lambda = [i, &res]() mutable - { res.push_back(i--); res.push_back(i--); }; - f._threads.executeLambda(ISequencedTaskExecutor::ExecutorId(0), lambda); - f._threads.sync_all(); - std::vector<int> exp({5, 4}); - EXPECT_EQUAL(exp, res); - EXPECT_EQUAL(5, i); -} - -TEST("require that you get correct number of executors") { - AdaptiveSequencedExecutor seven(7, 1, 0, 10, true); - EXPECT_EQUAL(7u, seven.getNumExecutors()); -} - -TEST("require that you distribute well") { - AdaptiveSequencedExecutor seven(7, 1, 0, 10, true); - EXPECT_EQUAL(7u, seven.getNumExecutors()); - for (uint32_t id=0; id < 1000; id++) { - EXPECT_EQUAL(id%7, seven.getExecutorId(id).getId()); - } -} - -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/foregroundtaskexecutor_test.cpp b/staging_vespalib/src/tests/sequencedtaskexecutor/foregroundtaskexecutor_test.cpp deleted file mode 100644 index 56fb570209c..00000000000 --- a/staging_vespalib/src/tests/sequencedtaskexecutor/foregroundtaskexecutor_test.cpp +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/util/foregroundtaskexecutor.h> -#include <vespa/vespalib/testkit/testapp.h> - -#include <condition_variable> -#include <unistd.h> - -#include <vespa/log/log.h> -LOG_SETUP("foregroundtaskexecutor_test"); - -namespace vespalib { - - -class Fixture -{ -public: - ForegroundTaskExecutor _threads; - - Fixture() - : _threads() - { - } -}; - - -class TestObj -{ -public: - std::mutex _m; - std::condition_variable _cv; - int _done; - int _fail; - int _val; - - TestObj() noexcept - : _m(), - _cv(), - _done(0), - _fail(0), - _val(0) - { - } - - void - modify(int oldValue, int newValue) - { - { - std::lock_guard<std::mutex> guard(_m); - if (_val == oldValue) { - _val = newValue; - } else { - ++_fail; - } - ++_done; - } - _cv.notify_all(); - } - - void - wait(int wantDone) - { - std::unique_lock<std::mutex> guard(_m); - _cv.wait(guard, [this, wantDone] { return this->_done >= wantDone; }); - } -}; - -TEST_F("testExecute", Fixture) { - std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); - EXPECT_EQUAL(0, tv->_val); - f._threads.execute(1, [=]() { tv->modify(0, 42); }); - tv->wait(1); - EXPECT_EQUAL(0, tv->_fail); - EXPECT_EQUAL(42, tv->_val); - f._threads.sync_all(); - EXPECT_EQUAL(0, tv->_fail); - EXPECT_EQUAL(42, tv->_val); -} - - -TEST_F("require that task with same id are serialized", Fixture) -{ - std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); - EXPECT_EQUAL(0, tv->_val); - f._threads.execute(0, [=]() { usleep(2000); tv->modify(0, 14); }); - f._threads.execute(0, [=]() { tv->modify(14, 42); }); - tv->wait(2); - EXPECT_EQUAL(0, tv->_fail); - EXPECT_EQUAL(42, tv->_val); - f._threads.sync_all(); - EXPECT_EQUAL(0, tv->_fail); - EXPECT_EQUAL(42, tv->_val); -} - -TEST_F("require that task with different ids are serialized", Fixture) -{ - int tryCnt = 0; - for (tryCnt = 0; tryCnt < 100; ++tryCnt) { - std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); - EXPECT_EQUAL(0, tv->_val); - f._threads.execute(0, [=]() { usleep(2000); tv->modify(0, 14); }); - f._threads.execute(1, [=]() { tv->modify(14, 42); }); - tv->wait(2); - if (tv->_fail != 1) { - continue; - } - EXPECT_EQUAL(1, tv->_fail); - EXPECT_EQUAL(14, tv->_val); - f._threads.sync_all(); - EXPECT_EQUAL(1, tv->_fail); - EXPECT_EQUAL(14, tv->_val); - break; - } - EXPECT_TRUE(tryCnt >= 100); -} - - -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp b/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp deleted file mode 100644 index 0f7c82ef988..00000000000 --- a/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/util/sequencedtaskexecutor.h> -#include <vespa/vespalib/util/adaptive_sequenced_executor.h> -#include <vespa/vespalib/util/lambdatask.h> -#include <vespa/vespalib/util/time.h> -#include <atomic> -#include <cinttypes> - -using vespalib::ISequencedTaskExecutor; -using vespalib::SequencedTaskExecutor; -using vespalib::AdaptiveSequencedExecutor; -using ExecutorId = vespalib::ISequencedTaskExecutor::ExecutorId; - -size_t do_work(size_t size) { - size_t ret = 0; - for (size_t i = 0; i < size; ++i) { - for (size_t j = 0; j < 128; ++j) { - ret = (ret + i) * j; - } - } - return ret; -} - -struct SimpleParams { - int argc; - char **argv; - int idx; - SimpleParams(int argc_in, char **argv_in) : argc(argc_in), argv(argv_in), idx(0) {} - int next(const char *name, int fallback) { - ++idx; - int value = 0; - if (argc > idx) { - value = atoi(argv[idx]); - } else { - value = fallback; - } - fprintf(stderr, "param %s: %d\n", name, value); - return value; - } -}; - -VESPA_THREAD_STACK_TAG(sequenced_executor) - -int main(int argc, char **argv) { - SimpleParams params(argc, argv); - bool use_adaptive_executor = params.next("use_adaptive_executor", 0); - bool optimize_for_throughput = params.next("optimize_for_throughput", 0); - size_t num_tasks = params.next("num_tasks", 1000000); - size_t num_strands = params.next("num_strands", 4); - size_t task_limit = params.next("task_limit", 1000); - size_t num_threads = params.next("num_threads", num_strands); - size_t max_waiting = params.next("max_waiting", optimize_for_throughput ? 32 : 0); - size_t work_size = params.next("work_size", 0); - std::atomic<long> counter(0); - std::unique_ptr<ISequencedTaskExecutor> executor; - if (use_adaptive_executor) { - executor = std::make_unique<AdaptiveSequencedExecutor>(num_strands, num_threads, max_waiting, task_limit, true); - } else { - auto optimize = optimize_for_throughput - ? vespalib::Executor::OptimizeFor::THROUGHPUT - : vespalib::Executor::OptimizeFor::LATENCY; - executor = SequencedTaskExecutor::create(sequenced_executor, num_strands, task_limit, true, optimize); - } - vespalib::Timer timer; - for (size_t task_id = 0; task_id < num_tasks; ++task_id) { - executor->executeTask(ExecutorId(task_id % num_strands), - vespalib::makeLambdaTask([&counter,work_size] { (void) do_work(work_size); counter++; })); - } - executor.reset(); - fprintf(stderr, "\ntotal time: %" PRId64 " ms\n", vespalib::count_ms(timer.elapsed())); - return (size_t(counter) == num_tasks) ? 0 : 1; -} diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp b/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp deleted file mode 100644 index 705d6346e8c..00000000000 --- a/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp +++ /dev/null @@ -1,351 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/util/sequencedtaskexecutor.h> -#include <vespa/vespalib/util/adaptive_sequenced_executor.h> -#include <vespa/vespalib/util/blockingthreadstackexecutor.h> -#include <vespa/vespalib/util/singleexecutor.h> -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/test/insertion_operators.h> - -#include <condition_variable> -#include <unistd.h> - -#include <vespa/log/log.h> -LOG_SETUP("sequencedtaskexecutor_test"); - -VESPA_THREAD_STACK_TAG(sequenced_executor) - -namespace vespalib { - - -class Fixture -{ -public: - std::unique_ptr<ISequencedTaskExecutor> _threads; - - Fixture(bool is_task_limit_hard = true) : - _threads(SequencedTaskExecutor::create(sequenced_executor, 2, 1000, is_task_limit_hard, - Executor::OptimizeFor::LATENCY)) - { } -}; - - -class TestObj -{ -public: - std::mutex _m; - std::condition_variable _cv; - int _done; - int _fail; - int _val; - - TestObj() noexcept - : _m(), - _cv(), - _done(0), - _fail(0), - _val(0) - { - } - - void - modify(int oldValue, int newValue) - { - { - std::lock_guard<std::mutex> guard(_m); - if (_val == oldValue) { - _val = newValue; - } else { - ++_fail; - } - ++_done; - } - _cv.notify_all(); - } - - void - wait(int wantDone) - { - std::unique_lock<std::mutex> guard(_m); - _cv.wait(guard, [this, wantDone] { return this->_done >= wantDone; }); - } -}; - -vespalib::stringref ZERO("0"); - -TEST_F("testExecute", Fixture) { - std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); - EXPECT_EQUAL(0, tv->_val); - f._threads->execute(1, [=]() { tv->modify(0, 42); }); - tv->wait(1); - EXPECT_EQUAL(0, tv->_fail); - EXPECT_EQUAL(42, tv->_val); - f._threads->sync_all(); - EXPECT_EQUAL(0, tv->_fail); - EXPECT_EQUAL(42, tv->_val); -} - - -TEST_F("require that task with same component id are serialized", Fixture) -{ - std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); - EXPECT_EQUAL(0, tv->_val); - f._threads->execute(0, [=]() { usleep(2000); tv->modify(0, 14); }); - f._threads->execute(0, [=]() { tv->modify(14, 42); }); - tv->wait(2); - EXPECT_EQUAL(0, tv->_fail); - EXPECT_EQUAL(42, tv->_val); - f._threads->sync_all(); - EXPECT_EQUAL(0, tv->_fail); - EXPECT_EQUAL(42, tv->_val); -} - -TEST_F("require that task with same component id are serialized when executed with list", Fixture) -{ - std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); - EXPECT_EQUAL(0, tv->_val); - ISequencedTaskExecutor::ExecutorId executorId = f._threads->getExecutorId(0); - ISequencedTaskExecutor::TaskList list; - list.template emplace_back(executorId, makeLambdaTask([=]() { usleep(2000); tv->modify(0, 14); })); - list.template emplace_back(executorId, makeLambdaTask([=]() { tv->modify(14, 42); })); - f._threads->executeTasks(std::move(list)); - tv->wait(2); - EXPECT_EQUAL(0, tv->_fail); - EXPECT_EQUAL(42, tv->_val); - f._threads->sync_all(); - EXPECT_EQUAL(0, tv->_fail); - EXPECT_EQUAL(42, tv->_val); -} - -TEST_F("require that task with different component ids are not serialized", Fixture) -{ - int tryCnt = 0; - for (tryCnt = 0; tryCnt < 100; ++tryCnt) { - std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); - EXPECT_EQUAL(0, tv->_val); - f._threads->execute(0, [=]() { usleep(2000); tv->modify(0, 14); }); - f._threads->execute(2, [=]() { tv->modify(14, 42); }); - tv->wait(2); - if (tv->_fail != 1) { - continue; - } - EXPECT_EQUAL(1, tv->_fail); - EXPECT_EQUAL(14, tv->_val); - f._threads->sync_all(); - EXPECT_EQUAL(1, tv->_fail); - EXPECT_EQUAL(14, tv->_val); - break; - } - EXPECT_TRUE(tryCnt < 100); -} - - -TEST_F("require that task with same string component id are serialized", Fixture) -{ - std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); - EXPECT_EQUAL(0, tv->_val); - auto test2 = [=]() { tv->modify(14, 42); }; - f._threads->execute(f._threads->getExecutorIdFromName(ZERO), [=]() { usleep(2000); tv->modify(0, 14); }); - f._threads->execute(f._threads->getExecutorIdFromName(ZERO), test2); - tv->wait(2); - EXPECT_EQUAL(0, tv->_fail); - EXPECT_EQUAL(42, tv->_val); - f._threads->sync_all(); - EXPECT_EQUAL(0, tv->_fail); - EXPECT_EQUAL(42, tv->_val); -} - -namespace { - -int -detectSerializeFailure(Fixture &f, vespalib::stringref altComponentId, int tryLimit) -{ - int tryCnt = 0; - for (tryCnt = 0; tryCnt < tryLimit; ++tryCnt) { - std::shared_ptr<TestObj> tv(std::make_shared<TestObj>()); - EXPECT_EQUAL(0, tv->_val); - f._threads->execute(f._threads->getExecutorIdFromName(ZERO), [=]() { usleep(2000); tv->modify(0, 14); }); - f._threads->execute(f._threads->getExecutorIdFromName(altComponentId), [=]() { tv->modify(14, 42); }); - tv->wait(2); - if (tv->_fail != 1) { - continue; - } - EXPECT_EQUAL(1, tv->_fail); - EXPECT_EQUAL(14, tv->_val); - f._threads->sync_all(); - EXPECT_EQUAL(1, tv->_fail); - EXPECT_EQUAL(14, tv->_val); - break; - } - return tryCnt; -} - -vespalib::string -makeAltComponentId(Fixture &f) -{ - int tryCnt = 0; - char altComponentId[20]; - ISequencedTaskExecutor::ExecutorId executorId0 = f._threads->getExecutorIdFromName(ZERO); - for (tryCnt = 1; tryCnt < 100; ++tryCnt) { - sprintf(altComponentId, "%d", tryCnt); - if (f._threads->getExecutorIdFromName(altComponentId) == executorId0) { - break; - } - } - EXPECT_TRUE(tryCnt < 100); - return altComponentId; -} - -} - -TEST_F("require that task with different string component ids are not serialized", Fixture) -{ - int tryCnt = detectSerializeFailure(f, "2", 100); - EXPECT_TRUE(tryCnt < 100); -} - - -TEST_F("require that task with different string component ids mapping to the same executor id are serialized", - Fixture) -{ - vespalib::string altComponentId = makeAltComponentId(f); - LOG(info, "second string component id is \"%s\"", altComponentId.c_str()); - int tryCnt = detectSerializeFailure(f, altComponentId, 100); - EXPECT_TRUE(tryCnt == 100); -} - - -TEST_F("require that execute works with const lambda", Fixture) -{ - int i = 5; - std::vector<int> res; - const auto lambda = [i, &res]() mutable - { res.push_back(i--); res.push_back(i--); }; - f._threads->execute(0, lambda); - f._threads->execute(0, lambda); - f._threads->sync_all(); - std::vector<int> exp({5, 4, 5, 4}); - EXPECT_EQUAL(exp, res); - EXPECT_EQUAL(5, i); -} - -TEST_F("require that execute works with reference to lambda", Fixture) -{ - int i = 5; - std::vector<int> res; - auto lambda = [i, &res]() mutable - { res.push_back(i--); res.push_back(i--); }; - auto &lambdaref = lambda; - f._threads->execute(0, lambdaref); - f._threads->execute(0, lambdaref); - f._threads->sync_all(); - std::vector<int> exp({5, 4, 5, 4}); - EXPECT_EQUAL(exp, res); - EXPECT_EQUAL(5, i); -} - -TEST_F("require that executeLambda works", Fixture) -{ - int i = 5; - std::vector<int> res; - const auto lambda = [i, &res]() mutable - { res.push_back(i--); res.push_back(i--); }; - f._threads->executeLambda(ISequencedTaskExecutor::ExecutorId(0), lambda); - f._threads->sync_all(); - std::vector<int> exp({5, 4}); - EXPECT_EQUAL(exp, res); - EXPECT_EQUAL(5, i); -} - -TEST("require that you get correct number of executors") { - auto seven = SequencedTaskExecutor::create(sequenced_executor, 7); - EXPECT_EQUAL(7u, seven->getNumExecutors()); -} - -void verifyHardLimitForLatency(bool expect_hard) { - auto sequenced = SequencedTaskExecutor::create(sequenced_executor, 1, 100, expect_hard, Executor::OptimizeFor::LATENCY); - const SequencedTaskExecutor & seq = dynamic_cast<const SequencedTaskExecutor &>(*sequenced); - EXPECT_EQUAL(expect_hard,nullptr != dynamic_cast<const BlockingThreadStackExecutor *>(seq.first_executor())); -} - -void verifyHardLimitForThroughput(bool expect_hard) { - auto sequenced = SequencedTaskExecutor::create(sequenced_executor, 1, 100, expect_hard, Executor::OptimizeFor::THROUGHPUT); - const SequencedTaskExecutor & seq = dynamic_cast<const SequencedTaskExecutor &>(*sequenced); - const SingleExecutor * first = dynamic_cast<const SingleExecutor *>(seq.first_executor()); - EXPECT_TRUE(first != nullptr); - EXPECT_EQUAL(expect_hard, first->isBlocking()); -} - -TEST("require that you can get executor with both hard and soft limit") { - verifyHardLimitForLatency(true); - verifyHardLimitForLatency(false); - verifyHardLimitForThroughput(true); - verifyHardLimitForThroughput(false); -} - - -TEST("require that you distribute well") { - auto seven = SequencedTaskExecutor::create(sequenced_executor, 7); - const SequencedTaskExecutor & seq = dynamic_cast<const SequencedTaskExecutor &>(*seven); - const uint32_t NUM_EXACT = 8 * seven->getNumExecutors(); - EXPECT_EQUAL(7u, seven->getNumExecutors()); - EXPECT_EQUAL(97u, seq.getComponentHashSize()); - EXPECT_EQUAL(0u, seq.getComponentEffectiveHashSize()); - for (uint32_t id=0; id < 1000; id++) { - if (id < NUM_EXACT) { - EXPECT_EQUAL(id % seven->getNumExecutors(), seven->getExecutorId(id).getId()); - } else { - EXPECT_EQUAL(((id - NUM_EXACT) % 97) % seven->getNumExecutors(), seven->getExecutorId(id).getId()); - } - } - EXPECT_EQUAL(97u, seq.getComponentHashSize()); - EXPECT_EQUAL(97u, seq.getComponentEffectiveHashSize()); -} - -TEST("require that similar names get perfect distribution with 4 executors") { - auto four = SequencedTaskExecutor::create(sequenced_executor, 4); - EXPECT_EQUAL(0u, four->getExecutorIdFromName("f1").getId()); - EXPECT_EQUAL(1u, four->getExecutorIdFromName("f2").getId()); - EXPECT_EQUAL(2u, four->getExecutorIdFromName("f3").getId()); - EXPECT_EQUAL(3u, four->getExecutorIdFromName("f4").getId()); - EXPECT_EQUAL(0u, four->getExecutorIdFromName("f5").getId()); - EXPECT_EQUAL(1u, four->getExecutorIdFromName("f6").getId()); - EXPECT_EQUAL(2u, four->getExecutorIdFromName("f7").getId()); - EXPECT_EQUAL(3u, four->getExecutorIdFromName("f8").getId()); -} - -TEST("require that similar names get perfect distribution with 8 executors") { - auto four = SequencedTaskExecutor::create(sequenced_executor, 8); - EXPECT_EQUAL(0u, four->getExecutorIdFromName("f1").getId()); - EXPECT_EQUAL(1u, four->getExecutorIdFromName("f2").getId()); - EXPECT_EQUAL(2u, four->getExecutorIdFromName("f3").getId()); - EXPECT_EQUAL(3u, four->getExecutorIdFromName("f4").getId()); - EXPECT_EQUAL(4u, four->getExecutorIdFromName("f5").getId()); - EXPECT_EQUAL(5u, four->getExecutorIdFromName("f6").getId()); - EXPECT_EQUAL(6u, four->getExecutorIdFromName("f7").getId()); - EXPECT_EQUAL(7u, four->getExecutorIdFromName("f8").getId()); -} - -TEST("Test creation of different types") { - auto iseq = SequencedTaskExecutor::create(sequenced_executor, 1); - - EXPECT_EQUAL(1u, iseq->getNumExecutors()); - auto * seq = dynamic_cast<SequencedTaskExecutor *>(iseq.get()); - ASSERT_TRUE(seq != nullptr); - - iseq = SequencedTaskExecutor::create(sequenced_executor, 1, 1000, true, Executor::OptimizeFor::LATENCY); - seq = dynamic_cast<SequencedTaskExecutor *>(iseq.get()); - ASSERT_TRUE(seq != nullptr); - - iseq = SequencedTaskExecutor::create(sequenced_executor, 1, 1000, true, Executor::OptimizeFor::THROUGHPUT); - seq = dynamic_cast<SequencedTaskExecutor *>(iseq.get()); - ASSERT_TRUE(seq != nullptr); - - iseq = SequencedTaskExecutor::create(sequenced_executor, 1, 1000, true, Executor::OptimizeFor::ADAPTIVE, 17); - auto aseq = dynamic_cast<AdaptiveSequencedExecutor *>(iseq.get()); - ASSERT_TRUE(aseq != nullptr); -} - -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/staging_vespalib/src/tests/singleexecutor/CMakeLists.txt b/staging_vespalib/src/tests/singleexecutor/CMakeLists.txt deleted file mode 100644 index 8ff0ff1989a..00000000000 --- a/staging_vespalib/src/tests/singleexecutor/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_singleexecutor_test_app TEST - SOURCES - singleexecutor_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_singleexecutor_test_app COMMAND staging_vespalib_singleexecutor_test_app) diff --git a/staging_vespalib/src/tests/singleexecutor/singleexecutor_test.cpp b/staging_vespalib/src/tests/singleexecutor/singleexecutor_test.cpp deleted file mode 100644 index 56352ff3c0d..00000000000 --- a/staging_vespalib/src/tests/singleexecutor/singleexecutor_test.cpp +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/vespalib/testkit/testapp.h> - -#include <vespa/vespalib/util/singleexecutor.h> -#include <vespa/vespalib/util/lambdatask.h> -#include <vespa/vespalib/util/alloc.h> -#include <atomic> - -using namespace vespalib; - -VESPA_THREAD_STACK_TAG(sequenced_executor) - -TEST("test that all tasks are executed") { - - std::atomic<uint64_t> counter(0); - SingleExecutor executor(sequenced_executor, 10); - - for (uint64_t i(0); i < 10; i++) { - executor.execute(makeLambdaTask([&counter] {counter++;})); - } - executor.sync(); - EXPECT_EQUAL(10u, counter); - - counter = 0; - for (uint64_t i(0); i < 10000; i++) { - executor.execute(makeLambdaTask([&counter] {counter++;})); - } - executor.sync(); - EXPECT_EQUAL(10000u, counter); -} - -TEST("test that executor can overflow") { - constexpr size_t NUM_TASKS = 1000; - std::atomic<uint64_t> counter(0); - vespalib::Gate gate; - SingleExecutor executor(sequenced_executor, 10, false, 1, 1ms); - executor.execute(makeLambdaTask([&gate] { gate.await();})); - - for(size_t i(0); i < NUM_TASKS; i++) { - executor.execute(makeLambdaTask([&counter, i] { - EXPECT_EQUAL(i, counter); - counter++; - })); - } - EXPECT_EQUAL(0u, counter); - ExecutorStats stats = executor.getStats(); - EXPECT_EQUAL(NUM_TASKS + 1, stats.acceptedTasks); - EXPECT_EQUAL(NUM_TASKS, stats.queueSize.max()); - gate.countDown(); - executor.sync(); - EXPECT_EQUAL(NUM_TASKS, counter); -} - -void verifyResizeTaskLimit(bool up) { - std::mutex lock; - std::condition_variable cond; - std::atomic<uint64_t> started(0); - std::atomic<uint64_t> allowed(0); - constexpr uint32_t INITIAL = 20; - const uint32_t INITIAL_2inN = roundUp2inN(INITIAL); - double waterMarkRatio = 0.5; - SingleExecutor executor(sequenced_executor, INITIAL, true, INITIAL*waterMarkRatio, 10ms); - EXPECT_EQUAL(INITIAL_2inN, executor.getTaskLimit()); - EXPECT_EQUAL(uint32_t(INITIAL_2inN*waterMarkRatio), executor.get_watermark()); - - uint32_t targetTaskLimit = up ? 40 : 5; - uint32_t roundedTaskLimit = roundUp2inN(targetTaskLimit); - EXPECT_NOT_EQUAL(INITIAL_2inN, roundedTaskLimit); - - for (uint64_t i(0); i < INITIAL; i++) { - executor.execute(makeLambdaTask([&lock, &cond, &started, &allowed] { - started++; - std::unique_lock guard(lock); - while (allowed < started) { - cond.wait_for(guard, 1ms); - } - })); - } - while (started < 1); - EXPECT_EQUAL(1u, started); - executor.setTaskLimit(targetTaskLimit); - EXPECT_EQUAL(INITIAL_2inN, executor.getTaskLimit()); - EXPECT_EQUAL(INITIAL_2inN*waterMarkRatio, executor.get_watermark()); - allowed = 5; - while (started < 6); - EXPECT_EQUAL(6u, started); - EXPECT_EQUAL(INITIAL_2inN, executor.getTaskLimit()); - allowed = INITIAL; - while (started < INITIAL); - EXPECT_EQUAL(INITIAL, started); - EXPECT_EQUAL(INITIAL_2inN, executor.getTaskLimit()); - executor.execute(makeLambdaTask([&lock, &cond, &started, &allowed] { - started++; - std::unique_lock guard(lock); - while (allowed < started) { - cond.wait_for(guard, 1ms); - } - })); - while (started < INITIAL + 1); - EXPECT_EQUAL(INITIAL + 1, started); - EXPECT_EQUAL(roundedTaskLimit, executor.getTaskLimit()); - EXPECT_EQUAL(roundedTaskLimit*waterMarkRatio, executor.get_watermark()); - allowed = INITIAL + 1; -} - -TEST("test that resizing up and down works") { - TEST_DO(verifyResizeTaskLimit(true)); - TEST_DO(verifyResizeTaskLimit(false)); - - -} - -TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/staging_vespalib/src/tests/state_server/.gitignore b/staging_vespalib/src/tests/state_server/.gitignore deleted file mode 100644 index b1884d4e060..00000000000 --- a/staging_vespalib/src/tests/state_server/.gitignore +++ /dev/null @@ -1 +0,0 @@ -staging_vespalib_state_server_test_app diff --git a/staging_vespalib/src/tests/state_server/CMakeLists.txt b/staging_vespalib/src/tests/state_server/CMakeLists.txt deleted file mode 100644 index fc22d8b9178..00000000000 --- a/staging_vespalib/src/tests/state_server/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_state_server_test_app TEST - SOURCES - state_server_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_state_server_test_app NO_VALGRIND NO_VALGRIND COMMAND staging_vespalib_state_server_test_app ENVIRONMENT "VESPA_HOME=.") diff --git a/staging_vespalib/src/tests/state_server/state_server_test.cpp b/staging_vespalib/src/tests/state_server/state_server_test.cpp deleted file mode 100644 index cb80a44a675..00000000000 --- a/staging_vespalib/src/tests/state_server/state_server_test.cpp +++ /dev/null @@ -1,520 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <fcntl.h> -#include <stdlib.h> -#include <sys/stat.h> -#include <sys/types.h> - -#include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/host_name.h> -#include <vespa/vespalib/net/state_server.h> -#include <vespa/vespalib/net/simple_health_producer.h> -#include <vespa/vespalib/net/simple_metrics_producer.h> -#include <vespa/vespalib/net/simple_component_config_producer.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/process/process.h> -#include <vespa/vespalib/net/state_explorer.h> -#include <vespa/vespalib/net/slime_explorer.h> -#include <vespa/vespalib/net/generic_state_handler.h> - -using namespace vespalib; - -//----------------------------------------------------------------------------- - -vespalib::string root_path = "/state/v1/"; -vespalib::string short_root_path = "/state/v1"; -vespalib::string metrics_path = "/state/v1/metrics"; -vespalib::string health_path = "/state/v1/health"; -vespalib::string config_path = "/state/v1/config"; - -vespalib::string total_metrics_path = "/metrics/total"; - -vespalib::string unknown_path = "/this/path/is/not/known"; -vespalib::string unknown_state_path = "/state/v1/this/path/is/not/known"; -vespalib::string my_path = "/my/path"; - -vespalib::string host_tag = "HOST"; -std::map<vespalib::string,vespalib::string> empty_params; - -//----------------------------------------------------------------------------- - -vespalib::string run_cmd(const vespalib::string &cmd) { - vespalib::string out; - ASSERT_TRUE(Process::run(cmd.c_str(), out)); - return out; -} - -vespalib::string getPage(int port, const vespalib::string &path, const vespalib::string &extra_params = "") { - return run_cmd(make_string("curl -s %s 'http://localhost:%d%s'", extra_params.c_str(), port, path.c_str())); -} - -vespalib::string getFull(int port, const vespalib::string &path) { return getPage(port, path, "-D -"); } - -//----------------------------------------------------------------------------- - -struct DummyHandler : JsonGetHandler { - vespalib::string result; - DummyHandler(const vespalib::string &result_in) : result(result_in) {} - vespalib::string get(const vespalib::string &, const vespalib::string &, - const std::map<vespalib::string,vespalib::string> &) const override - { - return result; - } -}; - -//----------------------------------------------------------------------------- - -TEST_F("require that unknown url returns 404 response", HttpServer(0)) { - std::string expect("HTTP/1.1 404 Not Found\r\n" - "Connection: close\r\n" - "\r\n"); - std::string actual = getFull(f1.port(), unknown_path); - EXPECT_EQUAL(expect, actual); -} - -TEST_FF("require that empty known url returns 404 response", DummyHandler(""), HttpServer(0)) { - auto token = f2.repo().bind(my_path, f1); - std::string expect("HTTP/1.1 404 Not Found\r\n" - "Connection: close\r\n" - "\r\n"); - std::string actual = getFull(f2.port(), my_path); - EXPECT_EQUAL(expect, actual); -} - -TEST_FF("require that non-empty known url returns expected headers", DummyHandler("[123]"), HttpServer(0)) { - auto token = f2.repo().bind(my_path, f1); - vespalib::string expect("HTTP/1.1 200 OK\r\n" - "Connection: close\r\n" - "Content-Type: application/json\r\n" - "Content-Length: 5\r\n" - "X-XSS-Protection: 1; mode=block\r\n" - "X-Frame-Options: DENY\r\n" - "Content-Security-Policy: default-src 'none'\r\n" - "X-Content-Type-Options: nosniff\r\n" - "Cache-Control: no-store\r\n" - "Pragma: no-cache\r\n" - "\r\n" - "[123]"); - std::string actual = getFull(f2.port(), my_path); - EXPECT_EQUAL(expect, actual); -} - -TEST_FFFF("require that handler is selected based on longest matching url prefix", - DummyHandler("[1]"), DummyHandler("[2]"), DummyHandler("[3]"), - HttpServer(0)) -{ - auto token2 = f4.repo().bind("/foo/bar", f2); - auto token1 = f4.repo().bind("/foo", f1); - auto token3 = f4.repo().bind("/foo/bar/baz", f3); - int port = f4.port(); - EXPECT_EQUAL("", getPage(port, "/fox")); - EXPECT_EQUAL("[1]", getPage(port, "/foo")); - EXPECT_EQUAL("[1]", getPage(port, "/foo/fox")); - EXPECT_EQUAL("[2]", getPage(port, "/foo/bar")); - EXPECT_EQUAL("[2]", getPage(port, "/foo/bar/fox")); - EXPECT_EQUAL("[3]", getPage(port, "/foo/bar/baz")); - EXPECT_EQUAL("[3]", getPage(port, "/foo/bar/baz/fox")); -} - -struct EchoHost : JsonGetHandler { - ~EchoHost() override; - vespalib::string get(const vespalib::string &host, const vespalib::string &, - const std::map<vespalib::string,vespalib::string> &) const override - { - return "[\"" + host + "\"]"; - } -}; - -EchoHost::~EchoHost() = default; - -TEST_FF("require that host is passed correctly", EchoHost(), HttpServer(0)) { - auto token = f2.repo().bind(my_path, f1); - EXPECT_EQUAL(make_string("%s:%d", HostName::get().c_str(), f2.port()), f2.host()); - vespalib::string default_result = make_string("[\"%s\"]", f2.host().c_str()); - vespalib::string localhost_result = make_string("[\"%s:%d\"]", "localhost", f2.port()); - vespalib::string silly_result = "[\"sillyserver\"]"; - EXPECT_EQUAL(localhost_result, run_cmd(make_string("curl -s http://localhost:%d/my/path", f2.port()))); - EXPECT_EQUAL(silly_result, run_cmd(make_string("curl -s http://localhost:%d/my/path -H \"Host: sillyserver\"", f2.port()))); - EXPECT_EQUAL(default_result, run_cmd(make_string("curl -s http://localhost:%d/my/path -H \"Host:\"", f2.port()))); -} - -struct SamplingHandler : JsonGetHandler { - mutable std::mutex my_lock; - mutable vespalib::string my_host; - mutable vespalib::string my_path; - mutable std::map<vespalib::string,vespalib::string> my_params; - ~SamplingHandler() override; - vespalib::string get(const vespalib::string &host, const vespalib::string &path, - const std::map<vespalib::string,vespalib::string> ¶ms) const override - { - { - auto guard = std::lock_guard(my_lock); - my_host = host; - my_path = path; - my_params = params; - } - return "[]"; - } -}; - -SamplingHandler::~SamplingHandler() = default; - -TEST_FF("require that request parameters can be inspected", SamplingHandler(), HttpServer(0)) -{ - auto token = f2.repo().bind("/foo", f1); - EXPECT_EQUAL("[]", getPage(f2.port(), "/foo?a=b&x=y&z")); - { - auto guard = std::lock_guard(f1.my_lock); - EXPECT_EQUAL(f1.my_path, "/foo"); - EXPECT_EQUAL(f1.my_params.size(), 3u); - EXPECT_EQUAL(f1.my_params["a"], "b"); - EXPECT_EQUAL(f1.my_params["x"], "y"); - EXPECT_EQUAL(f1.my_params["z"], ""); - EXPECT_EQUAL(f1.my_params.size(), 3u); // "z" was present - } -} - -TEST_FF("require that request path is dequoted", SamplingHandler(), HttpServer(0)) -{ - auto token = f2.repo().bind("/[foo]", f1); - EXPECT_EQUAL("[]", getPage(f2.port(), "/%5bfoo%5D")); - { - auto guard = std::lock_guard(f1.my_lock); - EXPECT_EQUAL(f1.my_path, "/[foo]"); - EXPECT_EQUAL(f1.my_params.size(), 0u); - } -} - -//----------------------------------------------------------------------------- - -TEST_FFFF("require that the state server wires the appropriate url prefixes", - SimpleHealthProducer(), SimpleMetricsProducer(), SimpleComponentConfigProducer(), - StateServer(0, f1, f2, f3)) -{ - f2.setTotalMetrics("{}"); // avoid empty result - int port = f4.getListenPort(); - EXPECT_TRUE(getFull(port, short_root_path).find("HTTP/1.1 200 OK") == 0); - EXPECT_TRUE(getFull(port, total_metrics_path).find("HTTP/1.1 200 OK") == 0); - EXPECT_TRUE(getFull(port, unknown_path).find("HTTP/1.1 404 Not Found") == 0); -} - -TEST_FFFF("require that the state server exposes the state api handler repo", - SimpleHealthProducer(), SimpleMetricsProducer(), SimpleComponentConfigProducer(), - StateServer(0, f1, f2, f3)) -{ - int port = f4.getListenPort(); - vespalib::string page1 = getPage(port, root_path); - auto token = f4.repo().add_root_resource("state/v1/custom"); - vespalib::string page2 = getPage(port, root_path); - EXPECT_NOT_EQUAL(page1, page2); - token.reset(); - vespalib::string page3 = getPage(port, root_path); - EXPECT_EQUAL(page3, page1); -} - -//----------------------------------------------------------------------------- - -TEST_FFFF("require that json handlers can be removed from repo", - DummyHandler("[1]"), DummyHandler("[2]"), DummyHandler("[3]"), - JsonHandlerRepo()) -{ - auto token1 = f4.bind("/foo", f1); - auto token2 = f4.bind("/foo/bar", f2); - auto token3 = f4.bind("/foo/bar/baz", f3); - std::map<vespalib::string,vespalib::string> params; - EXPECT_EQUAL("[1]", f4.get("", "/foo", params)); - EXPECT_EQUAL("[2]", f4.get("", "/foo/bar", params)); - EXPECT_EQUAL("[3]", f4.get("", "/foo/bar/baz", params)); - token2.reset(); - EXPECT_EQUAL("[1]", f4.get("", "/foo", params)); - EXPECT_EQUAL("[1]", f4.get("", "/foo/bar", params)); - EXPECT_EQUAL("[3]", f4.get("", "/foo/bar/baz", params)); -} - -TEST_FFFF("require that json handlers can be shadowed", - DummyHandler("[1]"), DummyHandler("[2]"), DummyHandler("[3]"), - JsonHandlerRepo()) -{ - auto token1 = f4.bind("/foo", f1); - auto token2 = f4.bind("/foo/bar", f2); - std::map<vespalib::string,vespalib::string> params; - EXPECT_EQUAL("[1]", f4.get("", "/foo", params)); - EXPECT_EQUAL("[2]", f4.get("", "/foo/bar", params)); - auto token3 = f4.bind("/foo/bar", f3); - EXPECT_EQUAL("[3]", f4.get("", "/foo/bar", params)); - token3.reset(); - EXPECT_EQUAL("[2]", f4.get("", "/foo/bar", params)); -} - -TEST_F("require that root resources can be tracked", JsonHandlerRepo()) -{ - EXPECT_TRUE(std::vector<vespalib::string>({}) == f1.get_root_resources()); - auto token1 = f1.add_root_resource("/health"); - EXPECT_TRUE(std::vector<vespalib::string>({"/health"}) == f1.get_root_resources()); - auto token2 = f1.add_root_resource("/config"); - EXPECT_TRUE(std::vector<vespalib::string>({"/health", "/config"}) == f1.get_root_resources()); - auto token3 = f1.add_root_resource("/custom/foo"); - EXPECT_TRUE(std::vector<vespalib::string>({"/health", "/config", "/custom/foo"}) == f1.get_root_resources()); - token2.reset(); - EXPECT_TRUE(std::vector<vespalib::string>({"/health", "/custom/foo"}) == f1.get_root_resources()); -} - -//----------------------------------------------------------------------------- - -TEST_FFFF("require that state api responds to the expected paths", - SimpleHealthProducer(), SimpleMetricsProducer(), SimpleComponentConfigProducer(), - StateApi(f1, f2, f3)) -{ - f2.setTotalMetrics("{}"); // avoid empty result - EXPECT_TRUE(!f4.get(host_tag, short_root_path, empty_params).empty()); - EXPECT_TRUE(!f4.get(host_tag, root_path, empty_params).empty()); - EXPECT_TRUE(!f4.get(host_tag, health_path, empty_params).empty()); - EXPECT_TRUE(!f4.get(host_tag, metrics_path, empty_params).empty()); - EXPECT_TRUE(!f4.get(host_tag, config_path, empty_params).empty()); - EXPECT_TRUE(!f4.get(host_tag, total_metrics_path, empty_params).empty()); - EXPECT_TRUE(f4.get(host_tag, unknown_path, empty_params).empty()); - EXPECT_TRUE(f4.get(host_tag, unknown_state_path, empty_params).empty()); -} - -TEST_FFFF("require that top-level urls are generated correctly", - SimpleHealthProducer(), SimpleMetricsProducer(), SimpleComponentConfigProducer(), - StateApi(f1, f2, f3)) -{ - EXPECT_EQUAL("{\"resources\":[" - "{\"url\":\"http://HOST/state/v1/health\"}," - "{\"url\":\"http://HOST/state/v1/metrics\"}," - "{\"url\":\"http://HOST/state/v1/config\"}]}", - f4.get(host_tag, root_path, empty_params)); - EXPECT_EQUAL(f4.get(host_tag, root_path, empty_params), - f4.get(host_tag, short_root_path, empty_params)); -} - -TEST_FFFF("require that top-level resource list can be extended", - SimpleHealthProducer(), SimpleMetricsProducer(), SimpleComponentConfigProducer(), - StateApi(f1, f2, f3)) -{ - auto token = f4.repo().add_root_resource("/state/v1/custom"); - EXPECT_EQUAL("{\"resources\":[" - "{\"url\":\"http://HOST/state/v1/health\"}," - "{\"url\":\"http://HOST/state/v1/metrics\"}," - "{\"url\":\"http://HOST/state/v1/config\"}," - "{\"url\":\"http://HOST/state/v1/custom\"}]}", - f4.get(host_tag, root_path, empty_params)); -} - -TEST_FFFF("require that health resource works as expected", - SimpleHealthProducer(), SimpleMetricsProducer(), SimpleComponentConfigProducer(), - StateApi(f1, f2, f3)) -{ - EXPECT_EQUAL("{\"status\":{\"code\":\"up\"}}", - f4.get(host_tag, health_path, empty_params)); - f1.setFailed("FAIL MSG"); - EXPECT_EQUAL("{\"status\":{\"code\":\"down\",\"message\":\"FAIL MSG\"}}", - f4.get(host_tag, health_path, empty_params)); -} - -TEST_FFFF("require that metrics resource works as expected", - SimpleHealthProducer(), SimpleMetricsProducer(), SimpleComponentConfigProducer(), - StateApi(f1, f2, f3)) -{ - EXPECT_EQUAL("{\"status\":{\"code\":\"up\"}}", - f4.get(host_tag, metrics_path, empty_params)); - f1.setFailed("FAIL MSG"); - EXPECT_EQUAL("{\"status\":{\"code\":\"down\",\"message\":\"FAIL MSG\"}}", - f4.get(host_tag, metrics_path, empty_params)); - f1.setOk(); - f2.setMetrics("{\"foo\":\"bar\"}"); - EXPECT_EQUAL("{\"status\":{\"code\":\"up\"},\"metrics\":{\"foo\":\"bar\"}}", - f4.get(host_tag, metrics_path, empty_params)); -} - -TEST_FFFF("require that config resource works as expected", - SimpleHealthProducer(), SimpleMetricsProducer(), SimpleComponentConfigProducer(), - StateApi(f1, f2, f3)) -{ - EXPECT_EQUAL("{\"config\":{}}", - f4.get(host_tag, config_path, empty_params)); - f3.addConfig(SimpleComponentConfigProducer::Config("foo", 3)); - EXPECT_EQUAL("{\"config\":{\"generation\":3,\"foo\":{\"generation\":3}}}", - f4.get(host_tag, config_path, empty_params)); - f3.addConfig(SimpleComponentConfigProducer::Config("foo", 4)); - f3.addConfig(SimpleComponentConfigProducer::Config("bar", 4, "error")); - EXPECT_EQUAL("{\"config\":{\"generation\":4,\"bar\":{\"generation\":4,\"message\":\"error\"},\"foo\":{\"generation\":4}}}", - f4.get(host_tag, config_path, empty_params)); - f3.removeConfig("bar"); - EXPECT_EQUAL("{\"config\":{\"generation\":4,\"foo\":{\"generation\":4}}}", - f4.get(host_tag, config_path, empty_params)); -} - -TEST_FFFF("require that state api also can return total metric", - SimpleHealthProducer(), SimpleMetricsProducer(), SimpleComponentConfigProducer(), - StateApi(f1, f2, f3)) -{ - f2.setTotalMetrics("{\"foo\":\"bar\"}"); - EXPECT_EQUAL("{\"foo\":\"bar\"}", - f4.get(host_tag, total_metrics_path, empty_params)); -} - -TEST_FFFFF("require that custom handlers can be added to the state server", - SimpleHealthProducer(), SimpleMetricsProducer(), SimpleComponentConfigProducer(), - StateApi(f1, f2, f3), DummyHandler("[123]")) -{ - EXPECT_EQUAL("", f4.get(host_tag, my_path, empty_params)); - auto token = f4.repo().bind(my_path, f5); - EXPECT_EQUAL("[123]", f4.get(host_tag, my_path, empty_params)); - token.reset(); - EXPECT_EQUAL("", f4.get(host_tag, my_path, empty_params)); -} - -struct EchoConsumer : MetricsProducer { - ~EchoConsumer() override; - vespalib::string getMetrics(const vespalib::string &consumer) override { - return "[\"" + consumer + "\"]"; - } - vespalib::string getTotalMetrics(const vespalib::string &consumer) override { - return "[\"" + consumer + "\"]"; - } -}; - -EchoConsumer::~EchoConsumer() = default; - -TEST_FFFF("require that empty v1 metrics consumer defaults to 'statereporter'", - SimpleHealthProducer(), EchoConsumer(), SimpleComponentConfigProducer(), - StateApi(f1, f2, f3)) -{ - std::map<vespalib::string,vespalib::string> my_params; - EXPECT_EQUAL("{\"status\":{\"code\":\"up\"},\"metrics\":[\"statereporter\"]}", f4.get(host_tag, metrics_path, empty_params)); -} - -TEST_FFFF("require that empty total metrics consumer defaults to the empty string", - SimpleHealthProducer(), EchoConsumer(), SimpleComponentConfigProducer(), - StateApi(f1, f2, f3)) -{ - std::map<vespalib::string,vespalib::string> my_params; - EXPECT_EQUAL("[\"\"]", f4.get(host_tag, total_metrics_path, empty_params)); -} - -TEST_FFFF("require that metrics consumer is passed correctly", - SimpleHealthProducer(), EchoConsumer(), SimpleComponentConfigProducer(), - StateApi(f1, f2, f3)) -{ - std::map<vespalib::string,vespalib::string> my_params; - my_params["consumer"] = "ME"; - EXPECT_EQUAL("{\"status\":{\"code\":\"up\"},\"metrics\":[\"ME\"]}", f4.get(host_tag, metrics_path, my_params)); - EXPECT_EQUAL("[\"ME\"]", f4.get(host_tag, total_metrics_path, my_params)); -} - -void check_json(const vespalib::string &expect_json, const vespalib::string &actual_json) { - Slime expect_slime; - Slime actual_slime; - EXPECT_TRUE(slime::JsonFormat::decode(expect_json, expect_slime) > 0); - EXPECT_TRUE(slime::JsonFormat::decode(actual_json, actual_slime) > 0); - EXPECT_EQUAL(expect_slime, actual_slime); -} - -TEST("require that generic state can be explored") { - vespalib::string json_model = - "{" - " foo: 'bar'," - " cnt: 123," - " engine: {" - " up: 'yes'," - " stats: {" - " latency: 5," - " qps: 100" - " }" - " }," - " list: {" - " one: {" - " size: {" - " value: 1" - " }" - " }," - " two: {" - " size: 2" - " }" - " }" - "}"; - vespalib::string json_root = - "{" - " full: true," - " foo: 'bar'," - " cnt: 123," - " engine: {" - " up: 'yes'," - " url: 'http://HOST/state/v1/engine'" - " }," - " list: {" - " one: {" - " size: {" - " value: 1," - " url: 'http://HOST/state/v1/list/one/size'" - " }" - " }," - " two: {" - " size: 2," - " url: 'http://HOST/state/v1/list/two'" - " }" - " }" - "}"; - vespalib::string json_engine = - "{" - " full: true," - " up: 'yes'," - " stats: {" - " latency: 5," - " qps: 100," - " url: 'http://HOST/state/v1/engine/stats'" - " }" - "}"; - vespalib::string json_engine_stats = - "{" - " full: true," - " latency: 5," - " qps: 100" - "}"; - vespalib::string json_list = - "{" - " one: {" - " size: {" - " value: 1," - " url: 'http://HOST/state/v1/list/one/size'" - " }" - " }," - " two: {" - " size: 2," - " url: 'http://HOST/state/v1/list/two'" - " }" - "}"; - vespalib::string json_list_one = - "{" - " size: {" - " value: 1," - " url: 'http://HOST/state/v1/list/one/size'" - " }" - "}"; - vespalib::string json_list_one_size = "{ full: true, value: 1 }"; - vespalib::string json_list_two = "{ full: true, size: 2 }"; - //------------------------------------------------------------------------- - Slime slime_state; - EXPECT_TRUE(slime::JsonFormat::decode(json_model, slime_state) > 0); - SlimeExplorer slime_explorer(slime_state.get()); - GenericStateHandler state_handler(short_root_path, slime_explorer); - EXPECT_EQUAL("", state_handler.get(host_tag, unknown_path, empty_params)); - EXPECT_EQUAL("", state_handler.get(host_tag, unknown_state_path, empty_params)); - check_json(json_root, state_handler.get(host_tag, root_path, empty_params)); - check_json(json_engine, state_handler.get(host_tag, root_path + "engine", empty_params)); - check_json(json_engine_stats, state_handler.get(host_tag, root_path + "engine/stats", empty_params)); - check_json(json_list, state_handler.get(host_tag, root_path + "list", empty_params)); - check_json(json_list_one, state_handler.get(host_tag, root_path + "list/one", empty_params)); - check_json(json_list_one_size, state_handler.get(host_tag, root_path + "list/one/size", empty_params)); - check_json(json_list_two, state_handler.get(host_tag, root_path + "list/two", empty_params)); -} - -TEST_MAIN() { - mkdir("var", S_IRWXU); - mkdir("var/run", S_IRWXU); - TEST_RUN_ALL(); - rmdir("var/run"); - rmdir("var"); -} diff --git a/staging_vespalib/src/vespa/vespalib/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/CMakeLists.txt index 43492d06689..053c04064ca 100644 --- a/staging_vespalib/src/vespa/vespalib/CMakeLists.txt +++ b/staging_vespalib/src/vespa/vespalib/CMakeLists.txt @@ -1,13 +1,9 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(staging_vespalib SOURCES - $<TARGET_OBJECTS:staging_vespalib_vespalib_encoding> $<TARGET_OBJECTS:staging_vespalib_vespalib_util> - $<TARGET_OBJECTS:staging_vespalib_vespalib_data> - $<TARGET_OBJECTS:staging_vespalib_vespalib_metrics> $<TARGET_OBJECTS:staging_vespalib_vespalib_objects> $<TARGET_OBJECTS:staging_vespalib_vespalib_stllike> - $<TARGET_OBJECTS:staging_vespalib_vespalib_net> INSTALL lib64 DEPENDS ) diff --git a/staging_vespalib/src/vespa/vespalib/data/.gitignore b/staging_vespalib/src/vespa/vespalib/data/.gitignore deleted file mode 100644 index 0b3af54ee50..00000000000 --- a/staging_vespalib/src/vespa/vespalib/data/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.So -.depend* -Makefile diff --git a/staging_vespalib/src/vespa/vespalib/data/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/data/CMakeLists.txt deleted file mode 100644 index d5adeadfb3b..00000000000 --- a/staging_vespalib/src/vespa/vespalib/data/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(staging_vespalib_vespalib_data OBJECT - SOURCES - fileheader.cpp - DEPENDS -) diff --git a/staging_vespalib/src/vespa/vespalib/data/fileheader.cpp b/staging_vespalib/src/vespa/vespalib/data/fileheader.cpp deleted file mode 100644 index 0cb5fa14ff4..00000000000 --- a/staging_vespalib/src/vespa/vespalib/data/fileheader.cpp +++ /dev/null @@ -1,594 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "fileheader.h" -#include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/data/databuffer.h> -#include <vespa/fastos/file.h> - -#include <vespa/log/log.h> -LOG_SETUP(".fileheader"); - -namespace vespalib { - -VESPA_IMPLEMENT_EXCEPTION(IllegalHeaderException, vespalib::Exception); - -const uint32_t GenericHeader::MAGIC(0x5ca1ab1e); -const uint32_t GenericHeader::VERSION(1); -const GenericHeader::Tag GenericHeader::EMPTY; -const size_t ALIGNMENT=0x1000; - -GenericHeader::Tag::~Tag() = default; -GenericHeader::Tag::Tag(const Tag &) = default; -GenericHeader::Tag & GenericHeader::Tag::operator=(const Tag &) = default; - -GenericHeader::Tag::Tag() : - _type(TYPE_EMPTY), - _name(""), - _fVal(0), - _iVal(0), - _sVal("") -{ - // empty -} - -GenericHeader::Tag::Tag(const vespalib::string &name, float val) : - _type(TYPE_FLOAT), - _name(name), - _fVal(val), - _iVal(0), - _sVal("") -{ - // empty -} - -GenericHeader::Tag::Tag(const vespalib::string &name, double val) : - _type(TYPE_FLOAT), - _name(name), - _fVal(val), - _iVal(0), - _sVal("") -{ - // empty -} - -GenericHeader::Tag::Tag(const vespalib::string &name, int8_t val) : - _type(TYPE_INTEGER), - _name(name), - _fVal(0), - _iVal(val), - _sVal("") -{ - // empty -} - -GenericHeader::Tag::Tag(const vespalib::string &name, uint8_t val) : - _type(TYPE_INTEGER), - _name(name), - _fVal(0), - _iVal(val), - _sVal("") -{ - // empty -} - -GenericHeader::Tag::Tag(const vespalib::string &name, int16_t val) : - _type(TYPE_INTEGER), - _name(name), - _fVal(0), - _iVal(val), - _sVal("") -{ - // empty -} - -GenericHeader::Tag::Tag(const vespalib::string &name, uint16_t val) : - _type(TYPE_INTEGER), - _name(name), - _fVal(0), - _iVal(val), - _sVal("") -{ - // empty -} - -GenericHeader::Tag::Tag(const vespalib::string &name, int32_t val) : - _type(TYPE_INTEGER), - _name(name), - _fVal(0), - _iVal(val), - _sVal("") -{ - // empty -} - -GenericHeader::Tag::Tag(const vespalib::string &name, uint32_t val) : - _type(TYPE_INTEGER), - _name(name), - _fVal(0), - _iVal(val), - _sVal("") -{ - // empty -} - -GenericHeader::Tag::Tag(const vespalib::string &name, int64_t val) : - _type(TYPE_INTEGER), - _name(name), - _fVal(0), - _iVal(val), - _sVal("") -{ - // empty -} - -GenericHeader::Tag::Tag(const vespalib::string &name, uint64_t val) : - _type(TYPE_INTEGER), - _name(name), - _fVal(0), - _iVal(val), - _sVal("") -{ - // empty -} - -GenericHeader::Tag::Tag(const vespalib::string &name, bool val) - : _type(TYPE_INTEGER), - _name(name), - _fVal(0), - _iVal(val ? 1 : 0), - _sVal("") -{ - // empty -} - -GenericHeader::Tag::Tag(const vespalib::string &name, const char *val) - : _type(TYPE_STRING), - _name(name), - _fVal(0), - _iVal(0), - _sVal(val) -{ - // empty -} - -GenericHeader::Tag::Tag(const vespalib::string &name, const vespalib::string &val) : - _type(TYPE_STRING), - _name(name), - _fVal(0), - _iVal(0), - _sVal(val) -{ - // empty -} - -size_t -GenericHeader::Tag::getSize() const -{ - size_t ret = _name.size() + 2; - switch (_type) { - case TYPE_FLOAT: - case TYPE_INTEGER: - ret += 8; - break; - case TYPE_STRING: - ret += _sVal.size() + 1; - break; - default: - LOG_ASSERT(false); - } - return ret; -} - -size_t -GenericHeader::Tag::read(DataBuffer &buf) -{ - char *pos = buf.getData(); - vespalib::string name(pos); - buf.moveDataToDead(name.size() + 1); - uint8_t type = buf.readInt8(); - switch (type) { - case TYPE_FLOAT: - _fVal = buf.readDouble(); - break; - case TYPE_INTEGER: - _iVal = buf.readInt64(); - break; - case TYPE_STRING: - _sVal = vespalib::string(buf.getData()); - buf.moveDataToDead(_sVal.size() + 1); - break; - default: - throw IllegalHeaderException("Can not deserialize empty tag."); - } - _name = name; // assign here for exception safety - _type = (Type)type; - return buf.getData() - pos; -} - -size_t -GenericHeader::Tag::write(DataBuffer &buf) const -{ - int pos = buf.getDataLen(); - buf.writeBytes(_name.c_str(), _name.size() + 1); - buf.writeInt8(_type); - switch (_type) { - case TYPE_FLOAT: - buf.writeDouble(_fVal); - break; - case TYPE_INTEGER: - buf.writeInt64(_iVal); - break; - case TYPE_STRING: - buf.writeBytes(_sVal.c_str(), _sVal.size() + 1); - break; - default: - LOG_ASSERT(false); - } - return buf.getDataLen() - pos; -} - -GenericHeader::BufferReader::BufferReader(DataBuffer &buf) : - _buf(buf) -{ - // empty -} - -size_t -GenericHeader::BufferReader::getData(char *buf, size_t len) -{ - if (len > _buf.getDataLen()) { - len = _buf.getDataLen(); - } - _buf.readBytes(buf, len); - return len; -} - -GenericHeader::BufferWriter::BufferWriter(DataBuffer &buf) : - _buf(buf) -{ - // empty -} - -size_t -GenericHeader::BufferWriter::putData(const char *buf, size_t len) -{ - if (len > _buf.getFreeLen()) { - len = _buf.getFreeLen(); - } - _buf.writeBytes(buf, len); - return len; -} - - -GenericHeader::MMapReader::MMapReader(const char *buf, size_t sz) - : _buf(buf), - _sz(sz) -{ -} - - -size_t -GenericHeader::MMapReader::getData(char *buf, size_t len) -{ - size_t clen = std::min(len, _sz); - memcpy(buf, _buf, clen); - _buf += clen; - _sz -= clen; - return clen; -} - - -GenericHeader::GenericHeader() : - _tags() -{ - // empty -} - -GenericHeader::~GenericHeader() { } - -const GenericHeader::Tag & -GenericHeader::getTag(size_t idx) const -{ - if (idx >= _tags.size()) { - return EMPTY; - } - TagMap::const_iterator it = _tags.begin(); - std::advance(it, idx); - return it->second; -} - -const GenericHeader::Tag & -GenericHeader::getTag(const vespalib::string &key) const -{ - TagMap::const_iterator it = _tags.find(key); - if (it == _tags.end()) { - return EMPTY; - } - return it->second; -} - -bool -GenericHeader::hasTag(const vespalib::string &key) const -{ - return _tags.find(key) != _tags.end(); -} - -bool -GenericHeader::putTag(const GenericHeader::Tag &tag) -{ - const vespalib::string &key = tag.getName(); - TagMap::iterator it = _tags.find(key); - if (it != _tags.end()) { - it->second = tag; - return false; - } - _tags.insert(TagMap::value_type(key, tag)); - return true; -} -bool -GenericHeader::removeTag(const vespalib::string &key) -{ - TagMap::iterator it = _tags.find(key); - if (it == _tags.end()) { - return false; - } - _tags.erase(it); - return true; -} - - -size_t -GenericHeader::getMinSize(void) -{ - return 4 /* magic */ + 4 /* size */ + 4 /* version */ + 4 /* num tags */; -} - - -size_t -GenericHeader::getSize() const -{ - size_t ret = getMinSize(); - for (TagMap::const_iterator it = _tags.begin(); - it != _tags.end(); ++it) - { - ret += it->second.getSize(); - } - return ret; -} - - -size_t -GenericHeader::readSize(IDataReader &reader) -{ - size_t hhSize = getMinSize(); - DataBuffer buf(hhSize, ALIGNMENT); - size_t numBytesRead = reader.getData(buf.getFree(), hhSize); - buf.moveFreeToData(numBytesRead); - - if (numBytesRead < hhSize) { - throw IllegalHeaderException("Failed to read header info."); - } - uint32_t magic = buf.readInt32(); - if (magic != MAGIC) { - throw IllegalHeaderException("Failed to verify magic bits."); - } - uint32_t numBytesTotal = buf.readInt32(); - if (numBytesTotal == 0) { - throw IllegalHeaderException("Failed to read header size."); - } - if (numBytesTotal < getMinSize()) { - throw IllegalHeaderException("Failed to verify header size."); - } - uint32_t version = buf.readInt32(); - if (version != VERSION) { - throw IllegalHeaderException("Failed to verify header version."); - } - return numBytesTotal; -} - - -size_t -GenericHeader::read(IDataReader &reader) -{ - size_t bufLen = 1024 * 32; - DataBuffer buf(bufLen, ALIGNMENT); - size_t numBytesRead = reader.getData(buf.getFree(), bufLen); - buf.moveFreeToData(numBytesRead); - - if (numBytesRead < 4 /* magic */ + 4 /* size */) { - throw IllegalHeaderException("Failed to read header info."); - } - uint32_t magic = buf.readInt32(); - if (magic != MAGIC) { - throw IllegalHeaderException("Failed to verify magic bits."); - } - uint32_t numBytesTotal = buf.readInt32(); - if (numBytesTotal == 0) { - throw IllegalHeaderException("Failed to read header size."); - } - if (numBytesTotal < getMinSize()) { - throw IllegalHeaderException("Failed to verify header size."); - } - if (numBytesRead < numBytesTotal) { - LOG(debug, "Read %d of %d header bytes, performing backfill.", - (uint32_t)numBytesRead, numBytesTotal); - uint32_t numBytesRemain = numBytesTotal - numBytesRead; - buf.ensureFree(numBytesRemain); - LOG(debug, "Reading remaining %d bytes of header.", numBytesRemain); - numBytesRead += reader.getData(buf.getFree(), numBytesRemain); - if (numBytesRead != numBytesTotal) { - throw IllegalHeaderException("Failed to read full header."); - } - buf.moveFreeToData(numBytesRemain); - } else { - buf.moveDataToFree(numBytesRead - numBytesTotal); - } - - uint32_t version = buf.readInt32(); - if (version != VERSION) { - throw IllegalHeaderException("Failed to verify header version."); - } - uint32_t numTags = buf.readInt32(); - TagMap tags; - for (uint32_t i = 0; i < numTags; ++i) { - Tag tag; - tag.read(buf); - tags.insert(TagMap::value_type(tag.getName(), tag)); - } - _tags.swap(tags); - return numBytesTotal; -} - -size_t -GenericHeader::write(IDataWriter &writer) const -{ - size_t numBytesTotal = getSize(); - DataBuffer buf(numBytesTotal, ALIGNMENT); - buf.writeInt32(MAGIC); - buf.writeInt32((uint32_t)numBytesTotal); - buf.writeInt32(VERSION); - buf.writeInt32((uint32_t)_tags.size()); - uint32_t numBytesInBuf = 16; - for (TagMap::const_iterator it = _tags.begin(); - it != _tags.end(); ++it) - { - numBytesInBuf += it->second.write(buf); - } - if (numBytesInBuf < numBytesTotal) { - buf.zeroFill(numBytesTotal - numBytesInBuf); - } - size_t numBytesWritten = writer.putData(buf.getData(), numBytesTotal); - if (numBytesWritten != numBytesTotal) { - throw IllegalHeaderException("Failed to write header."); - } - return numBytesWritten; -} - -FileHeader::FileReader::FileReader(FastOS_FileInterface &file) : - _file(file) -{ - // empty -} - -size_t -FileHeader::FileReader::getData(char *buf, size_t len) -{ - LOG_ASSERT(_file.IsOpened()); - LOG_ASSERT(_file.IsReadMode()); - - return _file.Read(buf, len); -} - -FileHeader::FileWriter::FileWriter(FastOS_FileInterface &file) : - _file(file) -{ - // empty -} - -size_t -FileHeader::FileWriter::putData(const char *buf, size_t len) -{ - LOG_ASSERT(_file.IsOpened()); - LOG_ASSERT(_file.IsWriteMode()); - - return _file.Write2(buf, len); -} - -FileHeader::FileHeader(size_t alignTo, size_t minSize) : - _alignTo(alignTo), - _minSize(minSize), - _fileSize(0) -{ - // empty -} - -size_t -FileHeader::getSize() const -{ - size_t ret = GenericHeader::getSize(); - if (_fileSize > ret) { - return _fileSize; - } - if (_minSize > ret) { - return _minSize; - } - size_t pad = ret % _alignTo; - return ret + (pad > 0 ? _alignTo - pad : 0); -} - -size_t -FileHeader::readFile(FastOS_FileInterface &file) -{ - FileReader reader(file); - return GenericHeader::read(reader); -} - -size_t -FileHeader::writeFile(FastOS_FileInterface &file) const -{ - FileWriter writer(file); - return GenericHeader::write(writer); -} - -size_t -FileHeader::rewriteFile(FastOS_FileInterface &file) -{ - LOG_ASSERT(file.IsOpened()); - LOG_ASSERT(file.IsReadMode()); - LOG_ASSERT(file.IsWriteMode()); - - // Store current position in file. - int64_t pos = file.GetPosition(); - if (pos != 0) { - file.SetPosition(0); - } - - // Assert that header size agrees with file content. - FileReader reader(file); - size_t wantSize = 4 /* magic */ + 4 /* size */; - DataBuffer buf(wantSize, ALIGNMENT); - size_t numBytesRead = reader.getData(buf.getFree(), wantSize); - if (numBytesRead < wantSize) { - throw IllegalHeaderException("Failed to read header info."); - } - uint32_t magic = buf.readInt32(); - if (magic != MAGIC) { - throw IllegalHeaderException("Failed to verify magic bits."); - } - uint32_t size = buf.readInt32(); - if (size == 0) { - throw IllegalHeaderException("Failed to read header size."); - } - if (size < GenericHeader::getSize()) { - throw IllegalHeaderException("Failed to rewrite resized header."); - } - _fileSize = size; - - // Write new header and reset file position. - file.SetPosition(0); - size_t ret = writeFile(file); - if (file.GetPosition() != pos) { - file.SetPosition(pos); - } - return ret; -} - -vespalib::asciistream & -operator<<(vespalib::asciistream &out, const GenericHeader::Tag &tag) -{ - switch (tag.getType()) { - case GenericHeader::Tag::TYPE_FLOAT: - out << tag.asFloat(); - break; - case GenericHeader::Tag::TYPE_INTEGER: - out << tag.asInteger(); - break; - case GenericHeader::Tag::TYPE_STRING: - out << tag.asString(); - break; - default: - LOG_ASSERT(false); - } - return out; -} - -} // namespace diff --git a/staging_vespalib/src/vespa/vespalib/data/fileheader.h b/staging_vespalib/src/vespa/vespalib/data/fileheader.h deleted file mode 100644 index ca475971932..00000000000 --- a/staging_vespalib/src/vespa/vespalib/data/fileheader.h +++ /dev/null @@ -1,340 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/vespalib/util/exception.h> -#include <map> - -class FastOS_FileInterface; - -namespace vespalib { - -class DataBuffer; -class asciistream; - -/** - * This exception can be thrown when serializing or deserializing header content. - */ -VESPA_DEFINE_EXCEPTION(IllegalHeaderException, vespalib::Exception); - -/** - * This class implements a collection of GenericHeader::Tag objects that can be set and retrieved by name. The - * interfaces GenericHeader::IDataReader or GenericHeader::IDataWriter define the api to allow deserialization - * and serialization of an instance of this class to any underlying data buffer. - */ -class GenericHeader { -public: - static const uint32_t MAGIC; - static const uint32_t VERSION; - - /** - * This class implements a immutable named value of a specified type. This type can be any of - * Tag::Type. There is no enforcement of type, so using asInteger() on a TYPE_STRING instance will simple - * return the default integer value. - */ - class Tag { - public: - enum Type { - TYPE_EMPTY = 'e', - TYPE_FLOAT = 'f', - TYPE_INTEGER = 'i', - TYPE_STRING = 's' - }; - - private: - Type _type; - vespalib::string _name; - double _fVal; - int64_t _iVal; - vespalib::string _sVal; - - public: - Tag(); - Tag(const Tag &); - Tag & operator=(const Tag &); - Tag(const vespalib::string &name, float val); - Tag(const vespalib::string &name, double val); - Tag(const vespalib::string &name, int8_t val); - Tag(const vespalib::string &name, uint8_t val); - Tag(const vespalib::string &name, int16_t val); - Tag(const vespalib::string &name, uint16_t val); - Tag(const vespalib::string &name, int32_t val); - Tag(const vespalib::string &name, uint32_t val); - Tag(const vespalib::string &name, int64_t val); - Tag(const vespalib::string &name, uint64_t val); - Tag(const vespalib::string &name, bool val); - Tag(const vespalib::string &name, const char *val); - Tag(const vespalib::string &name, const vespalib::string &val); - ~Tag(); - - size_t read(DataBuffer &buf); - size_t write(DataBuffer &buf) const; - size_t getSize() const; - - bool isEmpty() const { return _type == TYPE_EMPTY; } - Type getType() const { return _type; } - const vespalib::string &getName() const { return _name; } - - double asFloat() const { return _fVal; } - int64_t asInteger() const { return _iVal; } - const vespalib::string &asString() const { return _sVal; } - bool asBool() const { return _iVal != 0; } - }; - - /** - * This class defines the interface used by GenericHeader to deserialize content. It has implementation - * GenericHeader::BufferReader for reading from a buffer, and FileHeader::FileReader for reading from a - * file. - */ - class IDataReader { - public: - virtual ~IDataReader() { /* empty */ } - virtual size_t getData(char *buf, size_t len) = 0; - }; - - /** - * This class defines the interface used by GenericHeader to serialize content. It has implementation - * GenericHeader::BufferWriter for reading from a buffer, and FileHeader::FileWriter for reading from a - * file. - */ - class IDataWriter { - public: - virtual ~IDataWriter() { /* empty */ } - virtual size_t putData(const char *buf, size_t len) = 0; - }; - - /** - * Implements the GenericHeader::IDataReader interface for deserializing header content from a - * DataBuffer instance. - */ - class BufferReader : public IDataReader { - private: - DataBuffer &_buf; - - public: - BufferReader(DataBuffer &buf); - size_t getData(char *buf, size_t len) override; - }; - - /** - * Implements the GenericHeader::IDataWriter interface for serializing header content to a - * DataBuffer instance. - */ - class BufferWriter : public IDataWriter { - private: - DataBuffer &_buf; - - public: - BufferWriter(DataBuffer &buf); - size_t putData(const char *buf, size_t len) override; - }; - - class MMapReader : public IDataReader - { - const char *_buf; - size_t _sz; - - public: - MMapReader(const char *buf, size_t sz); - - size_t getData(char *buf, size_t len) override; - }; - -private: - static const Tag EMPTY; - - typedef std::map<vespalib::string, Tag> TagMap; - TagMap _tags; - -public: - /** - * Constructs a new instance of this class. - */ - GenericHeader(); - - /** - * Virtual destructor required for inheritance. - */ - virtual ~GenericHeader(); - - /** - * Returns the number of tags contained in this header. - * - * @return The size of the tag map. - */ - size_t getNumTags() const { return _tags.size(); } - - /** - * Returns the tag at the given index. This can be used along with getNumTags() to iterate over all the - * tags in a header. This is not an efficient way of accessing tags, since the underlying map does not - * support random access. If you are interested in a specific tag, use getTag() by name instead. - * - * @param idx The index of the tag to return. - * @return The tag at the given index. - */ - const Tag &getTag(size_t idx) const; - - /** - * Returns a reference to the named tag. If hasTag() returned false for the same key, this method returns - * a tag that is of type Tag::TYPE_EMPTY and returns true for Tag::isEmpty(). - * - * @param key The name of the tag to return. - * @return A reference to the named tag. - */ - const Tag &getTag(const vespalib::string &key) const; - - /** - * Returns whether or not there exists a tag with the given name. - * - * @param key The name of the tag to look for. - * @return True if the named tag exists. - */ - bool hasTag(const vespalib::string &key) const; - - /** - * Adds the given tag to this header. If a tag already exists with the given name, this method replaces - * that tag and returns false. - * - * @param tag The tag to add. - * @return True if no tag was overwritten. - */ - bool putTag(const Tag &tag); - - /** - * Removes a named tag. If no tag exists with the given name, this method returns false. - * - * @param key The name of the tag to remove. - * @return True if a tag was removed. - */ - bool removeTag(const vespalib::string &key); - - /** - * Returns whether or not this header contains any data. The current implementation only checks for tags, - * but as this class evolves it might include other data as well. - * - * @return True if this header has no data. - */ - bool isEmpty() const { return _tags.empty(); } - - static size_t - getMinSize(void); - - /** - * Returns the number of bytes required to hold the content of this when calling write(). - * - * @return The number of bytes. - */ - virtual size_t getSize() const; - - static size_t - readSize(IDataReader &reader); - - /** - * Deserializes header content from the given provider into this. - * - * @param reader The provider to read from. - * @return The number of bytes read. - */ - size_t read(IDataReader &reader); - - /** - * Serializes the content of this into the given consumer. - * - * @param writer The consumer to write to. - * @return The number of bytes written. - */ - size_t write(IDataWriter &writer) const; -}; - -/** - * This class adds file-specific functionality to the GenericHeader class. This includes alignment of size to - * some set number of bytes, as well as the ability to update a header in-place (see FileHeader::rewrite()). - */ -class FileHeader : public GenericHeader { -public: - /** - * Implements the GenericHeader::IDataReader interface for deserializing header content from a - * FastOS_FileInterface instance. - */ - class FileReader : public IDataReader { - private: - FastOS_FileInterface &_file; - - public: - FileReader(FastOS_FileInterface &file); - size_t getData(char *buf, size_t len) override; - }; - - /** - * Implements the GenericHeader::IDataWriter interface for serializing header content to a - * FastOS_FileInterface instance. - */ - class FileWriter : public IDataWriter { - private: - FastOS_FileInterface &_file; - - public: - FileWriter(FastOS_FileInterface &file); - size_t putData(const char *buf, size_t len) override; - }; - -private: - size_t _alignTo; - size_t _minSize; - size_t _fileSize; - -public: - /** - * Constructs a new instance of this class. - * - * @param alignTo The number of bytes to which the serialized size must be aligned to. - * @param minSize The minimum number of bytes of the serialized size. - */ - FileHeader(size_t alignTo = 8u, size_t minSize = 0u); - - /** - * This function overrides GenericHeader::getSize() to align header size to the number of bytes supplied - * to the constructor of this class. Furthermore, it will attempt to keep the header size constant after - * an initial read() or write() so that one can later rewrite() it. - * - * @return The number of bytes required to hold the content of this. - */ - size_t getSize() const override; - - /** - * Deserializes header content from the given file into this. This requires that the file is open in read - * mode, and that it is positioned at the start of the file. - * - * @param file The file to read from. - * @return The number of bytes read. - */ - size_t readFile(FastOS_FileInterface &file); - - /** - * Serializes the content of this into the given file. This requires that the file is open in write mode, - * and that it is positioned at the start of the file. - * - * @param file The file to write to. - * @return The number of bytes written. - */ - size_t writeFile(FastOS_FileInterface &file) const; - - /** - * Serializes the content of this into the given file. This requires that the file is open in read-write - * mode. This method reads the first 64 bits of the file to ensure that it is compatible with this, then - * write its content to it. Finally, it moves the file position back to where it was before. - * - * @param file The file to write to. - * @return The number of bytes written. - */ - size_t rewriteFile(FastOS_FileInterface &file); -}; - -/** - * Implements ostream operator for GenericHeader::Tag class. This method will only output the actual value of - * the tag, not its name or type. Without this operator you would have to switch on tag type to decide which - * value accessor to use. - */ -vespalib::asciistream &operator<<(vespalib::asciistream &out, const GenericHeader::Tag &tag); - -} // namespace - diff --git a/staging_vespalib/src/vespa/vespalib/encoding/.gitignore b/staging_vespalib/src/vespa/vespalib/encoding/.gitignore deleted file mode 100644 index ee8938b6bf4..00000000000 --- a/staging_vespalib/src/vespa/vespalib/encoding/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -*.So -*.exe -*.ilk -*.pdb -.depend* -Makefile diff --git a/staging_vespalib/src/vespa/vespalib/encoding/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/encoding/CMakeLists.txt deleted file mode 100644 index 48fed5ae1cc..00000000000 --- a/staging_vespalib/src/vespa/vespalib/encoding/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(staging_vespalib_vespalib_encoding OBJECT - SOURCES - base64.cpp - DEPENDS -) diff --git a/staging_vespalib/src/vespa/vespalib/encoding/base64.cpp b/staging_vespalib/src/vespa/vespalib/encoding/base64.cpp deleted file mode 100644 index bad3c168b15..00000000000 --- a/staging_vespalib/src/vespa/vespalib/encoding/base64.cpp +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * - * $Id$ - * This class convert a string to a base64 representation of the string. - * - */ - -#include <vespa/vespalib/encoding/base64.h> -#include <vespa/vespalib/util/exceptions.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <cassert> - -namespace vespalib { - -static const char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz" - "0123456789+/="; - -// Set -1 for illegal chars that will cause an error. -// Set -2 for illegal chars that will be ignored. (whitespace " \r\t\f\n") -static const signed char base64Backwards[] = { - -1, -1, -1, -1, -1, -1, -1, -1, -1, -2, -2, -1, -2, -2, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -2, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, - -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, - -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, - 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }; - -std::string -Base64::encode(const char* source, int len) -{ - // Assign a string that we know is long enough - std::string result(getMaximumEncodeLength(len), '\0'); - int outlen = encode(source, len, &result[0], result.size()); - assert(outlen >= 0); // Make sure buffer was big enough. - result.resize(outlen); - return result; -} - -std::string -Base64::decode(const char* source, int len) -{ - std::string result(getMaximumDecodeLength(len), '\0'); - int outlen = decode(source, len, &result[0], len); - assert(outlen >= 0); - result.resize(outlen); - return result; -} - -int -Base64::encode(const char *inBuffer, int inLen, char *outBuffer, int outBufLen) -{ - int i; - int outLen = 0; - for (i = 0; inLen >= 3; inLen -= 3) { - if (outBufLen - outLen < 4) { - return -1; - } - // Do this to keep chars > 127 - unsigned char a = inBuffer[i]; - unsigned char b = inBuffer[i+1]; - unsigned char c = inBuffer[i+2]; - i += 3; - - outBuffer[outLen ] = base64Chars[ a >> 2 ]; - outBuffer[outLen + 1] = base64Chars[ (a << 4 & 0x30) | (b >> 4) ]; - outBuffer[outLen + 2] = base64Chars[ (b << 2 & 0x3c) | (c >> 6) ]; - outBuffer[outLen + 3] = base64Chars[ c & 0x3f ]; - - outLen += 4; - } - - if (inLen) { - if (outBufLen - outLen < 4) { - return -1; - } - // Do this to keep chars with value>127 - unsigned char a = inBuffer[i]; - - outBuffer[outLen] = base64Chars[ a >> 2 ]; - - if (inLen == 1) { - outBuffer[outLen + 1] = base64Chars[ (a << 4 & 0x30) ]; - outBuffer[outLen + 2] = '='; - } else { - unsigned char b = inBuffer[i + 1]; - outBuffer[outLen + 1] = base64Chars[ (a << 4 & 0x30) | (b >> 4) ]; - outBuffer[outLen + 2] = base64Chars[ b << 2 & 0x3c ]; - } - - outBuffer[outLen + 3] = '='; - - outLen += 4; - } - - if (outLen >= outBufLen) - return -1; - - outBuffer[outLen] = '\0'; - - return outLen; -} - -int -Base64::decode(const char* inBuffer, int inLen, char* outBuffer, int outLen) -{ - // Read char by char to better support skipping of illegal character. - int readbytes = 0; - int num_valid_chars = 0; - const char* thischar = inBuffer; - signed char curchar; - int curOut = 0; - char tmp = 0; - - while( (readbytes++ < inLen) && (*thischar != '\0') && (*thischar != '=')) { - curchar = base64Backwards[ (unsigned int)(*thischar++) ]; - - if (curchar == -2) { - continue; // Some illegal chars will be skipped. - } else if (curchar == -1) { - // Other illegal characters will generate failure - throw IllegalArgumentException(make_string("Illegal base64 character %u found.", - (unsigned int) *thischar), VESPA_STRLOC); - } else { - - // Four bytes from input (eqals three bytes in output) - if (outLen <= curOut) { - return -1; - } - switch( num_valid_chars % 4 ) { - case 0: - tmp = (curchar << 2); - break; - case 1: - outBuffer[curOut++] = tmp | ((curchar >> 4) & 0x03); - tmp = ((curchar & 0xf) << 4); - break; - case 2: - outBuffer[curOut++] = tmp | ((curchar >> 2) & 0x0f); - tmp = ((curchar & 0x03 ) << 6); - break; - case 3: - outBuffer[curOut++] = tmp | curchar; - tmp = 0; - break; - } - num_valid_chars++; - } - } - return curOut; -} - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/encoding/base64.h b/staging_vespalib/src/vespa/vespalib/encoding/base64.h deleted file mode 100644 index e54ae5765d9..00000000000 --- a/staging_vespalib/src/vespa/vespalib/encoding/base64.h +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class vespalib::Base64 - * @group util - * - * Utility class for conversion between binary and base64 encoded data. - * - * $Id$ - */ - -#pragma once - -#include <string> -#include <algorithm> - -namespace vespalib { - -struct Base64 { - - /** - * @param sourcelen The length of the source string. - * - * @return The maximum number of characters needed to encode a string of - * given length, including terminating '\0' byte. - * - * @todo This seems to be more than needed. Inspect what encode() does and - * make this the exact size. - **/ - static int getMaximumEncodeLength(int sourcelen) - { return std::max(6, 2 * sourcelen + 2); } - - /** - * @param The length of the base64 encoded string to decode. - * - * @return The maximum size of the decoded data of a base64 string of the - * given length. - * - * @todo This seems to be more than needed. Inspect what decode() does and - * make this the exact size. - **/ - static int getMaximumDecodeLength(int sourcelen) - { return sourcelen; } - - /** - * Encodes a string of binary data to base 64. - * - * @param source The buffer to convert. - * @param len The length of the buffer. - * - * @return The base64 encoded string. - */ - static std::string encode(const std::string& source) - { return encode(source.c_str(), source.size()); } - - /** - * Encodes binary data to base 64. - * - * @param source The buffer to convert. - * @param len The length of the buffer. - * - * @return The base64 encoded string. - */ - static std::string encode(const char* source, int len); - - /** - * Encodes binary data pointed to by source, to base 64 data - * written into dest. - * - * @param source The input buffer. - * @param sourcelen The length of the input buffer. - * @param dest The buffer to write the encoded data to. - * @param destlen The length of the output buffer. This may need to be - * up to getMaximumEncodeLength(sourcelen) bytes. - * - * @return The number of characters used in dest to store the encoded - * data. Excluding '\0' termination of the string (which is always - * added). -1 is returned if there was not enough space in the dest - * buffer to store all of the data. - */ - static int encode(const char* source, int sourcelen, - char* dest, int destlen); - - /** - * Decodes base64 data to binary format. - * - * @param source The buffer to convert. - * @param len The length of the buffer. - * - * @return The base64 decoded string. - * - * @throws Throw IllegalArgumentException if source contains illegal base 64 - * characters that are not whitespace. - */ - static std::string decode(const std::string& source) - { return decode(source.c_str(), source.size()); } - - /** - * Decodes base64 data to binary format. - * - * @param source The buffer to convert. - * @param len The length of the buffer. - * - * @return The base64 decoded string. - */ - static std::string decode(const char* source, int len); - - /** - * Decodes base 64 data in source to binary format written into dest. - * - * @param source The input buffer. - * @param sourcelen The length of the input buffer. - * @param dest The buffer to write the encoded data to. - * @param destlen The length of the output buffer. - * - * @return The number of bytes used in dest to store the binary - * representation, or -1 if there wasn't enough bytes available in - * dest. - */ - static int decode(const char* source, int sourcelen, - char* dest, int destlen); -}; - -} // namespace vespalib - diff --git a/staging_vespalib/src/vespa/vespalib/metrics/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/metrics/CMakeLists.txt deleted file mode 100644 index efe9e848136..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/CMakeLists.txt +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(staging_vespalib_vespalib_metrics OBJECT - SOURCES - bucket.cpp - clock.cpp - counter_aggregator.cpp - counter.cpp - current_samples.cpp - dimension.cpp - dummy_metrics_manager.cpp - gauge_aggregator.cpp - gauge.cpp - handle.cpp - json_formatter.cpp - label.cpp - metric_id.cpp - metrics_manager.cpp - metric_types.cpp - name_collection.cpp - name_repo.cpp - point_builder.cpp - point.cpp - point_map_collection.cpp - point_map.cpp - producer.cpp - simple_metrics.cpp - simple_metrics_manager.cpp - simple_tick.cpp - snapshots.cpp - stable_store.cpp - - DEPENDS -) diff --git a/staging_vespalib/src/vespa/vespalib/metrics/bucket.cpp b/staging_vespalib/src/vespa/vespalib/metrics/bucket.cpp deleted file mode 100644 index 8d4d5558c3d..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/bucket.cpp +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "bucket.h" -#include <assert.h> -#include <map> -#include <vespa/vespalib/util/overload.h> -#include <vespa/vespalib/util/visit_ranges.h> - -namespace vespalib { -namespace metrics { - -namespace { - -template<typename T> -std::vector<typename T::aggregator_type> -mergeFromSamples(const StableStore<typename T::sample_type> &source) -{ - using Aggregator = typename T::aggregator_type; - using Sample = typename T::sample_type; - using Key = std::pair<MetricId, Point>; - using Map = std::map<Key, Aggregator>; - using MapValue = typename Map::value_type; - - Map map; - source.for_each([&map] (const Sample &sample) { - Key id = sample.idx; - auto iter_check = map.emplace(id, sample); - if (!iter_check.second) { - iter_check.first->second.merge(sample); - } - }); - std::vector<typename T::aggregator_type> result; - for (const MapValue &entry : map) { - result.push_back(entry.second); - } - return result; -} - -template<typename T> -struct IdxComparator { - bool operator() (const T& a, const T& b) { return a.idx < b.idx; } -}; - -template<typename T> -std::vector<T> -mergeVectors(const std::vector<T> &a, - const std::vector<T> &b) -{ - std::vector<T> result; - visit_ranges(overload - { - [&result](visit_ranges_either, const T& x) { result.push_back(x); }, - [&result](visit_ranges_both, const T& x, const T& y) { - result.push_back(x); - result.back().merge(y); - } - }, a.begin(), a.end(), b.begin(), b.end(), IdxComparator<T>()); - return result; -} - -template<typename T> -std::vector<T> -findMissing(const std::vector<T> &already, - const std::vector<T> &complete) -{ - std::vector<T> result; - visit_ranges(overload - { - // missing from "complete", should not happen: - [](visit_ranges_first, const T&) { }, - // missing this: - [&result](visit_ranges_second, const T& x) { result.push_back(x); }, - // already have this: - [](visit_ranges_both, const T&, const T&) { } - }, - already.begin(), already.end(), - complete.begin(), complete.end(), - IdxComparator<T>()); - return result; -} - - -} // namespace <unnamed> - -void Bucket::merge(const CurrentSamples &samples) -{ - counters = mergeFromSamples<Counter>(samples.counterIncrements); - gauges = mergeFromSamples<Gauge>(samples.gaugeMeasurements); -} - -void Bucket::merge(const Bucket &other) -{ - assert(genCnt < other.genCnt); - genCnt = other.genCnt; - startTime = std::min(startTime, other.startTime); - endTime = std::max(endTime, other.endTime); - - std::vector<CounterAggregator> nextCounters = mergeVectors(counters, other.counters); - counters = std::move(nextCounters); - - std::vector<GaugeAggregator> nextGauges = mergeVectors(gauges, other.gauges); - gauges = std::move(nextGauges); -} - -void Bucket::padMetrics(const Bucket &source) -{ - std::vector<CounterAggregator> missingC = findMissing(counters, source.counters); - for (CounterAggregator aggr : missingC) { - aggr.count = 0; - counters.push_back(aggr); - } - std::vector<GaugeAggregator> missingG = findMissing(gauges, source.gauges); - for (GaugeAggregator aggr : missingG) { - aggr.observedCount = 0; - aggr.sumValue = 0; - aggr.minValue = 0; - aggr.maxValue = 0; - gauges.push_back(aggr); - } -} - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/bucket.h b/staging_vespalib/src/vespa/vespalib/metrics/bucket.h deleted file mode 100644 index 75387aad1b3..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/bucket.h +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <mutex> -#include "stable_store.h" -#include "metric_id.h" -#include "point.h" -#include "counter.h" -#include "gauge.h" -#include "clock.h" -#include "counter_aggregator.h" -#include "gauge_aggregator.h" -#include "current_samples.h" - -namespace vespalib { -namespace metrics { - -// internal -struct Bucket { - size_t genCnt; - TimeStamp startTime; - TimeStamp endTime; - std::vector<CounterAggregator> counters; - std::vector<GaugeAggregator> gauges; - - void merge(const CurrentSamples &other); - void merge(const Bucket &other); - void padMetrics(const Bucket &source); - - Bucket(size_t generation, TimeStamp started, TimeStamp ended) - : genCnt(generation), - startTime(started), - endTime(ended), - counters(), - gauges() - {} - ~Bucket() {} - Bucket(Bucket &&) = default; - Bucket(const Bucket &) = default; - Bucket& operator= (Bucket &&) = default; -}; - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/clock.cpp b/staging_vespalib/src/vespa/vespalib/metrics/clock.cpp deleted file mode 100644 index 8593e07998e..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/clock.cpp +++ /dev/null @@ -1,2 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "clock.h" diff --git a/staging_vespalib/src/vespa/vespalib/metrics/clock.h b/staging_vespalib/src/vespa/vespalib/metrics/clock.h deleted file mode 100644 index e1b0a7da003..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/clock.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <chrono> -#include <memory> - -namespace vespalib::metrics { - -using TimeStamp = std::chrono::duration<double, std::ratio<1,1>>; - -/** - * Simple interface abstracting both timing and time measurement for - * threads wanting to do stuff at regular intervals and also knowing - * at what time stuff was done. The 'next' function blocks until the - * next tick is due and returns the current number of seconds since - * epoch. The parameter passed to the 'next' function should be its - * previous return value, except the first time it is called, then 0 - * should be used. A convenience function called 'first' is added for - * this purpose. - **/ -struct Tick { - using UP = std::unique_ptr<Tick>; - virtual TimeStamp next(TimeStamp prev) = 0; - virtual TimeStamp first() = 0; - virtual void kill() = 0; - virtual bool alive() const = 0; - virtual ~Tick() {} -}; - -} // namespace vespalib::metrics diff --git a/staging_vespalib/src/vespa/vespalib/metrics/counter.cpp b/staging_vespalib/src/vespa/vespalib/metrics/counter.cpp deleted file mode 100644 index 2b94ffce842..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/counter.cpp +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "counter.h" -#include "metrics_manager.h" - -namespace vespalib { -namespace metrics { - - -void -Counter::add(size_t count, Point point) const -{ - if (_manager) { - _manager->add(Increment(std::make_pair(_id, point), count)); - } -} - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/counter.h b/staging_vespalib/src/vespa/vespalib/metrics/counter.h deleted file mode 100644 index 751c2cc3806..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/counter.h +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <memory> -#include "metric_id.h" -#include "point.h" - -namespace vespalib { -namespace metrics { - -class MetricsManager; -struct CounterAggregator; - - -/** - * Represents a counter metric that can be incremented. - **/ -class Counter { - std::shared_ptr<MetricsManager> _manager; - MetricId _id; -public: - Counter() : _manager(), _id(0) {} - Counter(const Counter&) = delete; - Counter(Counter &&other) = default; - Counter& operator= (const Counter &) = delete; - Counter& operator= (Counter &&other) = default; - Counter(std::shared_ptr<MetricsManager> m, MetricId id) - : _manager(std::move(m)), _id(id) - {} - - // convenience methods: - void add() const { add(1, Point::empty); } - void add(Point p) { add(1, p); } - void add(size_t count) const { add(count, Point::empty); } - - /** - * Increment the counter. - * @param count the amount to increment by (default 1) - * @param p the point representing labels for this increment (default empty) - **/ - void add(size_t count, Point p) const; - - // internal - struct Increment { - using Key = std::pair<MetricId, Point>; - Key idx; - size_t value; - Increment() = delete; - Increment(Key k, size_t v) : idx(k), value(v) {} - }; - - typedef CounterAggregator aggregator_type; - typedef Increment sample_type; -}; - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/counter_aggregator.cpp b/staging_vespalib/src/vespa/vespalib/metrics/counter_aggregator.cpp deleted file mode 100644 index 583bb6e9a1c..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/counter_aggregator.cpp +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "counter_aggregator.h" -#include <assert.h> -#include <map> - -namespace vespalib { -namespace metrics { - -CounterAggregator::CounterAggregator(const Counter::Increment &increment) - : idx(increment.idx), count(increment.value) -{} - -void -CounterAggregator::merge(const CounterAggregator &other) -{ - assert(idx == other.idx); - count += other.count; -} - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/counter_aggregator.h b/staging_vespalib/src/vespa/vespalib/metrics/counter_aggregator.h deleted file mode 100644 index c650b67dad3..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/counter_aggregator.h +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "metric_id.h" -#include "point.h" -#include "counter.h" - -namespace vespalib { -namespace metrics { - -// internal -struct CounterAggregator { - std::pair<MetricId, Point> idx; - size_t count; - - CounterAggregator(const Counter::Increment &other); - void merge(const CounterAggregator &other); -}; - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/current_samples.cpp b/staging_vespalib/src/vespa/vespalib/metrics/current_samples.cpp deleted file mode 100644 index 67d56d6f748..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/current_samples.cpp +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "current_samples.h" - -namespace vespalib { -namespace metrics { - -using Guard = std::lock_guard<std::mutex>; - -void -CurrentSamples::add(Counter::Increment inc) -{ - Guard guard(lock); - counterIncrements.add(inc); -} - -void -CurrentSamples::sample(Gauge::Measurement value) -{ - Guard guard(lock); - gaugeMeasurements.add(value); -} - -void -CurrentSamples::extract(CurrentSamples &into) -{ - Guard guard(lock); - swap(into.counterIncrements, counterIncrements); - swap(into.gaugeMeasurements, gaugeMeasurements); -} - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/current_samples.h b/staging_vespalib/src/vespa/vespalib/metrics/current_samples.h deleted file mode 100644 index 4056a4bb6aa..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/current_samples.h +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <mutex> -#include "stable_store.h" -#include "counter.h" -#include "gauge.h" - -namespace vespalib { -namespace metrics { - -// internal -struct CurrentSamples { - std::mutex lock; - StableStore<Counter::Increment> counterIncrements; - StableStore<Gauge::Measurement> gaugeMeasurements; - - ~CurrentSamples() {} - - void add(Counter::Increment inc); - void sample(Gauge::Measurement value); - void extract(CurrentSamples &into); -}; - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/dimension.cpp b/staging_vespalib/src/vespa/vespalib/metrics/dimension.cpp deleted file mode 100644 index bea751a6680..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/dimension.cpp +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "dimension.h" -#include "name_repo.h" - -namespace vespalib::metrics { - -Dimension -Dimension::from_name(const vespalib::string& name) -{ - return NameRepo::instance.dimension(name); -} - -const vespalib::string& -Dimension::as_name() const -{ - return NameRepo::instance.dimensionName(*this); -} - -} // namespace vespalib::metrics diff --git a/staging_vespalib/src/vespa/vespalib/metrics/dimension.h b/staging_vespalib/src/vespa/vespalib/metrics/dimension.h deleted file mode 100644 index 7a3942b705c..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/dimension.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/vespalib/stllike/string.h> -#include "handle.h" - -namespace vespalib::metrics { - -using DimensionName = vespalib::string; - -struct DimensionTag {}; - -/** - * Opaque handle representing an uniquely named dimension. - **/ -struct Dimension : Handle<DimensionTag> -{ - explicit Dimension(size_t id) : Handle(id) {} - static Dimension from_name(const vespalib::string& name); - const vespalib::string& as_name() const; -}; - -} // namespace vespalib::metrics diff --git a/staging_vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.cpp b/staging_vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.cpp deleted file mode 100644 index 13cab477b2e..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.cpp +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "dummy_metrics_manager.h" - -namespace vespalib::metrics { - -DummyMetricsManager::~DummyMetricsManager() = default; - -Snapshot -DummyMetricsManager::snapshot() -{ - Snapshot snap(0, 0); - return snap; -} - -Snapshot -DummyMetricsManager::totalSnapshot() -{ - Snapshot snap(0, 0); - return snap; -} - -} diff --git a/staging_vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.h b/staging_vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.h deleted file mode 100644 index a707c3a5305..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.h +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <memory> -#include <thread> -#include <vespa/vespalib/stllike/string.h> -#include "name_collection.h" -#include "current_samples.h" -#include "snapshots.h" -#include "metrics_manager.h" -#include "clock.h" - -namespace vespalib::metrics { - -/** - * Dummy manager that discards everything, use - * for unit tests where you don't care about - * metrics. - **/ -class DummyMetricsManager : public MetricsManager -{ -protected: - DummyMetricsManager() noexcept {} -public: - ~DummyMetricsManager() override; - - static std::shared_ptr<MetricsManager> create() { - return std::shared_ptr<MetricsManager>(new DummyMetricsManager()); - } - - Counter counter(const vespalib::string &, const vespalib::string &) override { - return Counter(shared_from_this(), MetricId(0)); - } - Gauge gauge(const vespalib::string &, const vespalib::string &) override { - return Gauge(shared_from_this(), MetricId(0)); - } - - Dimension dimension(const vespalib::string &) override { - return Dimension(0); - } - Label label(const vespalib::string &) override { - return Label(0); - } - PointBuilder pointBuilder(Point) override { - return PointBuilder(shared_from_this()); - } - Point pointFrom(PointMap) override { - return Point(0); - } - - Snapshot snapshot() override; - Snapshot totalSnapshot() override; - - // for use from Counter only - void add(Counter::Increment) override {} - // for use from Gauge only - void sample(Gauge::Measurement) override {} -}; - -} diff --git a/staging_vespalib/src/vespa/vespalib/metrics/gauge.cpp b/staging_vespalib/src/vespa/vespalib/metrics/gauge.cpp deleted file mode 100644 index ca6b11697b4..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/gauge.cpp +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "gauge.h" -#include "metrics_manager.h" - -namespace vespalib { -namespace metrics { - -void -Gauge::sample(double value, Point point) const -{ - if (_manager) { - _manager->sample(Measurement(std::make_pair(_id, point), value)); - } -} - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/gauge.h b/staging_vespalib/src/vespa/vespalib/metrics/gauge.h deleted file mode 100644 index 59de4a6fe4d..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/gauge.h +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <memory> -#include "metric_id.h" -#include "point.h" - -namespace vespalib { -namespace metrics { - -class MetricsManager; -struct GaugeAggregator; - -/** - * Represents a gauge metric that can be measured. - **/ -class Gauge { -private: - std::shared_ptr<MetricsManager> _manager; - MetricId _id; -public: - Gauge(std::shared_ptr<MetricsManager> m, MetricId id) - : _manager(std::move(m)), _id(id) - {} - - /** - * Provide a sample for the gauge. - * @param value the measurement for this sample - * @param p the point representing labels for this sample (default empty) - **/ - void sample(double value, Point p = Point::empty) const; - - // internal - struct Measurement { - using Key = std::pair<MetricId, Point>; - Key idx; - double value; - Measurement() = delete; - Measurement(Key k, double v) : idx(k), value(v) {} - }; - - typedef GaugeAggregator aggregator_type; - typedef Measurement sample_type; -}; - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/gauge_aggregator.cpp b/staging_vespalib/src/vespa/vespalib/metrics/gauge_aggregator.cpp deleted file mode 100644 index a6c4559931f..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/gauge_aggregator.cpp +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "gauge_aggregator.h" -#include <assert.h> -#include <map> - -namespace vespalib { -namespace metrics { - -GaugeAggregator::GaugeAggregator(const Gauge::Measurement &sample) - : idx(sample.idx), - observedCount(1), - sumValue(sample.value), - minValue(sample.value), - maxValue(sample.value), - lastValue(sample.value) -{} - -void -GaugeAggregator::merge(const GaugeAggregator &other) -{ - assert(idx == other.idx); - minValue = std::min(minValue, other.minValue); - maxValue = std::max(maxValue, other.maxValue); - sumValue += other.sumValue; - lastValue = other.lastValue; - observedCount += other.observedCount; -} - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/gauge_aggregator.h b/staging_vespalib/src/vespa/vespalib/metrics/gauge_aggregator.h deleted file mode 100644 index f4fd4760cf1..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/gauge_aggregator.h +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "metric_id.h" -#include "point.h" -#include "gauge.h" - -namespace vespalib { -namespace metrics { - -// internal -struct GaugeAggregator { - std::pair<MetricId, Point> idx; - size_t observedCount; - double sumValue; - double minValue; - double maxValue; - double lastValue; - - GaugeAggregator(const Gauge::Measurement &other); - void merge(const GaugeAggregator &other); -}; - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/handle.cpp b/staging_vespalib/src/vespa/vespalib/metrics/handle.cpp deleted file mode 100644 index 50143adedd2..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/handle.cpp +++ /dev/null @@ -1,2 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "handle.h" diff --git a/staging_vespalib/src/vespa/vespalib/metrics/handle.h b/staging_vespalib/src/vespa/vespalib/metrics/handle.h deleted file mode 100644 index b901f56af7e..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/handle.h +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <cstddef> - -namespace vespalib { -namespace metrics { - -/** - * Common implementation of an opaque handle identified only - * by a (64-bit) integer. Templated to avoid different concepts - * sharing a superclass. - **/ -template <typename T> -class Handle { -private: - size_t _id; - constexpr Handle() : _id(0) {} -public: - explicit Handle(size_t id) : _id(id) {} - size_t id() const { return _id; } - - static const Handle empty_handle; -}; - -template <typename T> -const Handle<T> Handle<T>::empty_handle; - -template <typename T> -bool -operator< (const Handle<T> &a, const Handle<T> &b) noexcept -{ - return a.id() < b.id(); -} - -template <typename T> -bool -operator> (const Handle<T> &a, const Handle<T> &b) noexcept -{ - return a.id() > b.id(); -} - -template <typename T> -bool -operator== (const Handle<T> &a, const Handle<T> &b) noexcept -{ - return a.id() == b.id(); -} - -template <typename T> -bool -operator!= (const Handle<T> &a, const Handle<T> &b) noexcept -{ - return a.id() != b.id(); -} - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/json_formatter.cpp b/staging_vespalib/src/vespa/vespalib/metrics/json_formatter.cpp deleted file mode 100644 index dddc5781458..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/json_formatter.cpp +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "json_formatter.h" - -namespace vespalib { -namespace metrics { - -JsonFormatter::JsonFormatter(const Snapshot &snapshot) - : _data(), - _top(_data.setObject()), - _snapLen(snapshot.endTime() - snapshot.startTime()) -{ - if (_snapLen < 0.1) { - _snapLen = 0.1; - } - // cosmetics: ordering inside objects - _data.insert("name"); - _data.insert("dimensions"); - vespalib::slime::Cursor& meta = _top.setObject("snapshot"); - meta.setLong("from", (long)snapshot.startTime()); - meta.setLong("to", (long)snapshot.endTime()); - handle(snapshot, _top.setArray("values")); -} - -void -JsonFormatter::handle(const Snapshot &snapshot, vespalib::slime::Cursor &target) -{ - for (const CounterSnapshot &entry : snapshot.counters()) { - handle(entry, target.addObject()); - } - for (const GaugeSnapshot &entry : snapshot.gauges()) { - handle(entry, target.addObject()); - } -} - -void -JsonFormatter::handle(const CounterSnapshot &snapshot, vespalib::slime::Cursor &target) -{ - target.setString("name", snapshot.name()); - // target.setString("description", ?); - handle(snapshot.point(), target); - Cursor& inner = target.setObject("values"); - inner.setLong("count", snapshot.count()); - inner.setDouble("rate", snapshot.count() / _snapLen); -} - -void -JsonFormatter::handle(const GaugeSnapshot &snapshot, vespalib::slime::Cursor &target) -{ - target.setString("name", snapshot.name()); - // target.setString("description", ?); - handle(snapshot.point(), target); - Cursor& inner = target.setObject("values"); - inner.setDouble("average", snapshot.averageValue()); - inner.setDouble("sum", snapshot.sumValue()); - inner.setDouble("min", snapshot.minValue()); - inner.setDouble("max", snapshot.maxValue()); - inner.setDouble("last", snapshot.lastValue()); - inner.setLong("count", snapshot.observedCount()); - inner.setDouble("rate", snapshot.observedCount() / _snapLen); -} - -void -JsonFormatter::handle(const PointSnapshot &snapshot, vespalib::slime::Cursor &target) -{ - if (snapshot.dimensions.size() == 0) { - return; - } - Cursor& inner = target.setObject("dimensions"); - for (const DimensionBinding &entry : snapshot.dimensions) { - inner.setString(entry.dimensionName(), entry.labelValue()); - } -} - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/json_formatter.h b/staging_vespalib/src/vespa/vespalib/metrics/json_formatter.h deleted file mode 100644 index f9d3664e5db..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/json_formatter.h +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "snapshots.h" -#include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/data/slime/slime.h> - -namespace vespalib { -namespace metrics { - -/** - * utility for converting a snapshot to JSON format - * (which can be inserted into /state/v1/metrics page). - **/ -class JsonFormatter -{ -private: - using Cursor = vespalib::slime::Cursor; - vespalib::Slime _data; - Cursor& _top; - double _snapLen; - - void handle(const Snapshot &snapshot, Cursor &target); - void handle(const PointSnapshot &snapshot, Cursor &target); - void handle(const CounterSnapshot &snapshot, Cursor &target); - void handle(const GaugeSnapshot &snapshot, Cursor &target); -public: - JsonFormatter(const Snapshot &snapshot); - - vespalib::string asString() const { - return _data.toString(); - } -}; - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/label.cpp b/staging_vespalib/src/vespa/vespalib/metrics/label.cpp deleted file mode 100644 index fe287e16d46..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/label.cpp +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "label.h" -#include "name_repo.h" - -namespace vespalib::metrics { - -Label -Label::from_value(const vespalib::string& value) -{ - return NameRepo::instance.label(value); -} - -const vespalib::string& -Label::as_value() const -{ - return NameRepo::instance.labelValue(*this); -} - -} // namespace vespalib::metrics diff --git a/staging_vespalib/src/vespa/vespalib/metrics/label.h b/staging_vespalib/src/vespa/vespalib/metrics/label.h deleted file mode 100644 index 755b3e83bef..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/label.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/vespalib/stllike/string.h> -#include "handle.h" - -namespace vespalib::metrics { - -using LabelValue = vespalib::string; - -struct LabelTag {}; - -/** - * Opaque handle representing an unique label value. - **/ -struct Label : Handle<LabelTag> -{ - explicit Label(size_t id) : Handle(id) {} - static Label from_value(const vespalib::string& value); - const vespalib::string& as_value() const; -}; - -} // namespace vespalib::metrics diff --git a/staging_vespalib/src/vespa/vespalib/metrics/metric_id.cpp b/staging_vespalib/src/vespa/vespalib/metrics/metric_id.cpp deleted file mode 100644 index 7ce012ad5fd..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/metric_id.cpp +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "metric_id.h" -#include "name_repo.h" - -namespace vespalib::metrics { - -MetricId -MetricId::from_name(const vespalib::string& name) -{ - return NameRepo::instance.metric(name); -} - -const vespalib::string& -MetricId::as_name() const -{ - return NameRepo::instance.metricName(*this); -} - -} // namespace vespalib::metrics diff --git a/staging_vespalib/src/vespa/vespalib/metrics/metric_id.h b/staging_vespalib/src/vespa/vespalib/metrics/metric_id.h deleted file mode 100644 index 5aa9d1b868e..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/metric_id.h +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "handle.h" -#include <vespa/vespalib/stllike/string.h> - -namespace vespalib::metrics { - -struct MetricIdTag {}; - -/** - * Opaque handle representing an uniquely named metric. - **/ -struct MetricId : Handle<MetricIdTag> -{ - explicit MetricId(size_t id) : Handle(id) {} - static MetricId from_name(const vespalib::string& name); - const vespalib::string& as_name() const; -}; - -} // namespace vespalib::metrics diff --git a/staging_vespalib/src/vespa/vespalib/metrics/metric_types.cpp b/staging_vespalib/src/vespa/vespalib/metrics/metric_types.cpp deleted file mode 100644 index 196dfaed16a..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/metric_types.cpp +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "metric_types.h" -#include <assert.h> - -#include <vespa/log/log.h> -LOG_SETUP(".vespalib.metrics.metric_types"); - -namespace vespalib { -namespace metrics { - -const char* MetricTypes::_typeNames[] = { - "INVALID", - "Counter", - "Gauge", - "Histogram", - "IntegerHistogram" -}; - -void -MetricTypes::check(size_t id, const vespalib::string &name, MetricType ty) -{ - std::lock_guard<std::mutex> guard(_lock); - if (id < _seen.size()) { - MetricType old = _seen[id]; - if (old == ty) { - return; - } - if (old == MetricType::INVALID) { - _seen[id] = ty; - } - LOG(warning, "metric '%s' with different types %s and %s, this will be confusing", - name.c_str(), _typeNames[ty], _typeNames[old]); - } - while (_seen.size() < id) { - _seen.push_back(MetricType::INVALID); - } - _seen.push_back(ty); -} - -} // namespace vespalib::metrics -} // namespace vespalib - - - diff --git a/staging_vespalib/src/vespa/vespalib/metrics/metric_types.h b/staging_vespalib/src/vespa/vespalib/metrics/metric_types.h deleted file mode 100644 index e8118b6bd63..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/metric_types.h +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <mutex> -#include <vector> -#include <vespa/vespalib/stllike/string.h> - -namespace vespalib { -namespace metrics { - -// internal class for typechecking -class MetricTypes { - static const char *_typeNames[]; -public: - enum MetricType { - INVALID, - COUNTER, - GAUGE, - HISTOGRAM, - INT_HISTOGRAM - }; - - void check(size_t id, const vespalib::string& name, MetricType ty); - - MetricTypes() = default; - ~MetricTypes() {} -private: - std::mutex _lock; - std::vector<MetricType> _seen; -}; - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/metrics_manager.cpp b/staging_vespalib/src/vespa/vespalib/metrics/metrics_manager.cpp deleted file mode 100644 index 3d598528777..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/metrics_manager.cpp +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "metrics_manager.h" - -namespace vespalib { -namespace metrics { - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/metrics_manager.h b/staging_vespalib/src/vespa/vespalib/metrics/metrics_manager.h deleted file mode 100644 index 6b80527b22e..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/metrics_manager.h +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <memory> -#include <thread> -#include <vespa/vespalib/stllike/string.h> -#include "counter.h" -#include "gauge.h" -#include "current_samples.h" -#include "snapshots.h" -#include "point.h" -#include "point_builder.h" -#include "dimension.h" -#include "label.h" - -namespace vespalib::metrics { - -/** - * Interface for a Metrics manager, for creating metrics - * and for fetching snapshots. - **/ -class MetricsManager - : public std::enable_shared_from_this<MetricsManager> -{ -public: - virtual ~MetricsManager() {} - - /** - * Get or create a counter metric. - * @param name the name of the metric. - **/ - virtual Counter counter(const vespalib::string &name, const vespalib::string &description) = 0; - - /** - * Get or create a gauge metric. - * @param name the name of the metric. - **/ - virtual Gauge gauge(const vespalib::string &name, const vespalib::string &description) = 0; - - /** - * Get or create a dimension for labeling metrics. - * @param name the name of the dimension. - **/ - virtual Dimension dimension(const vespalib::string &name) = 0; // get or create - - /** - * Get or create a label. - * @param value the label value. - **/ - virtual Label label(const vespalib::string &value) = 0; // get or create - - /** - * Create a PointBuilder for labeling metrics. - **/ - PointBuilder pointBuilder() { - return PointBuilder(shared_from_this()); - } - - /** - * Create a PointBuilder for labeling metrics, starting with - * an Point of already existing dimension/label pairs, which can - * then be added to or changed. - * @param from provide a Point to start from. - * - **/ - virtual PointBuilder pointBuilder(Point from) = 0; - - /** - * Create a snapshot of sampled metrics (usually for the last minute). - **/ - virtual Snapshot snapshot() = 0; - - /** - * Create a snapshot of all sampled metrics the manager has seen. - **/ - virtual Snapshot totalSnapshot() = 0; - - // for use from PointBuilder only - virtual Point pointFrom(PointMap map) = 0; - - // for use from Counter only - virtual void add(Counter::Increment inc) = 0; - - // for use from Gauge only - virtual void sample(Gauge::Measurement value) = 0; -}; - - -} // namespace vespalib::metrics diff --git a/staging_vespalib/src/vespa/vespalib/metrics/name_collection.cpp b/staging_vespalib/src/vespa/vespalib/metrics/name_collection.cpp deleted file mode 100644 index 964fd9c59a8..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/name_collection.cpp +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "name_collection.h" -#include <cassert> - -namespace vespalib::metrics { - -using Guard = std::lock_guard<std::mutex>; - -NameCollection::NameCollection() -{ - size_t first = resolve(""); - assert(first == 0); - assert(lookup(first) == ""); - assert(_names_by_id.size() == 1); - assert(_names.size() == 1); - (void) first; // in case of NOP asserts -} - -NameCollection::~NameCollection() = default; - -const vespalib::string & -NameCollection::lookup(size_t id) const -{ - Guard guard(_lock); - assert(id < _names_by_id.size()); - return _names_by_id[id]->first; -} - -size_t -NameCollection::resolve(const vespalib::string& name) -{ - Guard guard(_lock); - size_t nextId = _names_by_id.size(); - auto iter_check = _names.emplace(name, nextId); - if (iter_check.second) { - _names_by_id.push_back(iter_check.first); - } - return iter_check.first->second; -} - -size_t -NameCollection::size() const -{ - Guard guard(_lock); - return _names_by_id.size(); -} - -} diff --git a/staging_vespalib/src/vespa/vespalib/metrics/name_collection.h b/staging_vespalib/src/vespa/vespalib/metrics/name_collection.h deleted file mode 100644 index bef614b5a68..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/name_collection.h +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <mutex> -#include <map> -#include <vector> -#include <vespa/vespalib/stllike/string.h> - -namespace vespalib::metrics { - -// internal -class NameCollection { -private: - using Map = std::map<vespalib::string, size_t>; - mutable std::mutex _lock; - Map _names; - std::vector<Map::const_iterator> _names_by_id; -public: - const vespalib::string &lookup(size_t id) const; - size_t resolve(const vespalib::string& name); - size_t size() const; - - NameCollection(); - NameCollection(const NameCollection &) = delete; - NameCollection & operator = (const NameCollection &) = delete; - ~NameCollection(); - - static constexpr size_t empty_id = 0; -}; - -} diff --git a/staging_vespalib/src/vespa/vespalib/metrics/name_repo.cpp b/staging_vespalib/src/vespa/vespalib/metrics/name_repo.cpp deleted file mode 100644 index 2f31c90da29..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/name_repo.cpp +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "name_repo.h" - -#include <vespa/log/log.h> -LOG_SETUP(".vespalib.metrics.name_repo"); - -namespace vespalib { -namespace metrics { - -MetricId -NameRepo::metric(const vespalib::string &name) -{ - size_t id = _metricNames.resolve(name); - LOG(debug, "metric name %s -> %zu", name.c_str(), id); - return MetricId(id); -} - -Dimension -NameRepo::dimension(const vespalib::string &name) -{ - size_t id = _dimensionNames.resolve(name); - LOG(debug, "dimension name %s -> %zu", name.c_str(), id); - return Dimension(id); -} - -Label -NameRepo::label(const vespalib::string &value) -{ - size_t id = _labelValues.resolve(value); - LOG(debug, "label value %s -> %zu", value.c_str(), id); - return Label(id); -} - -const vespalib::string& -NameRepo::metricName(MetricId metric) const -{ - return _metricNames.lookup(metric.id()); -} - -const vespalib::string& -NameRepo::dimensionName(Dimension dim) const -{ - return _dimensionNames.lookup(dim.id()); -} - -const vespalib::string& -NameRepo::labelValue(Label l) const -{ - return _labelValues.lookup(l.id()); -} - - -const PointMap& -NameRepo::pointMap(Point from) const -{ - return _pointMaps.lookup(from.id()); -} - -Point -NameRepo::pointFrom(PointMap map) -{ - size_t id = _pointMaps.resolve(std::move(map)); - return Point(id); -} - - -NameRepo NameRepo::instance; - - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/name_repo.h b/staging_vespalib/src/vespa/vespalib/metrics/name_repo.h deleted file mode 100644 index e2230cf59d9..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/name_repo.h +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/vespalib/stllike/string.h> -#include "dimension.h" -#include "label.h" -#include "metric_id.h" -#include "point.h" - -#include "name_collection.h" -#include "point_map_collection.h" - -namespace vespalib::metrics { - -/** - * Simple repo class - **/ -class NameRepo -{ -private: - NameCollection _metricNames; - NameCollection _dimensionNames; - NameCollection _labelValues; - PointMapCollection _pointMaps; - - NameRepo() = default; - ~NameRepo() = default; -public: - - MetricId metric(const vespalib::string &name); - Dimension dimension(const vespalib::string &name); - Label label(const vespalib::string &value); - - const vespalib::string& metricName(MetricId metric) const; - const vespalib::string& dimensionName(Dimension dim) const; - const vespalib::string& labelValue(Label l) const; - - const PointMap& pointMap(Point from) const; - Point pointFrom(PointMap map); - - static NameRepo instance; -}; - -} diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point.cpp b/staging_vespalib/src/vespa/vespalib/metrics/point.cpp deleted file mode 100644 index b4d3d6e2738..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/point.cpp +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "point.h" -#include "name_repo.h" - -namespace vespalib { -namespace metrics { - -Point Point::empty(0); - -Point -Point::from_map(const PointMap& map) -{ - return NameRepo::instance.pointFrom(map); -} - -Point -Point::from_map(PointMap&& map) -{ - return NameRepo::instance.pointFrom(std::move(map)); -} - -const PointMap& -Point::as_map() const -{ - return NameRepo::instance.pointMap(*this); -} - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point.h b/staging_vespalib/src/vespa/vespalib/metrics/point.h deleted file mode 100644 index a8f483c4404..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/point.h +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "handle.h" -#include "point_map.h" - -namespace vespalib::metrics { - -/** - * Opaque handle representing an unique N-dimensional point - **/ -class Point : public Handle<Point> { -public: - static Point empty; - explicit Point(size_t id) : Handle<Point>(id) {} - - static Point from_map(const PointMap& map); - static Point from_map(PointMap&& map); - const PointMap& as_map() const; -}; - -} // namespace vespalib::metrics diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point_builder.cpp b/staging_vespalib/src/vespa/vespalib/metrics/point_builder.cpp deleted file mode 100644 index d085228c1a8..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/point_builder.cpp +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "point.h" -#include "metrics_manager.h" - -namespace vespalib { -namespace metrics { - -PointBuilder::PointBuilder(std::shared_ptr<MetricsManager> m) - : _owner(std::move(m)), _map() -{} - -PointBuilder::PointBuilder(std::shared_ptr<MetricsManager> m, - const PointMap ©From) - : _owner(std::move(m)), _map(copyFrom) -{} - -PointBuilder & -PointBuilder::bind(Dimension dimension, Label label) & -{ - _map.erase(dimension); - _map.emplace(dimension, label); - return *this; -} -PointBuilder & -PointBuilder::bind(Dimension dimension, LabelValue label) & -{ - Label c = _owner->label(label); - return bind(dimension, c); -} - -PointBuilder & -PointBuilder::bind(DimensionName dimension, LabelValue label) & -{ - Dimension a = _owner->dimension(dimension); - Label c = _owner->label(label); - return bind(a, c); -} - -PointBuilder && -PointBuilder::bind(Dimension dimension, Label label) && -{ - bind(dimension, label); - return std::move(*this); -} - -PointBuilder && -PointBuilder::bind(Dimension dimension, LabelValue label) && -{ - bind(dimension, label); - return std::move(*this); -} - -PointBuilder && -PointBuilder::bind(DimensionName dimension, LabelValue label) && -{ - bind(dimension, label); - return std::move(*this); -} - -Point -PointBuilder::build() -{ - return _owner->pointFrom(_map); -} - -PointBuilder::operator Point() && -{ - return _owner->pointFrom(std::move(_map)); -} - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point_builder.h b/staging_vespalib/src/vespa/vespalib/metrics/point_builder.h deleted file mode 100644 index d9c4b5a114f..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/point_builder.h +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <memory> -#include <vespa/vespalib/stllike/string.h> - -#include "point.h" -#include "point_map.h" - -namespace vespalib { -namespace metrics { - -class MetricsManager; - -/** - * Build a Point for labeling metrics - **/ -class PointBuilder { -private: - std::shared_ptr<MetricsManager> _owner; - PointMap _map; - -public: - // for use from MetricsManager - PointBuilder(std::shared_ptr<MetricsManager> m); - PointBuilder(std::shared_ptr<MetricsManager> m, const PointMap &from); - ~PointBuilder() {} - - /** - * Bind a dimension to a label. - * Overwrites any label already bound to that dimension. - **/ - PointBuilder &&bind(Dimension dimension, Label label) &&; - PointBuilder &bind(Dimension dimension, Label label) &; - - /** - * Bind a dimension to a label. - * Convenience method that converts the label value. - **/ - PointBuilder &&bind(Dimension dimension, LabelValue label) &&; - PointBuilder &bind(Dimension dimension, LabelValue label) &; - - /** - * Bind a dimension to a label. - * Convenience method that converts both the dimension name and the label value. - **/ - PointBuilder &&bind(DimensionName dimension, LabelValue label) &&; - PointBuilder &bind(DimensionName dimension, LabelValue label) &; - - /** make a Point from the builder */ - Point build(); - - /** make a Point from the builder */ - operator Point () &&; -}; - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point_map.cpp b/staging_vespalib/src/vespa/vespalib/metrics/point_map.cpp deleted file mode 100644 index cd5909707e3..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/point_map.cpp +++ /dev/null @@ -1,2 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "point_map.h" diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point_map.h b/staging_vespalib/src/vespa/vespalib/metrics/point_map.h deleted file mode 100644 index 7a435389566..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/point_map.h +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <map> -#include "dimension.h" -#include "label.h" - -namespace vespalib { -namespace metrics { - -using PointMap = std::map<Dimension, Label>; - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point_map_collection.cpp b/staging_vespalib/src/vespa/vespalib/metrics/point_map_collection.cpp deleted file mode 100644 index d9b7362521b..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/point_map_collection.cpp +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "point_map_collection.h" -#include <assert.h> - -namespace vespalib { -namespace metrics { - -HashedPointMap::HashedPointMap(PointMap &&from) - : _map(std::move(from)), - _hash(0) -{ - for (const PointMap::value_type &entry : _map) { - _hash = (_hash << 7) + (_hash >> 31) + entry.first.id(); - _hash = (_hash << 7) + (_hash >> 31) + entry.second.id(); - } -} - -bool -HashedPointMap::operator< (const HashedPointMap &other) const -{ - // cheap comparison first - if (_hash != other._hash) { - return _hash < other._hash; - } - if (_map.size() != other._map.size()) { - return _map.size() < other._map.size(); - } - // sizes equal, fall back to std::map::operator< - return _map < other._map; -} - -using Guard = std::lock_guard<std::mutex>; - -const PointMap & -PointMapCollection::lookup(size_t id) const -{ - Guard guard(_lock); - assert(id < _vec.size()); - PointMapMap::const_iterator iter = _vec[id]; - return iter->first.backingMap(); -} - -size_t -PointMapCollection::resolve(PointMap map) -{ - Guard guard(_lock); - size_t nextId = _vec.size(); - auto iter_check = _map.emplace(std::move(map), nextId); - if (iter_check.second) { - _vec.push_back(iter_check.first); - } - return iter_check.first->second; -} - -size_t -PointMapCollection::size() const -{ - Guard guard(_lock); - return _vec.size(); -} - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point_map_collection.h b/staging_vespalib/src/vespa/vespalib/metrics/point_map_collection.h deleted file mode 100644 index ee556ea2107..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/point_map_collection.h +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <mutex> -#include <map> -#include <vector> -#include "point_map.h" - -namespace vespalib { -namespace metrics { - -// internal -class HashedPointMap { -private: - const PointMap _map; - size_t _hash; -public: - HashedPointMap(PointMap &&from); - bool operator< (const HashedPointMap &other) const; - - const PointMap &backingMap() const { return _map; } -}; - -// internal -class PointMapCollection { -private: - using PointMapMap = std::map<HashedPointMap, size_t>; - - mutable std::mutex _lock; - PointMapMap _map; - std::vector<PointMapMap::const_iterator> _vec; -public: - const PointMap &lookup(size_t id) const; - size_t resolve(PointMap map); - size_t size() const; - - PointMapCollection() = default; - ~PointMapCollection() {} -}; - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/producer.cpp b/staging_vespalib/src/vespa/vespalib/metrics/producer.cpp deleted file mode 100644 index de3ed5d368f..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/producer.cpp +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "producer.h" -#include "metrics_manager.h" -#include "json_formatter.h" - -namespace vespalib { -namespace metrics { - -Producer::Producer(std::shared_ptr<MetricsManager> m) - : _manager(m) -{} - -vespalib::string -Producer::getMetrics(const vespalib::string &) -{ - Snapshot snap = _manager->snapshot(); - JsonFormatter fmt(snap); - return fmt.asString(); -} - -vespalib::string -Producer::getTotalMetrics(const vespalib::string &) -{ - Snapshot snap = _manager->totalSnapshot(); - JsonFormatter fmt(snap); - return fmt.asString(); -} - - - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/producer.h b/staging_vespalib/src/vespa/vespalib/metrics/producer.h deleted file mode 100644 index ac741b4e82d..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/producer.h +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <memory> -#include <vespa/vespalib/net/metrics_producer.h> - -namespace vespalib { -namespace metrics { - -class MetricsManager; - -/** - * Utility class for wiring a MetricsManager into a StateApi. - **/ -class Producer : public vespalib::MetricsProducer { -private: - std::shared_ptr<MetricsManager> _manager; -public: - Producer(std::shared_ptr<MetricsManager> m); - vespalib::string getMetrics(const vespalib::string &consumer) override; - vespalib::string getTotalMetrics(const vespalib::string &consumer) override; -}; - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.cpp b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.cpp deleted file mode 100644 index d438426518c..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.cpp +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "simple_metrics.h" - -namespace vespalib { -namespace metrics { - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.h b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.h deleted file mode 100644 index e558e93a638..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.h +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vector> -#include <chrono> -#include <memory> -#include <vespa/vespalib/stllike/string.h> - -#include "clock.h" -#include "counter.h" -#include "dimension.h" -#include "dummy_metrics_manager.h" -#include "gauge.h" -#include "label.h" -#include "metric_id.h" -#include "metrics_manager.h" -#include "point_builder.h" -#include "point.h" -#include "producer.h" -#include "simple_metrics_manager.h" -#include "snapshots.h" diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp deleted file mode 100644 index 4b6b82697f7..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp +++ /dev/null @@ -1,230 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "simple_metrics_manager.h" -#include "simple_tick.h" - -#include <vespa/log/log.h> -LOG_SETUP(".vespalib.metrics.simple_metrics_manager"); - -namespace vespalib { -namespace metrics { - -using Guard = std::lock_guard<std::mutex>; - -SimpleMetricsManager::SimpleMetricsManager(const SimpleManagerConfig &config, - Tick::UP tick_supplier) - : _currentSamples(), - _tickSupplier(std::move(tick_supplier)), - _startTime(_tickSupplier->first()), - _curTime(_startTime), - _collectCnt(0), - _buckets(), - _firstBucket(0), - _maxBuckets(config.sliding_window_seconds), - _totalsBucket(0, _startTime, _startTime), - _thread(&SimpleMetricsManager::tickerLoop, this) -{ - if (_maxBuckets < 1) _maxBuckets = 1; - Point empty = pointFrom(PointMap()); - assert(empty.id() == 0); -} - -SimpleMetricsManager::~SimpleMetricsManager() -{ - stopThread(); -} - -std::shared_ptr<MetricsManager> -SimpleMetricsManager::create(const SimpleManagerConfig &config) -{ - return std::shared_ptr<MetricsManager>( - new SimpleMetricsManager(config, std::make_unique<SimpleTick>())); -} - -std::shared_ptr<MetricsManager> -SimpleMetricsManager::createForTest(const SimpleManagerConfig &config, - Tick::UP tick_supplier) -{ - return std::shared_ptr<MetricsManager>( - new SimpleMetricsManager(config, std::move(tick_supplier))); -} - -Counter -SimpleMetricsManager::counter(const vespalib::string &name, const vespalib::string &) -{ - MetricId mi = MetricId::from_name(name); - _metricTypes.check(mi.id(), name, MetricTypes::MetricType::COUNTER); - LOG(debug, "counter with metric name %s -> %zu", name.c_str(), mi.id()); - return Counter(shared_from_this(), mi); -} - -Gauge -SimpleMetricsManager::gauge(const vespalib::string &name, const vespalib::string &) -{ - MetricId mi = MetricId::from_name(name); - _metricTypes.check(mi.id(), name, MetricTypes::MetricType::GAUGE); - LOG(debug, "gauge with metric name %s -> %zu", name.c_str(), mi.id()); - return Gauge(shared_from_this(), mi); -} - -Bucket -SimpleMetricsManager::mergeBuckets() -{ - Guard bucketsGuard(_bucketsLock); - if (_buckets.size() > 0) { - TimeStamp startTime = _buckets[_firstBucket].startTime; - Bucket merger(0, startTime, startTime); - for (size_t i = 0; i < _buckets.size(); i++) { - size_t off = (_firstBucket + i) % _buckets.size(); - merger.merge(_buckets[off]); - } - merger.padMetrics(_totalsBucket); - return merger; - } - // no data - return Bucket(0, _startTime, _curTime); -} - -Bucket -SimpleMetricsManager::totalsBucket() -{ - Guard bucketsGuard(_bucketsLock); - return _totalsBucket; -} - -Snapshot -SimpleMetricsManager::snapshotFrom(const Bucket &bucket) -{ - std::vector<PointSnapshot> points; - - double s = bucket.startTime.count(); - double e = bucket.endTime.count(); - - size_t max_point_id = 0; - for (const CounterAggregator& entry : bucket.counters) { - Point p = entry.idx.second; - max_point_id = std::max(max_point_id, p.id()); - } - for (const GaugeAggregator& entry : bucket.gauges) { - Point p = entry.idx.second; - max_point_id = std::max(max_point_id, p.id()); - } - Snapshot snap(s, e); - { - for (size_t point_id = 0; point_id <= max_point_id; ++point_id) { - const PointMap &map = Point(point_id).as_map(); - PointSnapshot point; - for (const PointMap::value_type &kv : map) { - point.dimensions.emplace_back(kv.first.as_name(), kv.second.as_value()); - } - snap.add(point); - } - } - for (const CounterAggregator& entry : bucket.counters) { - MetricId mi = entry.idx.first; - Point p = entry.idx.second; - size_t pi = p.id(); - const vespalib::string &name = mi.as_name(); - CounterSnapshot val(name, snap.points()[pi], entry); - snap.add(val); - } - for (const GaugeAggregator& entry : bucket.gauges) { - MetricId mi = entry.idx.first; - Point p = entry.idx.second; - size_t pi = p.id(); - const vespalib::string &name = mi.as_name(); - GaugeSnapshot val(name, snap.points()[pi], entry); - snap.add(val); - } - return snap; -} - -Snapshot -SimpleMetricsManager::snapshot() -{ - Bucket merged = mergeBuckets(); - return snapshotFrom(merged); -} - -Snapshot -SimpleMetricsManager::totalSnapshot() -{ - Bucket totals = totalsBucket(); - return snapshotFrom(totals); -} - -void -SimpleMetricsManager::collectCurrentSamples(TimeStamp prev, - TimeStamp curr) -{ - CurrentSamples samples; - _currentSamples.extract(samples); - Bucket newBucket(++_collectCnt, prev, curr); - newBucket.merge(samples); - - Guard guard(_bucketsLock); - _totalsBucket.merge(newBucket); - if (_buckets.size() < _maxBuckets) { - _buckets.push_back(std::move(newBucket)); - } else { - _buckets[_firstBucket] = std::move(newBucket); - _firstBucket = (_firstBucket + 1) % _buckets.size(); - } -} - -Dimension -SimpleMetricsManager::dimension(const vespalib::string &name) -{ - Dimension dim = Dimension::from_name(name); - LOG(debug, "dimension name %s -> %zu", name.c_str(), dim.id()); - return dim; -} - -Label -SimpleMetricsManager::label(const vespalib::string &value) -{ - Label l = Label::from_value(value); - LOG(debug, "label value %s -> %zu", value.c_str(), l.id()); - return l; -} - -PointBuilder -SimpleMetricsManager::pointBuilder(Point from) -{ - const PointMap &map = from.as_map(); - return PointBuilder(shared_from_this(), map); -} - -Point -SimpleMetricsManager::pointFrom(PointMap map) -{ - return Point::from_map(std::move(map)); -} - -void -SimpleMetricsManager::tickerLoop() -{ - while (_tickSupplier->alive()) { - TimeStamp now = _tickSupplier->next(_curTime); - if (_tickSupplier->alive()) { - tick(now); - } - } -} - -void -SimpleMetricsManager::stopThread() -{ - _tickSupplier->kill(); - _thread.join(); -} - -void -SimpleMetricsManager::tick(TimeStamp now) -{ - TimeStamp prev = _curTime; - collectCurrentSamples(prev, now); - _curTime = now; -} - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.h b/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.h deleted file mode 100644 index fe3023ae020..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.h +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <atomic> -#include <mutex> -#include <memory> -#include <thread> -#include <vespa/vespalib/stllike/string.h> -#include "current_samples.h" -#include "snapshots.h" -#include "metrics_manager.h" -#include "metric_types.h" -#include "clock.h" -#include "bucket.h" - -namespace vespalib { -namespace metrics { - -struct SimpleManagerConfig { - int sliding_window_seconds; - // possibly more config later - SimpleManagerConfig() : sliding_window_seconds(60) {} -}; - - -/** - * Simple manager class that puts everything into a - * single global repo with std::mutex locks used around - * most operations. Only implements sliding window - * and a fixed (1 Hz) collecting interval. - * XXX: Consider renaming this to "SlidingWindowManager". - **/ -class SimpleMetricsManager : public MetricsManager -{ -private: - MetricTypes _metricTypes; - - CurrentSamples _currentSamples; - - Tick::UP _tickSupplier; - TimeStamp _startTime; - TimeStamp _curTime; - - std::mutex _bucketsLock; - size_t _collectCnt; - std::vector<Bucket> _buckets; - size_t _firstBucket; - size_t _maxBuckets; - Bucket _totalsBucket; - - std::thread _thread; - void tickerLoop(); - void stopThread(); - void tick(TimeStamp now); // called once per second from another thread - - void collectCurrentSamples(TimeStamp prev, TimeStamp curr); - Bucket mergeBuckets(); - Bucket totalsBucket(); - Snapshot snapshotFrom(const Bucket &bucket); - - SimpleMetricsManager(const SimpleManagerConfig &config, - Tick::UP tick_supplier); -public: - ~SimpleMetricsManager(); - static std::shared_ptr<MetricsManager> create(const SimpleManagerConfig &config); - static std::shared_ptr<MetricsManager> createForTest(const SimpleManagerConfig &config, - Tick::UP tick_supplier); - Counter counter(const vespalib::string &name, const vespalib::string &description) override; - Gauge gauge(const vespalib::string &name, const vespalib::string &description) override; - Dimension dimension(const vespalib::string &name) override; - Label label(const vespalib::string &value) override; - PointBuilder pointBuilder(Point from) override; - Point pointFrom(PointMap map) override; - Snapshot snapshot() override; - Snapshot totalSnapshot() override; - - // for use from Counter only - void add(Counter::Increment inc) override { - _currentSamples.add(inc); - } - // for use from Gauge only - void sample(Gauge::Measurement value) override { - _currentSamples.sample(value); - } -}; - -} // namespace vespalib::metrics -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_tick.cpp b/staging_vespalib/src/vespa/vespalib/metrics/simple_tick.cpp deleted file mode 100644 index 0eed2ed3400..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/simple_tick.cpp +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "simple_tick.h" - -namespace vespalib::metrics { - -namespace { - -const TimeStamp oneSec{1.0}; - -TimeStamp now() -{ - using Clock = std::chrono::system_clock; - Clock::time_point now = Clock::now(); - return now.time_since_epoch(); -} - -} // namespace <unnamed> - -SimpleTick::SimpleTick() - : _lock(), _runFlag(true), _cond() -{} - -TimeStamp -SimpleTick::first() -{ - return now(); -} - -TimeStamp -SimpleTick::next(TimeStamp prev) -{ - std::unique_lock<std::mutex> locker(_lock); - while (_runFlag) { - TimeStamp curr = now(); - if (curr - prev >= oneSec) { - return curr; - } else if (curr < prev) { - // clock was adjusted backwards - prev = curr; - _cond.wait_for(locker, oneSec); - } else { - _cond.wait_for(locker, oneSec - (curr - prev)); - } - } - return now(); -} - -void -SimpleTick::kill() -{ - std::unique_lock<std::mutex> locker(_lock); - _runFlag.store(false); - _cond.notify_all(); -} - -bool -SimpleTick::alive() const -{ - return _runFlag; -} - -} // namespace vespalib::metrics diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_tick.h b/staging_vespalib/src/vespa/vespalib/metrics/simple_tick.h deleted file mode 100644 index 8ab18e74c83..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/simple_tick.h +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "clock.h" - -#include <atomic> -#include <condition_variable> -#include <mutex> - -namespace vespalib::metrics { - -// internal -class SimpleTick : public Tick { -private: - std::mutex _lock; - std::atomic<bool> _runFlag; - std::condition_variable _cond; -public: - SimpleTick(); - TimeStamp first() override; - TimeStamp next(TimeStamp prev) override; - void kill() override; - bool alive() const override; -}; - -} // namespace vespalib::metrics diff --git a/staging_vespalib/src/vespa/vespalib/metrics/snapshots.cpp b/staging_vespalib/src/vespa/vespalib/metrics/snapshots.cpp deleted file mode 100644 index 6ef9dca1e26..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/snapshots.cpp +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "snapshots.h" - -namespace vespalib::metrics { - -} diff --git a/staging_vespalib/src/vespa/vespalib/metrics/snapshots.h b/staging_vespalib/src/vespa/vespalib/metrics/snapshots.h deleted file mode 100644 index 2d2f29e7c23..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/snapshots.h +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/vespalib/stllike/string.h> -#include <vector> -#include "counter_aggregator.h" -#include "gauge_aggregator.h" - -namespace vespalib::metrics { - -class DimensionBinding { -private: - const vespalib::string _dimensionName; - const vespalib::string _labelValue; -public: - const vespalib::string &dimensionName() const { return _dimensionName; } - const vespalib::string &labelValue() const { return _labelValue; } - DimensionBinding(const vespalib::string &a, - const vespalib::string &v) noexcept - : _dimensionName(a), _labelValue(v) - {} - ~DimensionBinding() {} -}; - -struct PointSnapshot { - std::vector<DimensionBinding> dimensions; -}; - -class CounterSnapshot { -private: - const vespalib::string _name; - const PointSnapshot &_point; - const size_t _count; -public: - CounterSnapshot(const vespalib::string &n, const PointSnapshot &p, const CounterAggregator &c) - : _name(n), _point(p), _count(c.count) - {} - ~CounterSnapshot() {} - const vespalib::string &name() const { return _name; } - const PointSnapshot &point() const { return _point; } - size_t count() const { return _count; } -}; - -class GaugeSnapshot { -private: - const vespalib::string _name; - const PointSnapshot &_point; - const size_t _observedCount; - const double _averageValue; - const double _sumValue; - const double _minValue; - const double _maxValue; - const double _lastValue; -public: - GaugeSnapshot(const vespalib::string &n, const PointSnapshot &p, const GaugeAggregator &c) - : _name(n), - _point(p), - _observedCount(c.observedCount), - _averageValue(c.sumValue / (c.observedCount > 0 ? c.observedCount : 1)), - _sumValue(c.sumValue), - _minValue(c.minValue), - _maxValue(c.maxValue), - _lastValue(c.lastValue) - {} - ~GaugeSnapshot() {} - const vespalib::string &name() const { return _name; } - const PointSnapshot &point() const { return _point; } - size_t observedCount() const { return _observedCount; } - double averageValue() const { return _averageValue; } - double sumValue() const { return _sumValue; } - double minValue() const { return _minValue; } - double maxValue() const { return _maxValue; } - double lastValue() const { return _lastValue; } -}; - -class Snapshot { -private: - double _start; - double _end; - std::vector<CounterSnapshot> _counters; - std::vector<GaugeSnapshot> _gauges; - std::vector<PointSnapshot> _points; -public: - double startTime() const { return _start; }; // seconds since 1970 - double endTime() const { return _end; }; // seconds since 1970 - - const std::vector<CounterSnapshot> &counters() const { - return _counters; - } - const std::vector<GaugeSnapshot> &gauges() const { - return _gauges; - } - const std::vector<PointSnapshot> &points() const { - return _points; - } - - // builders: - Snapshot(double s, double e) - : _start(s), _end(e), _counters(), _gauges() - {} - ~Snapshot() {} - void add(const PointSnapshot &entry) { _points.push_back(entry); } - void add(const CounterSnapshot &entry) { _counters.push_back(entry); } - void add(const GaugeSnapshot &entry) { _gauges.push_back(entry); } -}; - -} diff --git a/staging_vespalib/src/vespa/vespalib/metrics/stable_store.cpp b/staging_vespalib/src/vespa/vespalib/metrics/stable_store.cpp deleted file mode 100644 index dcb18d0cc82..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/stable_store.cpp +++ /dev/null @@ -1,4 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "stable_store.h" - diff --git a/staging_vespalib/src/vespa/vespalib/metrics/stable_store.h b/staging_vespalib/src/vespa/vespalib/metrics/stable_store.h deleted file mode 100644 index 289d8f84d23..00000000000 --- a/staging_vespalib/src/vespa/vespalib/metrics/stable_store.h +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <memory> -#include <vector> -#include <assert.h> - -namespace vespalib { - -/** metrics-internal utility class */ -template <typename T> -class StableStore -{ - using MyClass = StableStore<T>; - template<typename U> - friend void swap(StableStore<U> &a, StableStore<U> &b); -public: - typedef std::unique_ptr<MyClass> UP; - - StableStore(); - ~StableStore() {} - - void add(T t) { - size_t sz = _mine.size(); - if (sz == _mine.capacity()) { - UP next(new MyClass(_size, std::move(_more), std::move(_mine)));; - _mine.clear(); - _mine.reserve(sz << 1); - _more = std::move(next); - } - _mine.push_back(t); - ++_size; - } - - template<typename FUNC> - void for_each(FUNC &&func) const { - std::vector<const MyClass *> vv; - dffill(vv); - for (const MyClass *p : vv) { - for (const T& elem : p->_mine) { - func(elem); - } - } - } - - size_t size() const { return _size; } - -private: - void dffill(std::vector<const MyClass *> &vv) const { - if (_more) { _more->dffill(vv); } - vv.push_back(this); - } - - StableStore(size_t sz, UP &&more, std::vector<T> &&mine); - - size_t _size; - UP _more; - std::vector<T> _mine; -}; - -template<typename T> -StableStore<T>::StableStore() - : _size(0), - _more(), - _mine() -{ - _mine.reserve(3); -} - -template<typename T> -StableStore<T>::StableStore(size_t sz, UP &&more, std::vector<T> &&mine) - : _size(sz), - _more(std::move(more)), - _mine(std::move(mine)) -{} - -template <typename T> -void swap(StableStore<T> &a, - StableStore<T> &b) -{ - using std::swap; - swap(a._size, b._size); - swap(a._mine, b._mine); - swap(a._more, b._more); -} - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/net/CMakeLists.txt deleted file mode 100644 index f4d0ffdbc4a..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(staging_vespalib_vespalib_net OBJECT - SOURCES - component_config_producer.cpp - generic_state_handler.cpp - http_server.cpp - json_handler_repo.cpp - simple_component_config_producer.cpp - simple_health_producer.cpp - simple_metric_snapshot.cpp - simple_metrics_producer.cpp - slime_explorer.cpp - state_api.cpp - state_explorer.cpp - state_server.cpp - DEPENDS -) diff --git a/staging_vespalib/src/vespa/vespalib/net/component_config_producer.cpp b/staging_vespalib/src/vespa/vespalib/net/component_config_producer.cpp deleted file mode 100644 index 9011ff9abc9..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/component_config_producer.cpp +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "component_config_producer.h" - -namespace vespalib { - -ComponentConfigProducer::Config::~Config() {} - -} diff --git a/staging_vespalib/src/vespa/vespalib/net/component_config_producer.h b/staging_vespalib/src/vespa/vespalib/net/component_config_producer.h deleted file mode 100644 index 81ba8f8b03d..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/component_config_producer.h +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/vespalib/stllike/string.h> - -namespace vespalib { - -struct ComponentConfigProducer { - struct Config { - vespalib::string name; - size_t gen; - vespalib::string msg; - Config(const vespalib::string &n, size_t g) : name(n), gen(g), msg() {} - Config(const vespalib::string &n, size_t g, const vespalib::string &m) - : name(n), gen(g), msg(m) {} - ~Config(); - }; - struct Consumer { - virtual void add(const Config &config) = 0; - virtual ~Consumer() {} - }; - virtual void getComponentConfig(Consumer &consumer) = 0; - virtual ~ComponentConfigProducer() {} -}; - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/generic_state_handler.cpp b/staging_vespalib/src/vespa/vespalib/net/generic_state_handler.cpp deleted file mode 100644 index 91fc6d7e617..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/generic_state_handler.cpp +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "generic_state_handler.h" -#include <vespa/vespalib/data/slime/slime.h> -#include <vespa/vespalib/data/simple_buffer.h> - -namespace vespalib { - -namespace { - -// escape a path component in the URL -// (needed to avoid java.net.URI throwing an exception) - -vespalib::string url_escape(const vespalib::string &item) { - static const char hexdigits[] = "0123456789ABCDEF"; - vespalib::string r; - r.reserve(item.size()); - for (const char c : item) { - if ( ('a' <= c && c <= 'z') - || ('0' <= c && c <= '9') - || ('A' <= c && c <= 'Z') - || (c == '_') - || (c == '-')) - { - r.append(c); - } else { - r.append('%'); - r.append(hexdigits[0xF & (c >> 4)]); - r.append(hexdigits[0xF & c]); - } - } - return r; -} - -class Url { -private: - vespalib::string _url; - void append(const vespalib::string &item) { - if (*_url.rbegin() != '/') { - _url.append('/'); - } - _url.append(url_escape(item)); - } -public: - Url(const vespalib::string &host, const std::vector<vespalib::string> &items) - : _url("http://") - { - _url.append(host); - _url.append('/'); - for (const auto &item: items) { - append(item); - } - } - Url(const Url &parent, const vespalib::string &item) - : _url(parent._url) - { - append(item); - } - const vespalib::string &get() const { return _url; } -}; - -std::vector<vespalib::string> split_path(const vespalib::string &path) { - vespalib::string tmp; - std::vector<vespalib::string> items; - for (size_t i = 0; (i < path.size()) && (path[i] != '?'); ++i) { - if (path[i] == '/') { - if (!tmp.empty()) { - items.push_back(tmp); - tmp.clear(); - } - } else { - tmp.push_back(path[i]); - } - } - if (!tmp.empty()) { - items.push_back(tmp); - } - return items; -} - -bool is_prefix(const std::vector<vespalib::string> &root, const std::vector<vespalib::string> &full) { - if (root.size() > full.size()) { - return false; - } - for (size_t i = 0; i < root.size(); ++i) { - if (root[i] != full[i]) { - return false; - } - } - return true; -} - -void inject_children(const StateExplorer &state, const Url &url, slime::Cursor &self); - -Slime child_state(const StateExplorer &state, const Url &url) { - Slime child_state; - state.get_state(slime::SlimeInserter(child_state), false); - if (child_state.get().type().getId() == slime::NIX::ID) { - inject_children(state, url, child_state.setObject()); - } else { - child_state.get().setString("url", url.get()); - } - return child_state; -} - -void inject_children(const StateExplorer &state, const Url &url, slime::Cursor &self) { - std::vector<vespalib::string> children_names = state.get_children_names(); - for (const vespalib::string &child_name: children_names) { - std::unique_ptr<StateExplorer> child = state.get_child(child_name); - if (child) { - Slime fragment = child_state(*child, Url(url, child_name)); - slime::inject(fragment.get(), slime::ObjectInserter(self, child_name)); - } - } -} - -vespalib::string render(const StateExplorer &state, const Url &url) { - Slime top; - state.get_state(slime::SlimeInserter(top), true); - if (top.get().type().getId() == slime::NIX::ID) { - top.setObject(); - } - inject_children(state, url, top.get()); - SimpleBuffer buf; - slime::JsonFormat::encode(top, buf, true); - return buf.get().make_string(); -} - -vespalib::string explore(const StateExplorer &state, const vespalib::string &host, - const std::vector<vespalib::string> &items, size_t pos) { - if (pos == items.size()) { - return render(state, Url(host, items)); - } - std::unique_ptr<StateExplorer> child = state.get_child(items[pos]); - if (!child) { - return ""; - } - return explore(*child, host, items, pos + 1); -} - -} // namespace vespalib::<unnamed> - -GenericStateHandler::GenericStateHandler(const vespalib::string &root_path, const StateExplorer &state) - : _root(split_path(root_path)), - _state(state) -{ -} - -vespalib::string -GenericStateHandler::get(const vespalib::string &host, - const vespalib::string &path, - const std::map<vespalib::string,vespalib::string> &) const -{ - std::vector<vespalib::string> items = split_path(path); - if (!is_prefix(_root, items)) { - return ""; - } - return explore(_state, host, items, _root.size()); -} - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/generic_state_handler.h b/staging_vespalib/src/vespa/vespalib/net/generic_state_handler.h deleted file mode 100644 index 6065aa165ec..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/generic_state_handler.h +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "json_get_handler.h" -#include "state_explorer.h" -#include <vespa/vespalib/stllike/string.h> -#include <vector> -#include <map> - -namespace vespalib { - -/** - * An implementation of the json get handler interface that exposes - * the state represented by the given state explorer as a browsable - * REST sub-API located below the given root path. - **/ -class GenericStateHandler : public JsonGetHandler -{ -private: - std::vector<vespalib::string> _root; - const StateExplorer &_state; - -public: - GenericStateHandler(const vespalib::string &root_path, const StateExplorer &state); - virtual vespalib::string get(const vespalib::string &host, - const vespalib::string &path, - const std::map<vespalib::string,vespalib::string> ¶ms) const override; -}; - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/health_producer.h b/staging_vespalib/src/vespa/vespalib/net/health_producer.h deleted file mode 100644 index a16ffbd225f..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/health_producer.h +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/vespalib/stllike/string.h> - -namespace vespalib { - -struct HealthProducer { - struct Health { - bool ok; - vespalib::string msg; - Health(bool o, const vespalib::string &m) : ok(o), msg(m) {} - }; - virtual Health getHealth() const = 0; - virtual ~HealthProducer() {} -}; - -} // namespace vespalib - diff --git a/staging_vespalib/src/vespa/vespalib/net/http_server.cpp b/staging_vespalib/src/vespa/vespalib/net/http_server.cpp deleted file mode 100644 index f2c041cb648..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/http_server.cpp +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "http_server.h" -#include <vespa/vespalib/net/crypto_engine.h> - -namespace vespalib { - -void -HttpServer::get(Portal::GetRequest req) -{ - vespalib::string json_result = _handler_repo.get(req.get_host(), req.get_path(), req.export_params()); - if (json_result.empty()) { - req.respond_with_error(404, "Not Found"); - } else { - req.respond_with_content("application/json", json_result); - } -} - -//----------------------------------------------------------------------------- - -HttpServer::HttpServer(int port_in) - : _handler_repo(), - _server(Portal::create(CryptoEngine::get_default(), port_in)), - _root(_server->bind("/", *this)) -{ -} - -HttpServer::~HttpServer() -{ - _root.reset(); -} - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/http_server.h b/staging_vespalib/src/vespa/vespalib/net/http_server.h deleted file mode 100644 index 3d4cd557b58..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/http_server.h +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/vespalib/portal/portal.h> -#include "json_handler_repo.h" - -namespace vespalib { - -/** - * A simple HTTP server that can be used to handle GET requests - * returning json (typically simple read-only REST APIs). Either pass - * a specific port to the constructor or use 0 to bind to a random - * port. Note that you may not ask about the actual port until after - * the server has been started. Request dispatching is done using a - * JsonHandlerRepo. - **/ -class HttpServer : public Portal::GetHandler -{ -private: - JsonHandlerRepo _handler_repo; - Portal::SP _server; - Portal::Token::UP _root; - - void get(Portal::GetRequest req) override; -public: - typedef std::unique_ptr<HttpServer> UP; - HttpServer(int port_in); - ~HttpServer(); - const vespalib::string &host() const { return _server->my_host(); } - JsonHandlerRepo &repo() { return _handler_repo; } - int port() const { return _server->listen_port(); } -}; - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/json_get_handler.h b/staging_vespalib/src/vespa/vespalib/net/json_get_handler.h deleted file mode 100644 index d257bd0285f..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/json_get_handler.h +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/vespalib/stllike/string.h> -#include <map> - -namespace vespalib { - -struct JsonGetHandler { - virtual vespalib::string get(const vespalib::string &host, - const vespalib::string &path, - const std::map<vespalib::string,vespalib::string> ¶ms) const = 0; - virtual ~JsonGetHandler() {} -}; - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/json_handler_repo.cpp b/staging_vespalib/src/vespa/vespalib/net/json_handler_repo.cpp deleted file mode 100644 index 07b9306b5dc..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/json_handler_repo.cpp +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "json_handler_repo.h" -#include <algorithm> - -namespace vespalib { - -namespace { - -template <typename T> -void remove_seq(T &collection, size_t seq) { - collection.erase(std::remove_if(collection.begin(), collection.end(), - [seq](const typename T::value_type &item) - { return (item.seq == seq); }), - collection.end()); -} - -} // namespace vespalib::<unnamed> - -size_t -JsonHandlerRepo::State::bind(vespalib::stringref path_prefix, - const JsonGetHandler &get_handler) -{ - std::lock_guard<std::mutex> guard(lock); - size_t my_seq = ++seq; - hooks.emplace_back(my_seq, path_prefix, get_handler); - std::sort(hooks.begin(), hooks.end()); - return my_seq; -} - -size_t -JsonHandlerRepo::State::add_root_resource(vespalib::stringref path) -{ - std::lock_guard<std::mutex> guard(lock); - size_t my_seq = ++seq; - root_resources.emplace_back(my_seq, path); - return my_seq; -} - -void -JsonHandlerRepo::State::unbind(size_t my_seq) { - std::lock_guard<std::mutex> guard(lock); - remove_seq(hooks, my_seq); - remove_seq(root_resources, my_seq); -} - -//----------------------------------------------------------------------------- - -JsonHandlerRepo::Token::UP -JsonHandlerRepo::bind(vespalib::stringref path_prefix, - const JsonGetHandler &get_handler) -{ - return Token::UP(new Unbinder(_state, _state->bind(path_prefix, get_handler))); -} - -JsonHandlerRepo::Token::UP -JsonHandlerRepo::add_root_resource(vespalib::stringref path) -{ - return Token::UP(new Unbinder(_state, _state->add_root_resource(path))); -} - -std::vector<vespalib::string> -JsonHandlerRepo::get_root_resources() const -{ - std::lock_guard<std::mutex> guard(_state->lock); - std::vector<vespalib::string> result; - for (const Resource &resource: _state->root_resources) { - result.push_back(resource.path); - } - return result; -} - -vespalib::string -JsonHandlerRepo::get(const vespalib::string &host, - const vespalib::string &path, - const std::map<vespalib::string,vespalib::string> ¶ms) const -{ - std::lock_guard<std::mutex> guard(_state->lock); - for (const auto &hook: _state->hooks) { - if (path.find(hook.path_prefix) == 0) { - return hook.handler->get(host, path, params); - } - } - return ""; -} - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/json_handler_repo.h b/staging_vespalib/src/vespa/vespalib/net/json_handler_repo.h deleted file mode 100644 index adbe3010f43..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/json_handler_repo.h +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "json_get_handler.h" -#include <mutex> -#include <memory> -#include <vector> - -namespace vespalib { - -/** - * A repository of json get handlers that is also a json get - * handler. The get function will dispatch the request to the - * appropriate get handler in the repository. The bind function will - * register a handler and return a token that can later be deleted to - * unbind the handler. Each handler is registered with a path - * prefix. If the requested path matches multiple handlers, the one - * with the longest prefix will be selected. If multiple handlers are - * tied for longest prefix, the most recently added handler will be - * selected. - **/ -class JsonHandlerRepo : public JsonGetHandler -{ -public: - struct Token { - typedef std::unique_ptr<Token> UP; - virtual ~Token() {} - }; - -private: - struct Hook { - size_t seq; - vespalib::string path_prefix; - const JsonGetHandler *handler; - Hook(size_t seq_in, - vespalib::stringref prefix_in, - const JsonGetHandler &handler_in) noexcept - : seq(seq_in), path_prefix(prefix_in), handler(&handler_in) {} - bool operator <(const Hook &rhs) const { - if (path_prefix.size() == rhs.path_prefix.size()) { - return (seq > rhs.seq); - } - return (path_prefix.size() > rhs.path_prefix.size()); - } - }; - - struct Resource { - size_t seq; - vespalib::string path; - Resource(size_t seq_in, vespalib::stringref path_in) noexcept - : seq(seq_in), path(path_in) {} - }; - - struct State { - typedef std::shared_ptr<State> SP; - std::mutex lock; - size_t seq; - std::vector<Hook> hooks; - std::vector<Resource> root_resources; - State() noexcept : lock(), seq(0), hooks(), root_resources() {} - size_t bind(vespalib::stringref path_prefix, - const JsonGetHandler &get_handler); - size_t add_root_resource(vespalib::stringref path); - void unbind(size_t my_seq); - }; - - struct Unbinder : Token { - State::SP state; - size_t my_seq; - Unbinder(State::SP state_in, size_t my_seq_in) noexcept - : state(std::move(state_in)), my_seq(my_seq_in) {} - ~Unbinder() override { - state->unbind(my_seq); - } - }; - - std::shared_ptr<State> _state; - -public: - JsonHandlerRepo() : _state(std::make_shared<State>()) {} - Token::UP bind(vespalib::stringref path_prefix, - const JsonGetHandler &get_handler); - Token::UP add_root_resource(vespalib::stringref path); - std::vector<vespalib::string> get_root_resources() const; - vespalib::string get(const vespalib::string &host, - const vespalib::string &path, - const std::map<vespalib::string,vespalib::string> ¶ms) const override; -}; - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/metrics_producer.h b/staging_vespalib/src/vespa/vespalib/net/metrics_producer.h deleted file mode 100644 index d57797d75f6..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/metrics_producer.h +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/vespalib/stllike/string.h> - -namespace vespalib { - -struct MetricsProducer { - virtual vespalib::string getMetrics(const vespalib::string &consumer) = 0; - virtual vespalib::string getTotalMetrics(const vespalib::string &consumer) = 0; - virtual ~MetricsProducer() {} -}; - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_component_config_producer.cpp b/staging_vespalib/src/vespa/vespalib/net/simple_component_config_producer.cpp deleted file mode 100644 index 3fa5f09033f..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/simple_component_config_producer.cpp +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "simple_component_config_producer.h" - -namespace vespalib { - -SimpleComponentConfigProducer::SimpleComponentConfigProducer() - : _lock(), - _state() -{ -} - -SimpleComponentConfigProducer::~SimpleComponentConfigProducer() = default; - -void -SimpleComponentConfigProducer::addConfig(const Config &config) -{ - std::lock_guard guard(_lock); - _state.insert(std::make_pair(config.name, config)).first->second = config; -} - -void -SimpleComponentConfigProducer::removeConfig(const vespalib::string &name) -{ - std::lock_guard guard(_lock); - _state.erase(name); -} - -void -SimpleComponentConfigProducer::getComponentConfig(Consumer &consumer) -{ - std::lock_guard guard(_lock); - for (const auto & entry : _state) { - consumer.add(entry.second); - } -} - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_component_config_producer.h b/staging_vespalib/src/vespa/vespalib/net/simple_component_config_producer.h deleted file mode 100644 index 2f0cc0cf1fa..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/simple_component_config_producer.h +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "component_config_producer.h" -#include <map> -#include <mutex> - -namespace vespalib { - -class SimpleComponentConfigProducer : public ComponentConfigProducer -{ -private: - std::mutex _lock; - std::map<vespalib::string, Config> _state; - -public: - SimpleComponentConfigProducer(); - ~SimpleComponentConfigProducer() override; - void addConfig(const Config &config); - void removeConfig(const vespalib::string &name); - void getComponentConfig(Consumer &consumer) override; -}; - -} // namespace vespalib - diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_health_producer.cpp b/staging_vespalib/src/vespa/vespalib/net/simple_health_producer.cpp deleted file mode 100644 index 3c1959608e9..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/simple_health_producer.cpp +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "simple_health_producer.h" -#include <vespa/defaults.h> -#include <fcntl.h> -#include <sys/stat.h> -#include <unistd.h> - -namespace { -struct DiskPing { - std::string path; - - DiskPing() - : path(vespa::Defaults::underVespaHome("var/run/diskping.")) - { - int pid = getpid(); - while (pid > 0) { - char c = '0' + (pid % 10); - path.append(1, c); - pid /= 10; - } - } - bool failed() { - const char *fn = path.c_str(); - ::unlink(fn); - int fd = ::creat(fn, S_IRWXU); - if (fd < 0) { - return true; - } - int wr = ::write(fd, "foo\n", 4); - int cr = ::close(fd); - ::unlink(fn); - return (wr != 4 || cr != 0); - } -}; - -bool diskFailed() { - static DiskPing disk; - return disk.failed(); -} - -} - -namespace vespalib { - -SimpleHealthProducer::SimpleHealthProducer() - : _lock(), - _health(true, "") -{ - setOk(); -} - -SimpleHealthProducer::~SimpleHealthProducer() = default; - -void -SimpleHealthProducer::setOk() -{ - std::lock_guard guard(_lock); - _health = Health(true, "All OK"); -} - -void -SimpleHealthProducer::setFailed(const vespalib::string &msg) -{ - std::lock_guard guard(_lock); - _health = Health(false, msg); -} - -HealthProducer::Health -SimpleHealthProducer::getHealth() const -{ - std::lock_guard guard(_lock); - if (_health.ok && diskFailed()) { - return Health(false, "disk ping failed"); - } - return _health; -} - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_health_producer.h b/staging_vespalib/src/vespa/vespalib/net/simple_health_producer.h deleted file mode 100644 index b2a7725b89e..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/simple_health_producer.h +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "health_producer.h" -#include <mutex> - -namespace vespalib { - -class SimpleHealthProducer : public HealthProducer -{ -private: - mutable std::mutex _lock; - HealthProducer::Health _health; - -public: - SimpleHealthProducer(); - ~SimpleHealthProducer() override; - void setOk(); - void setFailed(const vespalib::string &msg); - Health getHealth() const override; -}; - -} // namespace vespalib - diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_metric_snapshot.cpp b/staging_vespalib/src/vespa/vespalib/net/simple_metric_snapshot.cpp deleted file mode 100644 index cad2a3567aa..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/simple_metric_snapshot.cpp +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "simple_metric_snapshot.h" - -namespace vespalib { - -SimpleMetricSnapshot::SimpleMetricSnapshot(uint32_t prevTime, uint32_t currTime) - : _data(), - _metrics(_data.setObject()), - _values(_metrics.setArray("values")), - _snapLen(currTime - prevTime) -{ - vespalib::slime::Cursor& snapshot = _metrics.setObject("snapshot"); - snapshot.setLong("from", prevTime); - snapshot.setLong("to", currTime); - if (_snapLen < 1.0) { - _snapLen = 1.0; - } -} - - -void -SimpleMetricSnapshot::addCount(const char *name, const char *desc, uint32_t count) -{ - using namespace vespalib::slime::convenience; - Cursor& value = _values.addObject(); - value.setString("name", name); - value.setString("description", desc); - Cursor& inner = value.setObject("values"); - inner.setLong("count", count); - inner.setDouble("rate", count / _snapLen); -} - -void -SimpleMetricSnapshot::addGauge(const char *name, const char *desc, long gauge) -{ - using namespace vespalib::slime::convenience; - Cursor& value = _values.addObject(); - value.setString("name", name); - value.setString("description", desc); - Cursor& inner = value.setObject("values"); - inner.setLong("average", gauge); - inner.setLong("min", gauge); - inner.setLong("max", gauge); - inner.setLong("last", gauge); - inner.setLong("count", 1); - inner.setDouble("rate", 1.0 / _snapLen); -} - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_metric_snapshot.h b/staging_vespalib/src/vespa/vespalib/net/simple_metric_snapshot.h deleted file mode 100644 index e19f26bd5d0..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/simple_metric_snapshot.h +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/data/slime/slime.h> - -namespace vespalib { - -class SimpleMetricSnapshot -{ -private: - vespalib::Slime _data; - vespalib::slime::Cursor& _metrics; - vespalib::slime::Cursor& _values; - double _snapLen; - -public: - SimpleMetricSnapshot(uint32_t prevTime, uint32_t currTime); - void addCount(const char *name, const char *desc, uint32_t count); - void addGauge(const char *name, const char *desc, long gauge); - - vespalib::string asString() const { - return _data.toString(); - } -}; - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_metrics_producer.cpp b/staging_vespalib/src/vespa/vespalib/net/simple_metrics_producer.cpp deleted file mode 100644 index 1c4539d70ee..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/simple_metrics_producer.cpp +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "simple_metrics_producer.h" - -namespace vespalib { - -SimpleMetricsProducer::SimpleMetricsProducer() - : _lock(), - _metrics(), - _totalMetrics() -{ -} - -SimpleMetricsProducer::~SimpleMetricsProducer() = default; - -void -SimpleMetricsProducer::setMetrics(const vespalib::string &metrics) -{ - std::lock_guard guard(_lock); - _metrics = metrics; -} - -vespalib::string -SimpleMetricsProducer::getMetrics(const vespalib::string &) -{ - std::lock_guard guard(_lock); - return _metrics; -} - -void -SimpleMetricsProducer::setTotalMetrics(const vespalib::string &metrics) -{ - std::lock_guard guard(_lock); - _totalMetrics = metrics; -} - -vespalib::string -SimpleMetricsProducer::getTotalMetrics(const vespalib::string &) -{ - std::lock_guard guard(_lock); - return _totalMetrics; -} - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_metrics_producer.h b/staging_vespalib/src/vespa/vespalib/net/simple_metrics_producer.h deleted file mode 100644 index ad463c63d57..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/simple_metrics_producer.h +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "metrics_producer.h" -#include <mutex> - -namespace vespalib { - -class SimpleMetricsProducer : public MetricsProducer -{ -private: - std::mutex _lock; - vespalib::string _metrics; - vespalib::string _totalMetrics; - -public: - SimpleMetricsProducer(); - ~SimpleMetricsProducer() override; - void setMetrics(const vespalib::string &metrics); - vespalib::string getMetrics(const vespalib::string &consumer) override; - void setTotalMetrics(const vespalib::string &metrics); - vespalib::string getTotalMetrics(const vespalib::string &consumer) override; -}; - -} // namespace vespalib - diff --git a/staging_vespalib/src/vespa/vespalib/net/slime_explorer.cpp b/staging_vespalib/src/vespa/vespalib/net/slime_explorer.cpp deleted file mode 100644 index dc61f673233..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/slime_explorer.cpp +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "slime_explorer.h" - -namespace vespalib { - -namespace { - -struct SelfState : slime::ObjectTraverser { - Slime result; - SelfState() : result() { result.setObject(); } - void field(const Memory &key, const slime::Inspector &value) override { - if (value.type().getId() != slime::OBJECT::ID) { - slime::inject(value, slime::ObjectInserter(result.get(), key)); - } - } -}; - -struct ChildrenNames : slime::ObjectTraverser { - std::vector<vespalib::string> result; - void field(const Memory &key, const slime::Inspector &value) override { - if (value.type().getId() == slime::OBJECT::ID) { - result.push_back(key.make_string()); - } - } -}; - -} // namespace vespalib::<unnamed> - -void -SlimeExplorer::get_state(const slime::Inserter &inserter, bool full) const -{ - SelfState state; - _self.traverse(state); - if (state.result.get().fields() > 0) { - if (full) { - state.result.get().setBool("full", true); - } - slime::inject(state.result.get(), inserter); - } -} - -std::vector<vespalib::string> -SlimeExplorer::get_children_names() const -{ - ChildrenNames names; - _self.traverse(names); - return names.result; -} - -std::unique_ptr<StateExplorer> -SlimeExplorer::get_child(vespalib::stringref name) const -{ - slime::Inspector &child = _self[name]; - if (!child.valid()) { - return std::unique_ptr<StateExplorer>(nullptr); - } - return std::unique_ptr<StateExplorer>(new SlimeExplorer(child)); -} - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/slime_explorer.h b/staging_vespalib/src/vespa/vespalib/net/slime_explorer.h deleted file mode 100644 index cbb0a89e36d..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/slime_explorer.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "state_explorer.h" -#include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/data/slime/slime.h> -#include <vector> -#include <memory> - -namespace vespalib { - -/** - * Simple class exposing the contents of a Slime object through the - * StateExplorer interface (to be used when testing clients of the - * StateExplorer interface). - **/ -class SlimeExplorer : public StateExplorer -{ -private: - const slime::Inspector &_self; - -public: - SlimeExplorer(const slime::Inspector &self) : _self(self) {} - virtual void get_state(const slime::Inserter &inserter, bool full) const override; - virtual std::vector<vespalib::string> get_children_names() const override; - virtual std::unique_ptr<StateExplorer> get_child(vespalib::stringref name) const override; -}; - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/state_api.cpp b/staging_vespalib/src/vespa/vespalib/net/state_api.cpp deleted file mode 100644 index ca00713352f..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/state_api.cpp +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "state_api.h" -#include <vespa/vespalib/util/jsonwriter.h> - -namespace vespalib { - -namespace { - -struct ConfigRenderer : ComponentConfigProducer::Consumer { - JSONStringer &json; - ConfigRenderer(JSONStringer &j) : json(j) {} - void add(const ComponentConfigProducer::Config &config) override { - json.appendKey(config.name); - json.beginObject(); - json.appendKey("generation"); - json.appendInt64(config.gen); - if (!config.msg.empty()) { - json.appendKey("message"); - json.appendString(config.msg); - } - json.endObject(); - } -}; - -struct ConfigGenerationObserver : ComponentConfigProducer::Consumer { - size_t maxGen; - bool seenSome; - ConfigGenerationObserver() : maxGen(0), seenSome(false) {} - void add(const ComponentConfigProducer::Config &config) override { - if (seenSome) { - maxGen = std::max(maxGen, config.gen); - } else { - maxGen = config.gen; - seenSome = true; - } - } -}; - -void build_health_status(JSONStringer &json, const HealthProducer &healthProducer) { - HealthProducer::Health health = healthProducer.getHealth(); - json.appendKey("status"); - json.beginObject(); - json.appendKey("code"); - if (health.ok) { - json.appendString("up"); - } else { - json.appendString("down"); - json.appendKey("message"); - json.appendString(health.msg); - } - json.endObject(); -} - -vespalib::string get_consumer(const std::map<vespalib::string,vespalib::string> ¶ms, - vespalib::stringref default_consumer) -{ - auto consumer_lookup = params.find("consumer"); - if (consumer_lookup == params.end()) { - return default_consumer; - } - return consumer_lookup->second; -} - -void render_link(JSONStringer &json, const vespalib::string &host, const vespalib::string &path) { - json.beginObject(); - json.appendKey("url"); - json.appendString("http://" + host + path); - json.endObject(); -} - -vespalib::string respond_root(const JsonHandlerRepo &repo, const vespalib::string &host) { - JSONStringer json; - json.beginObject(); - json.appendKey("resources"); - json.beginArray(); - for (auto path: {"/state/v1/health", "/state/v1/metrics", "/state/v1/config"}) { - render_link(json, host, path); - } - for (const vespalib::string &path: repo.get_root_resources()) { - render_link(json, host, path); - } - json.endArray(); - json.endObject(); - return json.toString(); -} - -vespalib::string respond_health(const HealthProducer &healthProducer) { - JSONStringer json; - json.beginObject(); - build_health_status(json, healthProducer); - json.endObject(); - return json.toString(); -} - -vespalib::string respond_metrics(const vespalib::string &consumer, - const HealthProducer &healthProducer, - MetricsProducer &metricsProducer) -{ - JSONStringer json; - json.beginObject(); - build_health_status(json, healthProducer); - { // metrics - vespalib::string metrics = metricsProducer.getMetrics(consumer); - if (!metrics.empty()) { - json.appendKey("metrics"); - json.appendJSON(metrics); - } - } - json.endObject(); - return json.toString(); -} - -vespalib::string respond_config(ComponentConfigProducer &componentConfigProducer) { - JSONStringer json; - json.beginObject(); - { // config - ConfigRenderer renderer(json); - json.appendKey("config"); - json.beginObject(); - ConfigGenerationObserver observer; - componentConfigProducer.getComponentConfig(observer); - if (observer.seenSome) { - json.appendKey("generation"); - json.appendInt64(observer.maxGen); - } - componentConfigProducer.getComponentConfig(renderer); - json.endObject(); - } - json.endObject(); - return json.toString(); -} - -} // namespace vespalib::<unnamed> - -vespalib::string -StateApi::get(const vespalib::string &host, - const vespalib::string &path, - const std::map<vespalib::string,vespalib::string> ¶ms) const -{ - if (path == "/state/v1/" || path == "/state/v1") { - return respond_root(_handler_repo, host); - } else if (path == "/state/v1/health") { - return respond_health(_healthProducer); - } else if (path == "/state/v1/metrics") { - // Using a 'statereporter' consumer by default removes many uninteresting per-thread - // metrics but retains their aggregates. - return respond_metrics(get_consumer(params, "statereporter"), _healthProducer, _metricsProducer); - } else if (path == "/state/v1/config") { - return respond_config(_componentConfigProducer); - } else if (path == "/metrics/total") { - return _metricsProducer.getTotalMetrics(get_consumer(params, "")); - } else { - return _handler_repo.get(host, path, params); - } -} - -//----------------------------------------------------------------------------- - -StateApi::StateApi(const HealthProducer &hp, - MetricsProducer &mp, - ComponentConfigProducer &ccp) - : _healthProducer(hp), - _metricsProducer(mp), - _componentConfigProducer(ccp) -{ -} - -StateApi::~StateApi() = default; - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/state_api.h b/staging_vespalib/src/vespa/vespalib/net/state_api.h deleted file mode 100644 index 2bdf320d091..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/state_api.h +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "json_get_handler.h" -#include "health_producer.h" -#include "metrics_producer.h" -#include "component_config_producer.h" -#include <memory> -#include "json_handler_repo.h" - -namespace vespalib { - -/** - * This class uses the underlying producer interfaces passed to the - * constructor to implement the 'state' REST API. The get function is - * a simple abstraction of a GET request returning json and can be - * wired into the HttpServer or called directly. - **/ -class StateApi : public JsonGetHandler -{ -private: - const HealthProducer &_healthProducer; - MetricsProducer &_metricsProducer; - ComponentConfigProducer &_componentConfigProducer; - JsonHandlerRepo _handler_repo; - -public: - StateApi(const HealthProducer &hp, - MetricsProducer &mp, - ComponentConfigProducer &ccp); - ~StateApi() override; - vespalib::string get(const vespalib::string &host, - const vespalib::string &path, - const std::map<vespalib::string,vespalib::string> ¶ms) const override; - JsonHandlerRepo &repo() { return _handler_repo; } -}; - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/state_explorer.cpp b/staging_vespalib/src/vespa/vespalib/net/state_explorer.cpp deleted file mode 100644 index 627b70a92fc..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/state_explorer.cpp +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "state_explorer.h" - -namespace vespalib { - -std::vector<vespalib::string> -StateExplorer::get_children_names() const -{ - return std::vector<vespalib::string>(); -} - -std::unique_ptr<StateExplorer> -StateExplorer::get_child(vespalib::stringref) const -{ - return std::unique_ptr<StateExplorer>(nullptr); -} - -StateExplorer::~StateExplorer() -{ -} - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/state_explorer.h b/staging_vespalib/src/vespa/vespalib/net/state_explorer.h deleted file mode 100644 index 01cb94663b3..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/state_explorer.h +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/data/slime/inserter.h> -#include <vector> -#include <memory> - -namespace vespalib { - -/** - * Interface used to traverse and expose state of a given component and its children. - */ -struct StateExplorer { - virtual void get_state(const slime::Inserter &inserter, bool full) const = 0; - virtual std::vector<vespalib::string> get_children_names() const; - virtual std::unique_ptr<StateExplorer> get_child(vespalib::stringref name) const; - virtual ~StateExplorer(); -}; - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/state_server.cpp b/staging_vespalib/src/vespa/vespalib/net/state_server.cpp deleted file mode 100644 index 8d927387727..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/state_server.cpp +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "state_server.h" - -namespace vespalib { - -StateServer::StateServer(int port, - const HealthProducer &hp, - MetricsProducer &mp, - ComponentConfigProducer &ccp) - : _api(hp, mp, ccp), - _server(port), - _tokens() -{ - _tokens.push_back(_server.repo().bind("/state/v1", _api)); - _tokens.push_back(_server.repo().bind("/metrics/total", _api)); -} - -StateServer::~StateServer() = default; - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/state_server.h b/staging_vespalib/src/vespa/vespalib/net/state_server.h deleted file mode 100644 index 22152fe1a1f..00000000000 --- a/staging_vespalib/src/vespa/vespalib/net/state_server.h +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "http_server.h" -#include "state_api.h" -#include "json_get_handler.h" -#include "health_producer.h" -#include "metrics_producer.h" -#include "component_config_producer.h" -#include "json_handler_repo.h" - -namespace vespalib { - -/** - * An all-in-one server making it simple for applications to serve the - * 'state' REST API over HTTP. - **/ -class StateServer -{ -private: - StateApi _api; - HttpServer _server; - std::vector<JsonHandlerRepo::Token::UP> _tokens; - -public: - typedef std::unique_ptr<StateServer> UP; - StateServer(int port, const HealthProducer &hp, MetricsProducer &mp, ComponentConfigProducer &ccp); - ~StateServer(); - int getListenPort() { return _server.port(); } - JsonHandlerRepo &repo() { return _api.repo(); } -}; - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt index 0bdfed505d3..dec951f9ef2 100644 --- a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -1,27 +1,20 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(staging_vespalib_vespalib_util OBJECT SOURCES - adaptive_sequenced_executor.cpp bits.cpp clock.cpp crc.cpp document_runnable.cpp doom.cpp - foregroundtaskexecutor.cpp growablebytebuffer.cpp - isequencedtaskexecutor.cpp jsonexception.cpp jsonstream.cpp - jsonwriter.cpp malloc_mmap_guard.cpp process_memory_stats.cpp programoptions.cpp programoptions_testutils.cpp rusage.cpp - sequencedtaskexecutor.cpp - sequencedtaskexecutorobserver.cpp shutdownguard.cpp - singleexecutor.cpp testclock.cpp xmlserializable.cpp xmlstream.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp b/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp deleted file mode 100644 index 6db97ff0761..00000000000 --- a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp +++ /dev/null @@ -1,340 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "adaptive_sequenced_executor.h" - -namespace vespalib { - -//----------------------------------------------------------------------------- - -AdaptiveSequencedExecutor::Strand::Strand() - : state(State::IDLE), - queue() -{ -} - -AdaptiveSequencedExecutor::Strand::~Strand() -{ - assert(queue.empty()); -} - -//----------------------------------------------------------------------------- - -AdaptiveSequencedExecutor::Worker::Worker() - : cond(), - idleTracker(), - state(State::RUNNING), - strand(nullptr) -{ -} - -AdaptiveSequencedExecutor::Worker::~Worker() -{ - assert(state == State::DONE); - assert(strand == nullptr); -} - -//----------------------------------------------------------------------------- - -AdaptiveSequencedExecutor::Self::Self() - : cond(), - state(State::OPEN), - waiting_tasks(0), - pending_tasks(0) -{ -} - -AdaptiveSequencedExecutor::Self::~Self() -{ - assert(state == State::CLOSED); - assert(waiting_tasks == 0); - assert(pending_tasks == 0); -} - -//----------------------------------------------------------------------------- - -AdaptiveSequencedExecutor::ThreadTools::ThreadTools(AdaptiveSequencedExecutor &parent_in) - : parent(parent_in), - pool(std::make_unique<FastOS_ThreadPool>(STACK_SIZE)), - allow_worker_exit() -{ -} - -AdaptiveSequencedExecutor::ThreadTools::~ThreadTools() -{ - assert(pool->isClosed()); -} - -void -AdaptiveSequencedExecutor::ThreadTools::Run(FastOS_ThreadInterface *, void *) -{ - parent.worker_main(); -} - -void -AdaptiveSequencedExecutor::ThreadTools::start(size_t num_threads) -{ - for (size_t i = 0; i < num_threads; ++i) { - FastOS_ThreadInterface *thread = pool->NewThread(this); - assert(thread != nullptr); - (void)thread; - } -} - -void -AdaptiveSequencedExecutor::ThreadTools::close() -{ - allow_worker_exit.countDown(); - pool->Close(); -} - -//----------------------------------------------------------------------------- - -void -AdaptiveSequencedExecutor::maybe_block_self(std::unique_lock<std::mutex> &lock) -{ - while (_self.state == Self::State::BLOCKED) { - _self.cond.wait(lock); - } - while ((_self.state == Self::State::OPEN) && _cfg.is_above_max_pending(_self.pending_tasks)) { - _self.state = Self::State::BLOCKED; - while (_self.state == Self::State::BLOCKED) { - _self.cond.wait(lock); - } - } -} - -void -AdaptiveSequencedExecutor::maybe_unblock_self(const std::unique_lock<std::mutex> &) -{ - if ((_self.state == Self::State::BLOCKED) && (_self.pending_tasks < _cfg.wakeup_limit)) { - _self.state = Self::State::OPEN; - _self.cond.notify_all(); - } -} - -void -AdaptiveSequencedExecutor::maybe_wake_worker(const std::unique_lock<std::mutex> &) -{ - if ((_self.waiting_tasks > _cfg.max_waiting) && (!_worker_stack.empty())) { - assert(!_wait_queue.empty()); - Worker *worker = _worker_stack.back(); - _worker_stack.popBack(); - assert(worker->state == Worker::State::BLOCKED); - assert(worker->strand == nullptr); - worker->state = Worker::State::RUNNING; - worker->strand = _wait_queue.front(); - _wait_queue.pop(); - assert(worker->strand->state == Strand::State::WAITING); - assert(!worker->strand->queue.empty()); - worker->strand->state = Strand::State::ACTIVE; - assert(_self.waiting_tasks >= worker->strand->queue.size()); - _self.waiting_tasks -= worker->strand->queue.size(); - worker->cond.notify_one(); - } -} - -bool -AdaptiveSequencedExecutor::obtain_strand(Worker &worker, std::unique_lock<std::mutex> &lock) -{ - assert(worker.strand == nullptr); - if (!_wait_queue.empty()) { - worker.strand = _wait_queue.front(); - _wait_queue.pop(); - assert(worker.strand->state == Strand::State::WAITING); - assert(!worker.strand->queue.empty()); - worker.strand->state = Strand::State::ACTIVE; - assert(_self.waiting_tasks >= worker.strand->queue.size()); - _self.waiting_tasks -= worker.strand->queue.size(); - } else if (_self.state == Self::State::CLOSED) { - worker.state = Worker::State::DONE; - } else { - worker.state = Worker::State::BLOCKED; - _worker_stack.push(&worker); - worker.idleTracker.set_idle(steady_clock::now()); - while (worker.state == Worker::State::BLOCKED) { - worker.cond.wait(lock); - } - _idleTracker.was_idle(worker.idleTracker.set_active(steady_clock::now())); - _stats.wakeupCount++; - } - return (worker.state == Worker::State::RUNNING); -} - -bool -AdaptiveSequencedExecutor::exchange_strand(Worker &worker, std::unique_lock<std::mutex> &lock) -{ - if (worker.strand == nullptr) { - return obtain_strand(worker, lock); - } - if (worker.strand->queue.empty()) { - worker.strand->state = Strand::State::IDLE; - worker.strand = nullptr; - return obtain_strand(worker, lock); - } - if (!_wait_queue.empty()) { - worker.strand->state = Strand::State::WAITING; - _self.waiting_tasks += worker.strand->queue.size(); - _wait_queue.push(worker.strand); - worker.strand = nullptr; - return obtain_strand(worker, lock); - } - return true; -} - -AdaptiveSequencedExecutor::TaggedTask -AdaptiveSequencedExecutor::next_task(Worker &worker, std::optional<uint32_t> prev_token) -{ - TaggedTask task; - auto guard = std::unique_lock(_mutex); - if (prev_token.has_value()) { - _barrier.completeEvent(prev_token.value()); - } - if (exchange_strand(worker, guard)) { - assert(worker.state == Worker::State::RUNNING); - assert(worker.strand != nullptr); - assert(!worker.strand->queue.empty()); - task = std::move(worker.strand->queue.front()); - worker.strand->queue.pop(); - _stats.queueSize.add(--_self.pending_tasks); - maybe_wake_worker(guard); - } else { - assert(worker.state == Worker::State::DONE); - assert(worker.strand == nullptr); - } - maybe_unblock_self(guard); - return task; -} - -void -AdaptiveSequencedExecutor::worker_main() -{ - Worker worker; - std::optional<uint32_t> prev_token = std::nullopt; - while (TaggedTask my_task = next_task(worker, prev_token)) { - my_task.task->run(); - prev_token = my_task.token; - } - _thread_tools->allow_worker_exit.await(); -} - -AdaptiveSequencedExecutor::AdaptiveSequencedExecutor(size_t num_strands, size_t num_threads, - size_t max_waiting, size_t max_pending, - bool is_max_pending_hard) - : ISequencedTaskExecutor(num_strands), - _thread_tools(std::make_unique<ThreadTools>(*this)), - _mutex(), - _strands(num_strands), - _wait_queue(num_strands), - _worker_stack(num_threads), - _self(), - _stats(), - _idleTracker(steady_clock::now()), - _cfg(num_threads, max_waiting, max_pending, is_max_pending_hard) -{ - _stats.queueSize.add(_self.pending_tasks); - _thread_tools->start(num_threads); -} - -AdaptiveSequencedExecutor::~AdaptiveSequencedExecutor() -{ - sync_all(); - { - auto guard = std::unique_lock(_mutex); - assert(_self.state == Self::State::OPEN); - _self.state = Self::State::CLOSED; - while (!_worker_stack.empty()) { - Worker *worker = _worker_stack.back(); - _worker_stack.popBack(); - assert(worker->state == Worker::State::BLOCKED); - assert(worker->strand == nullptr); - worker->state = Worker::State::DONE; - worker->cond.notify_one(); - } - _self.cond.notify_all(); - } - _thread_tools->close(); - assert(_wait_queue.empty()); - assert(_worker_stack.empty()); -} - -ISequencedTaskExecutor::ExecutorId -AdaptiveSequencedExecutor::getExecutorId(uint64_t component) const { - return ExecutorId(component % _strands.size()); -} - -void -AdaptiveSequencedExecutor::executeTask(ExecutorId id, Task::UP task) -{ - assert(id.getId() < _strands.size()); - Strand &strand = _strands[id.getId()]; - auto guard = std::unique_lock(_mutex); - assert(_self.state != Self::State::CLOSED); - maybe_block_self(guard); - strand.queue.push(TaggedTask(std::move(task), _barrier.startEvent())); - _stats.queueSize.add(++_self.pending_tasks); - ++_stats.acceptedTasks; - if (strand.state == Strand::State::WAITING) { - ++_self.waiting_tasks; - } else if (strand.state == Strand::State::IDLE) { - if (_worker_stack.size() < _cfg.num_threads) { - strand.state = Strand::State::WAITING; - _wait_queue.push(&strand); - _self.waiting_tasks += strand.queue.size(); - } else { - strand.state = Strand::State::ACTIVE; - assert(_wait_queue.empty()); - Worker *worker = _worker_stack.back(); - _worker_stack.popBack(); - assert(worker->state == Worker::State::BLOCKED); - assert(worker->strand == nullptr); - worker->state = Worker::State::RUNNING; - worker->strand = &strand; - worker->cond.notify_one(); - } - } -} - -void -AdaptiveSequencedExecutor::sync_all() -{ - BarrierCompletion barrierCompletion; - { - auto guard = std::lock_guard(_mutex); - if (!_barrier.startBarrier(barrierCompletion)) { - return; - } - } - barrierCompletion.gate.await(); -} - -void -AdaptiveSequencedExecutor::setTaskLimit(uint32_t task_limit) -{ - auto guard = std::unique_lock(_mutex); - _cfg.set_max_pending(task_limit); - maybe_unblock_self(guard); -} - -ExecutorStats -AdaptiveSequencedExecutor::getStats() -{ - auto guard = std::lock_guard(_mutex); - ExecutorStats stats = _stats; - steady_time now = steady_clock::now(); - for (size_t i(0); i < _worker_stack.size(); i++) { - _idleTracker.was_idle(_worker_stack.access(i)->idleTracker.reset(now)); - } - stats.setUtil(_cfg.num_threads, _idleTracker.reset(now, _cfg.num_threads)); - _stats = ExecutorStats(); - _stats.queueSize.add(_self.pending_tasks); - return stats; -} - -AdaptiveSequencedExecutor::Config -AdaptiveSequencedExecutor::get_config() const -{ - auto guard = std::lock_guard(_mutex); - return _cfg; -} - -} diff --git a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h b/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h deleted file mode 100644 index fbebf8b4e4c..00000000000 --- a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h +++ /dev/null @@ -1,165 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "isequencedtaskexecutor.h" -#include <vespa/vespalib/util/executor_idle_tracking.h> -#include <vespa/vespalib/util/arrayqueue.hpp> -#include <vespa/vespalib/util/gate.h> -#include <vespa/vespalib/util/eventbarrier.hpp> -#include <vespa/fastos/thread.h> -#include <mutex> -#include <condition_variable> -#include <optional> -#include <cassert> - -namespace vespalib { - -/** - * Sequenced executor that balances the number of active threads in - * order to optimize for throughput over latency by minimizing the - * number of critical-path wakeups. - **/ -class AdaptiveSequencedExecutor : public ISequencedTaskExecutor -{ -private: - using Task = Executor::Task; - - struct TaggedTask { - Task::UP task; - uint32_t token; - TaggedTask() : task(nullptr), token(0) {} - TaggedTask(Task::UP task_in, uint32_t token_in) - : task(std::move(task_in)), token(token_in) {} - TaggedTask(TaggedTask &&rhs) = default; - TaggedTask(const TaggedTask &rhs) = delete; - TaggedTask &operator=(const TaggedTask &rhs) = delete; - TaggedTask &operator=(TaggedTask &&rhs) { - assert(task.get() == nullptr); // no overwrites - task = std::move(rhs.task); - token = rhs.token; - return *this; - } - operator bool() const { return bool(task); } - }; - - /** - * Values used to configure the executor. - **/ - struct Config { - size_t num_threads; - size_t max_waiting; - size_t max_pending; - size_t wakeup_limit; - bool is_max_pending_hard; - void set_max_pending(size_t max_pending_in) { - max_pending = std::max(1uL, max_pending_in); - wakeup_limit = std::max(1uL, size_t(max_pending * 0.9)); - assert(wakeup_limit > 0); - assert(wakeup_limit <= max_pending); - } - bool is_above_max_pending(size_t pending) { - return (pending >= max_pending) && is_max_pending_hard; - } - Config(size_t num_threads_in, size_t max_waiting_in, size_t max_pending_in, bool is_max_pending_hard_in) - : num_threads(num_threads_in), - max_waiting(max_waiting_in), - max_pending(1000), - wakeup_limit(900), - is_max_pending_hard(is_max_pending_hard_in) - { - assert(num_threads > 0); - set_max_pending(max_pending_in); - } - }; - - /** - * Tasks that need to be sequenced are handled by a single strand. - **/ - struct Strand { - enum class State { IDLE, WAITING, ACTIVE }; - State state; - ArrayQueue<TaggedTask> queue; - Strand(); - ~Strand(); - }; - - /** - * The state of a single worker thread. - **/ - struct Worker { - enum class State { RUNNING, BLOCKED, DONE }; - std::condition_variable cond; - ThreadIdleTracker idleTracker; - State state; - Strand *strand; - Worker(); - ~Worker(); - }; - - /** - * State related to the executor itself. - **/ - struct Self { - enum class State { OPEN, BLOCKED, CLOSED }; - std::condition_variable cond; - State state; - size_t waiting_tasks; - size_t pending_tasks; - Self(); - ~Self(); - }; - - /** - * Stuff related to worker thread startup and shutdown. - **/ - struct ThreadTools : FastOS_Runnable { - static constexpr size_t STACK_SIZE = (256 * 1024); - AdaptiveSequencedExecutor &parent; - std::unique_ptr<FastOS_ThreadPool> pool; - Gate allow_worker_exit; - ThreadTools(AdaptiveSequencedExecutor &parent_in); - ~ThreadTools(); - void Run(FastOS_ThreadInterface *, void *) override; - void start(size_t num_threads); - void close(); - }; - - struct BarrierCompletion { - Gate gate; - void completeBarrier() { gate.countDown(); } - }; - - std::unique_ptr<ThreadTools> _thread_tools; - mutable std::mutex _mutex; - std::vector<Strand> _strands; - ArrayQueue<Strand*> _wait_queue; - ArrayQueue<Worker*> _worker_stack; - EventBarrier<BarrierCompletion> _barrier; - Self _self; - ExecutorStats _stats; - ExecutorIdleTracker _idleTracker; - Config _cfg; - - void maybe_block_self(std::unique_lock<std::mutex> &lock); - void maybe_unblock_self(const std::unique_lock<std::mutex> &lock); - - void maybe_wake_worker(const std::unique_lock<std::mutex> &lock); - bool obtain_strand(Worker &worker, std::unique_lock<std::mutex> &lock); - bool exchange_strand(Worker &worker, std::unique_lock<std::mutex> &lock); - TaggedTask next_task(Worker &worker, std::optional<uint32_t> prev_token); - void worker_main(); -public: - AdaptiveSequencedExecutor(size_t num_strands, size_t num_threads, - size_t max_waiting, size_t max_pending, - bool is_max_pending_hard); - ~AdaptiveSequencedExecutor() override; - ExecutorId getExecutorId(uint64_t component) const override; - void executeTask(ExecutorId id, Task::UP task) override; - void sync_all() override; - void setTaskLimit(uint32_t task_limit) override; - ExecutorStats getStats() override; - Config get_config() const; -}; - -} diff --git a/staging_vespalib/src/vespa/vespalib/util/foreground_thread_executor.h b/staging_vespalib/src/vespa/vespalib/util/foreground_thread_executor.h deleted file mode 100644 index c1e56572614..00000000000 --- a/staging_vespalib/src/vespa/vespalib/util/foreground_thread_executor.h +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/vespalib/util/threadexecutor.h> -#include <atomic> - -namespace vespalib { - -/** - * Implementation of the ThreadExecutor interface that runs all tasks in the foreground by the calling thread. - */ -class ForegroundThreadExecutor : public vespalib::ThreadExecutor { -private: - std::atomic<size_t> _accepted; - -public: - ForegroundThreadExecutor() : _accepted(0) { } - Task::UP execute(Task::UP task) override { - task->run(); - ++_accepted; - return Task::UP(); - } - size_t getNumThreads() const override { return 0; } - ExecutorStats getStats() override { - return ExecutorStats(ExecutorStats::QueueSizeT(), _accepted.load(std::memory_order_relaxed), 0, 0); - } - void setTaskLimit(uint32_t taskLimit) override { (void) taskLimit; } - uint32_t getTaskLimit() const override { return std::numeric_limits<uint32_t>::max(); } - void wakeup() override { } -}; - -} diff --git a/staging_vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.cpp deleted file mode 100644 index ce5237f41c9..00000000000 --- a/staging_vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.cpp +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "foregroundtaskexecutor.h" -#include <cassert> - -namespace vespalib { - -ForegroundTaskExecutor::ForegroundTaskExecutor() - : ForegroundTaskExecutor(1) -{ -} - -ForegroundTaskExecutor::ForegroundTaskExecutor(uint32_t threads) - : ISequencedTaskExecutor(threads), - _accepted(0) -{ -} - -ForegroundTaskExecutor::~ForegroundTaskExecutor() = default; - -void -ForegroundTaskExecutor::executeTask(ExecutorId id, Executor::Task::UP task) -{ - assert(id.getId() < getNumExecutors()); - task->run(); - _accepted++; -} - -void -ForegroundTaskExecutor::sync_all() -{ -} - -void ForegroundTaskExecutor::setTaskLimit(uint32_t) { - -} - -ExecutorStats ForegroundTaskExecutor::getStats() { - return ExecutorStats(ExecutorStats::QueueSizeT(0) , _accepted.load(std::memory_order_relaxed), 0, 0); -} - -ISequencedTaskExecutor::ExecutorId -ForegroundTaskExecutor::getExecutorId(uint64_t componentId) const { - return ExecutorId(componentId%getNumExecutors()); -} - -} diff --git a/staging_vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.h b/staging_vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.h deleted file mode 100644 index 615bf62afe5..00000000000 --- a/staging_vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.h +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "isequencedtaskexecutor.h" -#include <atomic> - -namespace vespalib { - -/** - * Class to run multiple tasks in parallel, but tasks with same - * id has to be run in sequence. - * - * Currently, this is a dummy version that runs everything in the foreground. - */ -class ForegroundTaskExecutor : public ISequencedTaskExecutor -{ -public: - using ISequencedTaskExecutor::getExecutorId; - - ForegroundTaskExecutor(); - ForegroundTaskExecutor(uint32_t threads); - ~ForegroundTaskExecutor() override; - - ExecutorId getExecutorId(uint64_t componentId) const override; - void executeTask(ExecutorId id, Executor::Task::UP task) override; - void sync_all() override; - void setTaskLimit(uint32_t taskLimit) override; - ExecutorStats getStats() override; -private: - std::atomic<uint64_t> _accepted; -}; - -} // namespace search diff --git a/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp deleted file mode 100644 index b31d72da3b1..00000000000 --- a/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "isequencedtaskexecutor.h" -#include <vespa/vespalib/stllike/hash_fun.h> - -namespace vespalib { - -ISequencedTaskExecutor::ISequencedTaskExecutor(uint32_t numExecutors) - : _numExecutors(numExecutors) -{ -} - -ISequencedTaskExecutor::~ISequencedTaskExecutor() = default; - -void -ISequencedTaskExecutor::executeTasks(TaskList tasks) { - for (auto & task : tasks) { - executeTask(task.first, std::move(task.second)); - } -} - -ISequencedTaskExecutor::ExecutorId -ISequencedTaskExecutor::getExecutorIdFromName(stringref componentId) const { - hash<stringref> hashfun; - return getExecutorId(hashfun(componentId)); -} - -ISequencedTaskExecutor::ExecutorId -ISequencedTaskExecutor::get_alternate_executor_id(ExecutorId id, uint32_t bias) const -{ - if ((bias % _numExecutors) == 0) { - bias = 1; - } - return ExecutorId((id.getId() + bias) % _numExecutors); -} - -} diff --git a/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.h b/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.h deleted file mode 100644 index ff90556e3e4..00000000000 --- a/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.h +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/vespalib/util/executor.h> -#include <vespa/vespalib/util/executor_stats.h> -#include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/util/lambdatask.h> -#include <vector> -#include <mutex> - -namespace vespalib { - -/** - * Interface class to run multiple tasks in parallel, but tasks with same - * id has to be run in sequence. - */ -class ISequencedTaskExecutor : public IWakeup -{ -public: - class ExecutorId { - public: - ExecutorId() noexcept : ExecutorId(0) { } - explicit ExecutorId(uint32_t id) noexcept : _id(id) { } - uint32_t getId() const noexcept { return _id; } - bool operator != (ExecutorId rhs) const noexcept { return _id != rhs._id; } - bool operator == (ExecutorId rhs) const noexcept { return _id == rhs._id; } - bool operator < (ExecutorId rhs) const noexcept { return _id < rhs._id; } - private: - uint32_t _id; - }; - using TaskList = std::vector<std::pair<ExecutorId, Executor::Task::UP>>; - ISequencedTaskExecutor(uint32_t numExecutors); - virtual ~ISequencedTaskExecutor(); - - /** - * Calculate which executor will handle an component. - * - * @param componentId component id - * @return executor id - */ - virtual ExecutorId getExecutorId(uint64_t componentId) const = 0; - uint32_t getNumExecutors() const { return _numExecutors; } - - ExecutorId getExecutorIdFromName(stringref componentId) const; - - /** - * Returns an executor id that is NOT equal to the given executor id, - * using the given bias to offset the new id. - * - * This is relevant for pipelining operations on the same component, - * by doing pipeline steps in different executors. - */ - ExecutorId get_alternate_executor_id(ExecutorId id, uint32_t bias) const; - - /** - * Schedule a task to run after all previously scheduled tasks with - * same id. - * - * @param id which internal executor to use - * @param task unique pointer to the task to be executed - */ - virtual void executeTask(ExecutorId id, Executor::Task::UP task) = 0; - /** - * Schedule a list of tasks to run after all previously scheduled tasks with - * same id. Default is to just iterate and execute one by one, but implementations - * that can schedule all in one go more efficiently can implement this one. - */ - virtual void executeTasks(TaskList tasks); - /** - * Call this one to ensure you get the attention of the workers. - */ - void wakeup() override { } - - /** - * Wrap lambda function into a task and schedule it to be run. - * Caller must ensure that pointers and references are valid and - * call sync_all before tearing down pointed to/referenced data. - * - * @param id which internal executor to use - * @param function function to be wrapped in a task and later executed - */ - template <class FunctionType> - void executeLambda(ExecutorId id, FunctionType &&function) { - executeTask(id, makeLambdaTask(std::forward<FunctionType>(function))); - } - /** - * Wait for all scheduled tasks to complete. - */ - virtual void sync_all() = 0; - - virtual void setTaskLimit(uint32_t taskLimit) = 0; - - virtual ExecutorStats getStats() = 0; - - /** - * Wrap lambda function into a task and schedule it to be run. - * Caller must ensure that pointers and references are valid and - * call sync_all before tearing down pointed to/referenced data. - * - * @param componentId component id - * @param function function to be wrapped in a task and later executed - */ - template <class FunctionType> - void execute(uint64_t componentId, FunctionType &&function) { - ExecutorId id = getExecutorId(componentId); - executeTask(id, makeLambdaTask(std::forward<FunctionType>(function))); - } - - /** - * Wrap lambda function into a task and schedule it to be run. - * Caller must ensure that pointers and references are valid and - * call sync_all before tearing down pointed to/referenced data. - * - * @param id executor id - * @param function function to be wrapped in a task and later executed - */ - template <class FunctionType> - void execute(ExecutorId id, FunctionType &&function) { - executeTask(id, makeLambdaTask(std::forward<FunctionType>(function))); - } - -private: - uint32_t _numExecutors; -}; - -} diff --git a/staging_vespalib/src/vespa/vespalib/util/jsonwriter.cpp b/staging_vespalib/src/vespa/vespalib/util/jsonwriter.cpp deleted file mode 100644 index eb56d8b367a..00000000000 --- a/staging_vespalib/src/vespa/vespalib/util/jsonwriter.cpp +++ /dev/null @@ -1,286 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "jsonwriter.h" -#include <vespa/vespalib/stllike/asciistream.h> -#include <cmath> -#include <cassert> - -namespace vespalib { - -void -JSONWriter::push(State next) -{ - _stack.push_back(next); -} - -void -JSONWriter::pop(State expected) -{ - State actual = _stack.back(); - assert(actual == expected); - (void) actual; - (void) expected; - _stack.pop_back(); -} - -void -JSONWriter::considerComma() -{ - if (_comma) { - (*_os) << ','; - } -} - -void -JSONWriter::updateCommaState() -{ - if (_stack.back() == ARRAY || _stack.back() == OBJECT) { - _comma = true; - } else { - _comma = false; - } -} - -void -JSONWriter::quote(const char * str, size_t len) -{ - std::vector<char> v((len+1)*2 + 1); - v[0] = '\"'; - size_t j(1); - for (size_t i = 0; i < len; ++i) { - switch (str[i]) { - case '\b': v[j++] = '\\'; v[j++] = 'b'; break; - case '\f': v[j++] = '\\'; v[j++] = 'f'; break; - case '\n': v[j++] = '\\'; v[j++] = 'n'; break; - case '\r': v[j++] = '\\'; v[j++] = 'r'; break; - case '\t': v[j++] = '\\'; v[j++] = 't'; break; - case '\"': - case '\\': - v[j++] = '\\'; - [[fallthrough]]; - default: - v[j++] = str[i]; - break; - } - } - v[j++] = '\"'; - v[j++] = 0; - (*_os) << &v[0]; -} - -JSONWriter::JSONWriter() : - _os(NULL), - _stack(), - _comma(false), - _pretty(false), - _indent(0) -{ - clear(); -} - -JSONWriter::JSONWriter(vespalib::asciistream & output) : - _os(&output), - _stack(), - _comma(false), - _pretty(false), - _indent(0) -{ - clear(); - (*_os) << vespalib::asciistream::Precision(16) << vespalib::forcedot; -} - -JSONWriter::~JSONWriter() {} - -JSONWriter& -JSONWriter::setOutputStream(vespalib::asciistream & output) { - _os = &output; - (*_os) << vespalib::asciistream::Precision(16) << vespalib::forcedot; - return *this; -} - -void -JSONWriter::indent() -{ - if (_pretty) { - (*_os) << "\n"; - for (uint32_t i = 0; i < _indent; ++i) { - (*_os) << " "; - } - } -} - -JSONWriter & -JSONWriter::clear() -{ - _stack.clear(); - _stack.push_back(INIT); - _comma = false; - return *this; -} - -JSONWriter & -JSONWriter::beginObject() -{ - push(OBJECT); - considerComma(); - indent(); - (*_os) << '{'; - _indent++; - _comma = false; - return *this; -} - -JSONWriter & -JSONWriter::endObject() -{ - pop(OBJECT); - _indent--; - indent(); - (*_os) << '}'; - updateCommaState(); - return *this; -} - -JSONWriter & -JSONWriter::beginArray() -{ - push(ARRAY); - considerComma(); - indent(); - (*_os) << '['; - _indent++; - _comma = false; - return *this; -} - -JSONWriter & -JSONWriter::endArray() -{ - pop(ARRAY); - _indent--; - indent(); - (*_os) << ']'; - updateCommaState(); - return *this; -} - -JSONWriter & -JSONWriter::appendNull() -{ - considerComma(); - (*_os) << "null"; - updateCommaState(); - return *this; -} - -JSONWriter & -JSONWriter::appendKey(stringref str) -{ - considerComma(); - indent(); - quote(str.data(), str.size()); - (*_os) << ':'; - _comma = false; - return *this; -} - -JSONWriter & -JSONWriter::appendBool(bool v) -{ - considerComma(); - (*_os) << (v ? "true" : "false"); - updateCommaState(); - return *this; -} - -JSONWriter & -JSONWriter::appendDouble(double v) -{ - considerComma(); - if (!std::isinf(v) && !std::isnan(v)) { - // Doubles can represent all whole numbers up to 2^53, which has - // 16 digits. A precision of 16 should allow thus fit. - (*_os) << vespalib::asciistream::Precision(16) - << vespalib::automatic << v; - } else { - (*_os) << "null"; - } - updateCommaState(); - return *this; -} - -JSONWriter & -JSONWriter::appendFloat(float v) -{ - considerComma(); - if (!std::isinf(v) && !std::isnan(v)) { - // Floats can represent all whole numbers up to 2^24, which has - // 8 digits. A precision of 8 should allow thus fit. - (*_os) << vespalib::asciistream::Precision(8) - << vespalib::automatic << v; - } else { - (*_os) << "null"; - } - updateCommaState(); - return *this; -} - -JSONWriter & -JSONWriter::appendInt64(int64_t v) -{ - considerComma(); - (*_os) << v; - updateCommaState(); - return *this; -} - -JSONWriter & -JSONWriter::appendUInt64(uint64_t v) -{ - considerComma(); - (*_os) << v; - updateCommaState(); - return *this; -} - -JSONWriter & -JSONWriter::appendString(stringref str) -{ - considerComma(); - quote(str.data(), str.size()); - updateCommaState(); - return *this; -} - -JSONWriter & -JSONWriter::appendJSON(stringref json) -{ - considerComma(); - (*_os) << json; - updateCommaState(); - return *this; -} - -JSONStringer::JSONStringer() : - JSONWriter(), - _oss(std::make_unique<asciistream>()) -{ - setOutputStream(*_oss); -} - -JSONStringer & -JSONStringer::clear() -{ - JSONWriter::clear(); - // clear the string stream as well - _oss->clear(); - return *this; -} - -JSONStringer::~JSONStringer() { } - -stringref -JSONStringer::toString() const { - return _oss->str(); -} - -} diff --git a/staging_vespalib/src/vespa/vespalib/util/jsonwriter.h b/staging_vespalib/src/vespa/vespalib/util/jsonwriter.h deleted file mode 100644 index 57380b46e5c..00000000000 --- a/staging_vespalib/src/vespa/vespalib/util/jsonwriter.h +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/vespalib/stllike/string.h> -#include <vector> -#include <memory> - -namespace vespalib { - -class asciistream; - -/** - * If you want a simpler interface to write JSON with, look at the JsonStream - * class in the same directory as this class, this uses this class to make a - * more user friendly writer. - */ -class JSONWriter { -private: - enum State { - INIT = 0, - OBJECT, - ARRAY - }; - asciistream * _os; - std::vector<State> _stack; - bool _comma; - bool _pretty; - uint32_t _indent; - - void push(State next); - void pop(State expected); - void considerComma(); - void updateCommaState(); - void quote(const char * str, size_t len); - void indent(); - -public: - JSONWriter(); - JSONWriter(asciistream & output); - JSONWriter(const JSONWriter &) = delete; - JSONWriter & operator = (const JSONWriter &) = delete; - JSONWriter(JSONWriter &&) = default; - JSONWriter & operator = (JSONWriter &&) = default; - ~JSONWriter(); - - JSONWriter & setOutputStream(asciistream & output); - JSONWriter & clear(); - JSONWriter & beginObject(); - JSONWriter & endObject(); - JSONWriter & beginArray(); - JSONWriter & endArray(); - JSONWriter & appendNull(); - JSONWriter & appendKey(stringref str); - JSONWriter & appendBool(bool v); - JSONWriter & appendDouble(double v); - JSONWriter & appendFloat(float v); - JSONWriter & appendInt64(int64_t v); - JSONWriter & appendUInt64(uint64_t v); - JSONWriter & appendString(stringref str); - JSONWriter & appendJSON(stringref json); - - void setPretty() { _pretty = true; }; -}; - -class JSONStringer : public JSONWriter { -private: - std::unique_ptr<asciistream> _oss; - -public: - JSONStringer(); - JSONStringer(JSONStringer &&) = default; - JSONStringer & operator = (JSONStringer &&) = default; - ~JSONStringer(); - JSONStringer & clear(); - stringref toString() const; -}; - -template<typename T> -struct JSONPrinter -{ - static void printJSON(JSONWriter& w, T v) { - w.appendInt64(v); - } -}; - -template<> -struct JSONPrinter<uint64_t> -{ - static void printJSON(JSONWriter& w, uint64_t v) { - w.appendUInt64(v); - } -}; - -template<> -struct JSONPrinter<float> -{ - static void printJSON(JSONWriter& w, float v) { - w.appendDouble(v); - } -}; - -template<> -struct JSONPrinter<double> -{ - static void printJSON(JSONWriter& w, double v) { - w.appendDouble(v); - } -}; - -} diff --git a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp deleted file mode 100644 index e12d2065d9f..00000000000 --- a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp +++ /dev/null @@ -1,205 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "sequencedtaskexecutor.h" -#include "adaptive_sequenced_executor.h" -#include "singleexecutor.h" -#include <vespa/vespalib/util/atomic.h> -#include <vespa/vespalib/util/threadstackexecutor.h> -#include <vespa/vespalib/util/blockingthreadstackexecutor.h> -#include <vespa/vespalib/util/size_literals.h> -#include <vespa/vespalib/stllike/hashtable.h> -#include <cassert> - -namespace vespalib { - -namespace { - -constexpr uint32_t stackSize = 128_Ki; -constexpr uint8_t MAGIC = 255; -constexpr uint32_t NUM_PERFECT_PER_EXECUTOR = 8; -constexpr uint16_t INVALID_KEY = 0x8000; - -bool -isLazy(const std::vector<std::unique_ptr<vespalib::SyncableThreadExecutor>> & executors) { - for (const auto &executor : executors) { - if (dynamic_cast<const vespalib::SingleExecutor *>(executor.get()) == nullptr) { - return false; - } - } - return true; -} - -ssize_t -find(uint16_t key, const uint16_t values[], size_t numValues) { - for (size_t i(0); i < numValues; i++) { - auto value = vespalib::atomic::load_ref_relaxed(values[i]); - if (key == value) { - return i; - } - if (INVALID_KEY == value) { - return -1; - } - } - return -1; -} - - -} - -std::unique_ptr<ISequencedTaskExecutor> -SequencedTaskExecutor::create(Runnable::init_fun_t func, uint32_t threads) { - return create(func, threads, 1000); -} - -std::unique_ptr<ISequencedTaskExecutor> -SequencedTaskExecutor::create(Runnable::init_fun_t func, uint32_t threads, uint32_t taskLimit) { - return create(func, threads, taskLimit, true, OptimizeFor::LATENCY); -} - -std::unique_ptr<ISequencedTaskExecutor> -SequencedTaskExecutor::create(Runnable::init_fun_t func, uint32_t threads, uint32_t taskLimit, bool is_task_limit_hard, OptimizeFor optimize) { - return create(func, threads, taskLimit, is_task_limit_hard, optimize, 0); -} - -std::unique_ptr<ISequencedTaskExecutor> -SequencedTaskExecutor::create(Runnable::init_fun_t func, uint32_t threads, uint32_t taskLimit, - bool is_task_limit_hard, OptimizeFor optimize, uint32_t kindOfWatermark) -{ - if (optimize == OptimizeFor::ADAPTIVE) { - size_t num_strands = std::min(taskLimit, threads*32); - return std::make_unique<AdaptiveSequencedExecutor>(num_strands, threads, kindOfWatermark, taskLimit, is_task_limit_hard); - } else { - auto executors = std::vector<std::unique_ptr<SyncableThreadExecutor>>(); - executors.reserve(threads); - for (uint32_t id = 0; id < threads; ++id) { - if (optimize == OptimizeFor::THROUGHPUT) { - uint32_t watermark = (kindOfWatermark == 0) ? taskLimit / 10 : kindOfWatermark; - executors.push_back(std::make_unique<SingleExecutor>(func, taskLimit, is_task_limit_hard, watermark, 100ms)); - } else { - if (is_task_limit_hard) { - executors.push_back(std::make_unique<BlockingThreadStackExecutor>(1, stackSize, taskLimit, func)); - } else { - executors.push_back(std::make_unique<ThreadStackExecutor>(1, stackSize, func)); - } - } - } - return std::unique_ptr<ISequencedTaskExecutor>(new SequencedTaskExecutor(std::move(executors))); - } -} - -SequencedTaskExecutor::~SequencedTaskExecutor() -{ - sync_all(); -} - -SequencedTaskExecutor::SequencedTaskExecutor(std::vector<std::unique_ptr<vespalib::SyncableThreadExecutor>> executors) - : ISequencedTaskExecutor(executors.size()), - _executors(std::move(executors)), - _lazyExecutors(isLazy(_executors)), - _component2IdPerfect(std::make_unique<PerfectKeyT[]>(getNumExecutors()*NUM_PERFECT_PER_EXECUTOR)), - _component2IdImperfect(vespalib::hashtable_base::getModuloStl(getNumExecutors()*NUM_PERFECT_PER_EXECUTOR), MAGIC), - _mutex(), - _nextId(0) -{ - assert(getNumExecutors() < 256); - - for (size_t i(0); i < getNumExecutors() * NUM_PERFECT_PER_EXECUTOR; i++) { - _component2IdPerfect[i] = INVALID_KEY; - } -} - -void -SequencedTaskExecutor::setTaskLimit(uint32_t taskLimit) -{ - for (const auto &executor : _executors) { - executor->setTaskLimit(taskLimit); - } -} - -void -SequencedTaskExecutor::executeTask(ExecutorId id, vespalib::Executor::Task::UP task) -{ - assert(id.getId() < _executors.size()); - auto rejectedTask = _executors[id.getId()]->execute(std::move(task)); - assert(!rejectedTask); -} - -void -SequencedTaskExecutor::sync_all() { - wakeup(); - for (auto &executor : _executors) { - executor->sync(); - } -} - -void -SequencedTaskExecutor::wakeup() { - if (_lazyExecutors) { - for (auto &executor : _executors) { - //Enforce parallel wakeup of napping executors. - executor->wakeup(); - } - } -} - -ExecutorStats -SequencedTaskExecutor::getStats() -{ - ExecutorStats accumulatedStats; - for (auto &executor : _executors) { - accumulatedStats.aggregate(executor->getStats()); - } - return accumulatedStats; -} - -ISequencedTaskExecutor::ExecutorId -SequencedTaskExecutor::getExecutorId(uint64_t componentId) const { - auto id = getExecutorIdPerfect(componentId); - return id ? id.value() : getExecutorIdImPerfect(componentId); -} - -std::optional<ISequencedTaskExecutor::ExecutorId> -SequencedTaskExecutor::getExecutorIdPerfect(uint64_t componentId) const { - PerfectKeyT key = componentId & 0x7fff; - ssize_t pos = find(key, _component2IdPerfect.get(), getNumExecutors() * NUM_PERFECT_PER_EXECUTOR); - if (pos < 0) { - std::unique_lock guard(_mutex); - pos = find(key, _component2IdPerfect.get(), getNumExecutors() * NUM_PERFECT_PER_EXECUTOR); - if (pos < 0) { - pos = find(INVALID_KEY, _component2IdPerfect.get(), getNumExecutors() * NUM_PERFECT_PER_EXECUTOR); - if (pos >= 0) { - vespalib::atomic::store_ref_relaxed(_component2IdPerfect[pos], key); - } else { - // There was a race for the last spots - return std::optional<ISequencedTaskExecutor::ExecutorId>(); - } - } - } - return std::optional<ISequencedTaskExecutor::ExecutorId>(ExecutorId(pos % getNumExecutors())); -} - -ISequencedTaskExecutor::ExecutorId -SequencedTaskExecutor::getExecutorIdImPerfect(uint64_t componentId) const { - uint32_t shrunkId = componentId % _component2IdImperfect.size(); - uint8_t executorId = vespalib::atomic::load_ref_relaxed(_component2IdImperfect[shrunkId]); - if (executorId == MAGIC) { - std::lock_guard guard(_mutex); - if (vespalib::atomic::load_ref_relaxed(_component2IdImperfect[shrunkId]) == MAGIC) { - vespalib::atomic::store_ref_relaxed(_component2IdImperfect[shrunkId], _nextId % getNumExecutors()); - _nextId++; - } - executorId = _component2IdImperfect[shrunkId]; - } - return ExecutorId(executorId); -} - -const vespalib::ThreadExecutor* -SequencedTaskExecutor::first_executor() const -{ - if (_executors.empty()) { - return nullptr; - } - return _executors.front().get(); -} - -} // namespace search diff --git a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h deleted file mode 100644 index a4b1b82aacf..00000000000 --- a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "isequencedtaskexecutor.h" -#include <vespa/vespalib/util/time.h> -#include <vespa/vespalib/util/runnable.h> -#include <optional> - -namespace vespalib { - -class ThreadExecutor; -class SyncableThreadExecutor; - -/** - * Class to run multiple tasks in parallel, but tasks with same - * id has to be run in sequence. - */ -class SequencedTaskExecutor final : public ISequencedTaskExecutor -{ -public: - using ISequencedTaskExecutor::getExecutorId; - using OptimizeFor = vespalib::Executor::OptimizeFor; - - ~SequencedTaskExecutor() override; - - void setTaskLimit(uint32_t taskLimit) override; - void executeTask(ExecutorId id, vespalib::Executor::Task::UP task) override; - ExecutorId getExecutorId(uint64_t componentId) const override; - void sync_all() override; - ExecutorStats getStats() override; - void wakeup() override; - - static std::unique_ptr<ISequencedTaskExecutor> - create(Runnable::init_fun_t func, uint32_t threads); - static std::unique_ptr<ISequencedTaskExecutor> - create(Runnable::init_fun_t func, uint32_t threads, uint32_t taskLimit); - static std::unique_ptr<ISequencedTaskExecutor> - create(Runnable::init_fun_t func, uint32_t threads, uint32_t taskLimit, bool is_task_limit_hard, OptimizeFor optimize); - static std::unique_ptr<ISequencedTaskExecutor> - create(Runnable::init_fun_t func, uint32_t threads, uint32_t taskLimit, - bool is_task_limit_hard, OptimizeFor optimize, uint32_t kindOfWatermark); - /** - * For testing only - */ - uint32_t getComponentHashSize() const { return _component2IdImperfect.size(); } - uint32_t getComponentEffectiveHashSize() const { return _nextId; } - const vespalib::ThreadExecutor* first_executor() const; - -private: - explicit SequencedTaskExecutor(std::vector<std::unique_ptr<vespalib::SyncableThreadExecutor>> executor); - std::optional<ExecutorId> getExecutorIdPerfect(uint64_t componentId) const; - ExecutorId getExecutorIdImPerfect(uint64_t componentId) const; - - std::vector<std::unique_ptr<vespalib::SyncableThreadExecutor>> _executors; - using PerfectKeyT = uint16_t; - const bool _lazyExecutors; - mutable std::unique_ptr<PerfectKeyT[]> _component2IdPerfect; - mutable std::vector<uint8_t> _component2IdImperfect; - mutable std::mutex _mutex; - mutable uint32_t _nextId; - -}; - -} // namespace search diff --git a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.cpp b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.cpp deleted file mode 100644 index d81b8ec1db6..00000000000 --- a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.cpp +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "sequencedtaskexecutorobserver.h" - -namespace vespalib { - -SequencedTaskExecutorObserver::SequencedTaskExecutorObserver(ISequencedTaskExecutor &executor) - : ISequencedTaskExecutor(executor.getNumExecutors()), - _executor(executor), - _executeCnt(0u), - _syncCnt(0u), - _executeHistory(), - _mutex() -{ -} - -SequencedTaskExecutorObserver::~SequencedTaskExecutorObserver() = default; - -void -SequencedTaskExecutorObserver::executeTask(ExecutorId id, Executor::Task::UP task) -{ - ++_executeCnt; - { - std::lock_guard<std::mutex> guard(_mutex); - _executeHistory.emplace_back(id.getId()); - } - _executor.executeTask(id, std::move(task)); -} - -void -SequencedTaskExecutorObserver::executeTasks(TaskList tasks) -{ - _executeCnt += tasks.size(); - { - std::lock_guard<std::mutex> guard(_mutex); - for (const auto & task : tasks) { - _executeHistory.emplace_back(task.first.getId()); - } - } - _executor.executeTasks(std::move(tasks)); -} - -void -SequencedTaskExecutorObserver::sync_all() -{ - ++_syncCnt; - _executor.sync_all(); -} - -std::vector<uint32_t> -SequencedTaskExecutorObserver::getExecuteHistory() -{ - std::lock_guard<std::mutex> guard(_mutex); - return _executeHistory; -} - -void SequencedTaskExecutorObserver::setTaskLimit(uint32_t taskLimit) { - _executor.setTaskLimit(taskLimit); -} - -ExecutorStats SequencedTaskExecutorObserver::getStats() { - return _executor.getStats(); -} - -ISequencedTaskExecutor::ExecutorId -SequencedTaskExecutorObserver::getExecutorId(uint64_t componentId) const { - return _executor.getExecutorId(componentId); -} - -} // namespace search diff --git a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.h b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.h deleted file mode 100644 index 1d54283c393..00000000000 --- a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.h +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "isequencedtaskexecutor.h" -#include <atomic> - -namespace vespalib { - -/** - * Observer class to observe class to run multiple tasks in parallel, - * but tasks with same id has to be run in sequence. - */ -class SequencedTaskExecutorObserver : public ISequencedTaskExecutor -{ - ISequencedTaskExecutor &_executor; - std::atomic<uint32_t> _executeCnt; - std::atomic<uint32_t> _syncCnt; - std::vector<uint32_t> _executeHistory; - std::mutex _mutex; -public: - using ISequencedTaskExecutor::getExecutorId; - - SequencedTaskExecutorObserver(ISequencedTaskExecutor &executor); - ~SequencedTaskExecutorObserver() override; - - ExecutorId getExecutorId(uint64_t componentId) const override; - void executeTask(ExecutorId id, Executor::Task::UP task) override; - void executeTasks(TaskList tasks) override; - void sync_all() override; - void setTaskLimit(uint32_t taskLimit) override; - ExecutorStats getStats() override; - - uint32_t getExecuteCnt() const { return _executeCnt; } - uint32_t getSyncCnt() const { return _syncCnt; } - std::vector<uint32_t> getExecuteHistory(); -}; - -} // namespace search diff --git a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp deleted file mode 100644 index 21ed90c3d22..00000000000 --- a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp +++ /dev/null @@ -1,241 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "singleexecutor.h" -#include <vespa/vespalib/util/alloc.h> -#include <cassert> - -namespace vespalib { - -SingleExecutor::SingleExecutor(init_fun_t func, uint32_t taskLimit) - : SingleExecutor(func, taskLimit, true, taskLimit/10, 100ms) -{ } - -SingleExecutor::SingleExecutor(init_fun_t func, uint32_t reservedQueueSize, bool isQueueSizeHard, uint32_t watermark, duration reactionTime) - : _watermarkRatio(watermark < reservedQueueSize ? double(watermark) / reservedQueueSize : 1.0), - _taskLimit(vespalib::roundUp2inN(reservedQueueSize)), - _wantedTaskLimit(_taskLimit.load()), - _rp(0), - _tasks(std::make_unique<Task::UP[]>(_taskLimit)), - _mutex(), - _consumerCondition(), - _producerCondition(), - _thread(*this, func), - _idleTracker(steady_clock::now()), - _threadIdleTracker(), - _wakeupCount(0), - _lastAccepted(0), - _queueSize(), - _wakeupConsumerAt(0), - _producerNeedWakeupAt(0), - _wp(0), - _watermark(_taskLimit.load()*_watermarkRatio), - _reactionTime(reactionTime), - _closed(false), - _overflow() -{ - assert(reservedQueueSize >= watermark); - if ( ! isQueueSizeHard) { - _overflow = std::make_unique<ArrayQueue<Task::UP>>(); - } - _thread.start(); -} - -SingleExecutor::~SingleExecutor() { - shutdown(); - sync(); - _thread.stop().join(); -} - -size_t -SingleExecutor::getNumThreads() const { - return 1; -} - -void -SingleExecutor::sleepProducer(Lock & lock, duration maxWaitTime, uint64_t wakeupAt) { - _producerNeedWakeupAt.store(wakeupAt, std::memory_order_relaxed); - _producerCondition.wait_for(lock, maxWaitTime); - _producerNeedWakeupAt.store(0, std::memory_order_relaxed); -} - -Executor::Task::UP -SingleExecutor::execute(Task::UP task) { - uint64_t wp; - { - Lock guard(_mutex); - if (_closed) { - return task; - } - task = wait_for_room_or_put_in_overflow_Q(guard, std::move(task)); - if (task) { - wp = move_to_main_q(guard, std::move(task)); - } else { - wp = _wp.load(std::memory_order_relaxed) + num_tasks_in_overflow_q(guard); - } - } - if (wp == _wakeupConsumerAt.load(std::memory_order_relaxed)) { - _consumerCondition.notify_one(); - } - return task; -} - -uint64_t -SingleExecutor::numTasks() { - if (_overflow) { - Lock guard(_mutex); - return num_tasks_in_main_q() + num_tasks_in_overflow_q(guard); - } else { - return num_tasks_in_main_q(); - } -} - -uint64_t -SingleExecutor::move_to_main_q(Lock &, Task::UP task) { - uint64_t wp = _wp.load(std::memory_order_relaxed); - _tasks[index(wp)] = std::move(task); - _wp.store(wp + 1, std::memory_order_release); - return wp; -} - -void -SingleExecutor::setTaskLimit(uint32_t taskLimit) { - _wantedTaskLimit = vespalib::roundUp2inN(taskLimit); -} - -void -SingleExecutor::drain(Lock & lock) { - uint64_t wp = _wp.load(std::memory_order_relaxed); - while (numTasks(lock) > 0) { - _consumerCondition.notify_one(); - sleepProducer(lock, 100us, wp); - } -} - -void -SingleExecutor::wakeup() { - if (numTasks() > 0) { - _consumerCondition.notify_one(); - } -} - -SingleExecutor & -SingleExecutor::sync() { - Lock lock(_mutex); - uint64_t wp = _wp.load(std::memory_order_relaxed) + num_tasks_in_overflow_q(lock); - while (wp > _rp.load(std::memory_order_acquire)) { - _consumerCondition.notify_one(); - sleepProducer(lock, 100us, wp); - } - return *this; -} - -SingleExecutor & -SingleExecutor::shutdown() { - Lock lock(_mutex); - _closed = true; - return *this; -} - -void -SingleExecutor::run() { - while (!_thread.stopped()) { - drain_tasks(); - _producerCondition.notify_all(); - _wakeupConsumerAt.store(_wp.load(std::memory_order_relaxed) + get_watermark(), std::memory_order_relaxed); - Lock lock(_mutex); - if (numTasks(lock) <= 0) { - steady_time now = steady_clock::now(); - _threadIdleTracker.set_idle(now); - _consumerCondition.wait_until(lock, now + _reactionTime); - _idleTracker.was_idle(_threadIdleTracker.set_active(steady_clock::now())); - _wakeupCount++; - } - _wakeupConsumerAt.store(0, std::memory_order_relaxed); - } -} - -void -SingleExecutor::drain_tasks() { - while (numTasks() > 0) { - run_tasks_till(_wp.load(std::memory_order_acquire)); - move_overflow_to_main_q(); - } -} - -void -SingleExecutor::move_overflow_to_main_q() -{ - if ( ! _overflow) return; - Lock guard(_mutex); - move_overflow_to_main_q(guard); -} -void -SingleExecutor::move_overflow_to_main_q(Lock & guard) { - while ( !_overflow->empty() && num_tasks_in_main_q() < _taskLimit.load(std::memory_order_relaxed)) { - move_to_main_q(guard, std::move(_overflow->front())); - _overflow->pop(); - } -} - -void -SingleExecutor::run_tasks_till(uint64_t available) { - uint64_t consumed = _rp.load(std::memory_order_relaxed); - uint64_t wakeupLimit = _producerNeedWakeupAt.load(std::memory_order_relaxed); - while (consumed < available) { - Task::UP task = std::move(_tasks[index(consumed)]); - task->run(); - _rp.store(++consumed, std::memory_order_release); - if (wakeupLimit == consumed) { - _producerCondition.notify_all(); - } - } -} - -Executor::Task::UP -SingleExecutor::wait_for_room_or_put_in_overflow_Q(Lock & guard, Task::UP task) { - uint64_t wp = _wp.load(std::memory_order_relaxed); - uint64_t taskLimit = _taskLimit.load(std::memory_order_relaxed); - if (taskLimit != _wantedTaskLimit.load(std::memory_order_relaxed)) { - drain(guard); - _tasks = std::make_unique<Task::UP[]>(_wantedTaskLimit); - _taskLimit = _wantedTaskLimit.load(); - _watermark = _taskLimit * _watermarkRatio; - } - uint64_t numTaskInQ = numTasks(guard); - _queueSize.add(numTaskInQ); - if (numTaskInQ >= _taskLimit.load(std::memory_order_relaxed)) { - if (_overflow) { - _overflow->push(std::move(task)); - } else { - while (numTasks(guard) >= _taskLimit.load(std::memory_order_relaxed)) { - sleepProducer(guard, _reactionTime, wp - get_watermark()); - } - } - } else { - if (_overflow && !_overflow->empty()) { - _overflow->push(std::move(task)); - } - } - if (_overflow && !_overflow->empty()) { - assert(!task); - move_overflow_to_main_q(guard); - } - return task; -} - -ExecutorStats -SingleExecutor::getStats() { - Lock lock(_mutex); - uint64_t accepted = _wp.load(std::memory_order_relaxed) + num_tasks_in_overflow_q(lock); - steady_time now = steady_clock::now(); - _idleTracker.was_idle(_threadIdleTracker.reset(now)); - ExecutorStats stats(_queueSize, (accepted - _lastAccepted), 0, _wakeupCount); - stats.setUtil(1, _idleTracker.reset(now, 1)); - _wakeupCount = 0; - _lastAccepted = accepted; - _queueSize = ExecutorStats::QueueSizeT() ; - return stats; -} - - -} diff --git a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h b/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h deleted file mode 100644 index dd755a76302..00000000000 --- a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/vespalib/util/threadexecutor.h> -#include <vespa/vespalib/util/thread.h> -#include <vespa/vespalib/util/time.h> -#include <vespa/vespalib/util/arrayqueue.hpp> -#include <vespa/vespalib/util/executor_idle_tracking.h> -#include <thread> -#include <atomic> - -namespace vespalib { - -/** - * Has a single thread consuming tasks from a fixed size ringbuffer. - * Made for throughput where the producer has no interaction with the consumer and - * it is hence very cheap to produce a task. High and low watermark at 25%/75% is used - * to reduce ping-pong. - */ -class SingleExecutor final : public vespalib::SyncableThreadExecutor, vespalib::Runnable { -public: - SingleExecutor(init_fun_t func, uint32_t reservedQueueSize); - SingleExecutor(init_fun_t func, uint32_t reservedQueueSize, bool isQueueSizeHard, uint32_t watermark, duration reactionTime); - ~SingleExecutor() override; - Task::UP execute(Task::UP task) override; - void setTaskLimit(uint32_t taskLimit) override; - SingleExecutor & sync() override; - void wakeup() override; - size_t getNumThreads() const override; - uint32_t getTaskLimit() const override { return _taskLimit.load(std::memory_order_relaxed); } - uint32_t get_watermark() const { return _watermark.load(std::memory_order_relaxed); } - duration get_reaction_time() const { return _reactionTime; } - ExecutorStats getStats() override; - SingleExecutor & shutdown() override; - bool isBlocking() const { return !_overflow; } -private: - using Lock = std::unique_lock<std::mutex>; - void drain(Lock & lock); - void run() override; - void drain_tasks(); - void sleepProducer(Lock & guard, duration maxWaitTime, uint64_t wakeupAt); - void run_tasks_till(uint64_t available); - Task::UP wait_for_room_or_put_in_overflow_Q(Lock & guard, Task::UP task); - uint64_t move_to_main_q(Lock & guard, Task::UP task); - void move_overflow_to_main_q(); - void move_overflow_to_main_q(Lock & guard); - uint64_t index(uint64_t counter) const { - return counter & (_taskLimit.load(std::memory_order_relaxed) - 1); - } - - uint64_t numTasks(); - uint64_t numTasks(Lock & guard) const { - return num_tasks_in_main_q() + num_tasks_in_overflow_q(guard); - } - uint64_t num_tasks_in_overflow_q(Lock &) const { - return _overflow ? _overflow->size() : 0; - } - uint64_t num_tasks_in_main_q() const { - return _wp.load(std::memory_order_relaxed) - _rp.load(std::memory_order_acquire); - } - const double _watermarkRatio; - std::atomic<uint32_t> _taskLimit; - std::atomic<uint32_t> _wantedTaskLimit; - std::atomic<uint64_t> _rp; - std::unique_ptr<Task::UP[]> _tasks; - std::mutex _mutex; - std::condition_variable _consumerCondition; - std::condition_variable _producerCondition; - vespalib::Thread _thread; - ExecutorIdleTracker _idleTracker; - ThreadIdleTracker _threadIdleTracker; - uint64_t _wakeupCount; - uint64_t _lastAccepted; - ExecutorStats::QueueSizeT _queueSize; - std::atomic<uint64_t> _wakeupConsumerAt; - std::atomic<uint64_t> _producerNeedWakeupAt; - std::atomic<uint64_t> _wp; - std::atomic<uint32_t> _watermark; - const duration _reactionTime; - bool _closed; - std::unique_ptr<ArrayQueue<Task::UP>> _overflow; -}; - -} |