diff options
Diffstat (limited to 'vespalib')
144 files changed, 11030 insertions, 2 deletions
diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 79cc32d2a60..fd1c52a868a 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -1,4 +1,9 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +if(NOT CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin") +set(VESPALIB_DIRECTIO_TESTDIR src/tests/directio) +set(VESPALIB_PROCESS_MEMORY_STATS_TESTDIR src/tests/util/process_memory_stats) +endif() + vespa_define_module( DEPENDS fastos @@ -25,16 +30,20 @@ vespa_define_module( src/tests/arrayref src/tests/assert src/tests/barrier + src/tests/benchmark src/tests/benchmark_timer + src/tests/bits src/tests/box src/tests/btree src/tests/btree/btree_store src/tests/btree/btree-scan-speed src/tests/btree/btree-stress + src/tests/clock src/tests/component src/tests/compress src/tests/compression src/tests/cpu_usage + src/tests/crc src/tests/crypto src/tests/data/databuffer src/tests/data/input_reader @@ -53,6 +62,7 @@ vespa_define_module( src/tests/datastore/unique_store src/tests/datastore/unique_store_dictionary src/tests/datastore/unique_store_string_allocator + ${VESPALIB_DIRECTIO_TESTDIR} src/tests/detect_type_benchmark src/tests/dotproduct src/tests/drop-file-from-cache @@ -63,21 +73,25 @@ vespa_define_module( src/tests/executor src/tests/executor_idle_tracking src/tests/explore_modern_cpp - src/tests/fileheader src/tests/false src/tests/fiddle + src/tests/fileheader + src/tests/floatingpointtype src/tests/fuzzy src/tests/gencnt + src/tests/growablebytebuffer src/tests/guard src/tests/host_name src/tests/hwaccelrated src/tests/io/fileutil src/tests/io/mapped_file_input src/tests/issue + src/tests/json src/tests/latch src/tests/left_right_heap src/tests/make_fixture_macros src/tests/memory + src/tests/memorydatastore src/tests/metrics src/tests/net/async_resolver src/tests/net/crypto_socket @@ -93,9 +107,13 @@ vespa_define_module( src/tests/net/tls/protocol_snooping src/tests/net/tls/transport_options src/tests/nice + src/tests/objects/identifiable src/tests/objects/nbostream + src/tests/objects/objectdump + src/tests/objects/objectselection src/tests/optimized src/tests/overload + src/tests/polymorphicarray src/tests/portal src/tests/portal/handle_manager src/tests/portal/http_request @@ -103,13 +121,17 @@ vespa_define_module( src/tests/printable src/tests/priority_queue src/tests/process + ${VESPALIB_PROCESS_MEMORY_STATS_TESTDIR} + src/tests/programoptions src/tests/random src/tests/referencecounter src/tests/regex src/tests/rendezvous src/tests/require src/tests/runnable_pair + src/tests/rusage src/tests/sequencedtaskexecutor + src/tests/shutdownguard src/tests/singleexecutor src/tests/sha1 src/tests/shared_operation_throttler @@ -169,6 +191,7 @@ vespa_define_module( src/tests/visit_ranges src/tests/invokeservice src/tests/wakeup + src/tests/xmlserializable src/tests/zcurve LIBS diff --git a/vespalib/src/tests/array/.gitignore b/vespalib/src/tests/array/.gitignore index 952bc0ecdf2..dd07c7d9777 100644 --- a/vespalib/src/tests/array/.gitignore +++ b/vespalib/src/tests/array/.gitignore @@ -1,3 +1,4 @@ /sort_benchmark /allocinarray_benchmark vespalib_array_test_app +vespalib_sort_benchmark_app diff --git a/vespalib/src/tests/array/CMakeLists.txt b/vespalib/src/tests/array/CMakeLists.txt index 18cb5c2b95e..fae7b32cd7e 100644 --- a/vespalib/src/tests/array/CMakeLists.txt +++ b/vespalib/src/tests/array/CMakeLists.txt @@ -6,3 +6,10 @@ vespa_add_executable(vespalib_array_test_app TEST vespalib ) vespa_add_test(NAME vespalib_array_test_app COMMAND vespalib_array_test_app) +vespa_add_executable(vespalib_sort_benchmark_app + SOURCES + sort_benchmark.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_sort_benchmark_app COMMAND vespalib_sort_benchmark_app BENCHMARK) diff --git a/vespalib/src/tests/array/sort_benchmark.cpp b/vespalib/src/tests/array/sort_benchmark.cpp new file mode 100644 index 00000000000..5d8a1efaa7a --- /dev/null +++ b/vespalib/src/tests/array/sort_benchmark.cpp @@ -0,0 +1,184 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/rusage.h> +#include <vespa/vespalib/util/array.hpp> +#include <csignal> +#include <algorithm> + +#include <vespa/log/log.h> +LOG_SETUP("sort_benchmark"); + +using namespace vespalib; + +class Test : public TestApp +{ +public: +private: + template<typename T> + vespalib::Array<T> create(size_t count); + template<typename T> + void sortDirect(size_t count); + template<typename T> + void sortInDirect(size_t count); + int Main() override; +}; + +template<size_t N> +class TT +{ +public: + TT(uint64_t v) : _v(v) { } + bool operator < (const TT & rhs) const { return _v < rhs._v; } +private: + uint64_t _v; + uint8_t _payLoad[N - sizeof(uint64_t)]; +}; + +template <typename T> +class I +{ +public: + I(const T * p) : _p(p) { } + bool operator < (const I & rhs) const { return *_p < *rhs._p; } +private: + const T * _p; +}; + +template<typename T> +vespalib::Array<T> +Test::create(size_t count) +{ + vespalib::Array<T> v; + v.reserve(count); + srand(0); + for (size_t i(0); i < count; i++) { + v.push_back(rand()); + } + return v; +} + +template<typename T> +void Test::sortDirect(size_t count) +{ + vespalib::Array<T> v(create<T>(count)); + LOG(info, "Running sortDirect with %ld count and payload of %ld", v.size(), sizeof(T)); + for (size_t j=0; j < 10; j++) { + vespalib::Array<T> t(v); + std::sort(t.begin(), t.end()); + } +} + +template<typename T> +void Test::sortInDirect(size_t count) +{ + vespalib::Array<T> k(create<T>(count)); + LOG(info, "Running sortInDirect with %ld count and payload of %ld", k.size(), sizeof(T)); + vespalib::Array< I<T> > v; + v.reserve(k.size()); + for (size_t i(0), m(k.size()); i < m; i++) { + v.push_back(&k[i]); + } + for (size_t j=0; j < 10; j++) { + vespalib::Array< I<T> > t(v); + std::sort(t.begin(), t.end()); + } +} + +int +Test::Main() +{ + std::string type("sortdirect"); + size_t count = 1000000; + size_t payLoad = 0; + if (_argc > 1) { + type = _argv[1]; + } + if (_argc > 2) { + count = strtol(_argv[2], NULL, 0); + } + if (_argc > 3) { + payLoad = strtol(_argv[3], NULL, 0); + } + TEST_INIT("sort_benchmark"); + steady_time start(steady_clock::now()); + if (payLoad < 8) { + typedef TT<8> T; + if (type == "sortdirect") { + sortDirect<T>(count); + } else if (type == "sortindirect") { + sortInDirect<T>(count); + } else { + LOG(warning, "type '%s' is unknown", type.c_str()); + } + } else if (payLoad < 16) { + typedef TT<16> T; + if (type == "sortdirect") { + sortDirect<T>(count); + } else if (type == "sortindirect") { + sortInDirect<T>(count); + } else { + LOG(warning, "type '%s' is unknown", type.c_str()); + } + } else if (payLoad < 32) { + typedef TT<32> T; + if (type == "sortdirect") { + sortDirect<T>(count); + } else if (type == "sortindirect") { + sortInDirect<T>(count); + } else { + LOG(warning, "type '%s' is unknown", type.c_str()); + } + } else if (payLoad < 64) { + typedef TT<64> T; + if (type == "sortdirect") { + sortDirect<T>(count); + } else if (type == "sortindirect") { + sortInDirect<T>(count); + } else { + LOG(warning, "type '%s' is unknown", type.c_str()); + } + } else if (payLoad < 128) { + typedef TT<128> T; + if (type == "sortdirect") { + sortDirect<T>(count); + } else if (type == "sortindirect") { + sortInDirect<T>(count); + } else { + LOG(warning, "type '%s' is unknown", type.c_str()); + } + } else if (payLoad < 256) { + typedef TT<256> T; + if (type == "sortdirect") { + sortDirect<T>(count); + } else if (type == "sortindirect") { + sortInDirect<T>(count); + } else { + LOG(warning, "type '%s' is unknown", type.c_str()); + } + } else if (payLoad < 512) { + typedef TT<512> T; + if (type == "sortdirect") { + sortDirect<T>(count); + } else if (type == "sortindirect") { + sortInDirect<T>(count); + } else { + LOG(warning, "type '%s' is unknown", type.c_str()); + } + } else { + typedef TT<1024> T; + LOG(info, "Payload %ld is too big to make any sense. Using %ld.", payLoad, sizeof(T)); + if (type == "sortdirect") { + sortDirect<T>(count); + } else if (type == "sortindirect") { + sortInDirect<T>(count); + } else { + LOG(warning, "type '%s' is unknown", type.c_str()); + } + } + LOG(info, "rusage = {\n%s\n}", vespalib::RUsage::createSelf(start).toString().c_str()); + ASSERT_EQUAL(0, kill(0, SIGPROF)); + TEST_DONE(); +} + +TEST_APPHOOK(Test); + diff --git a/vespalib/src/tests/benchmark/.gitignore b/vespalib/src/tests/benchmark/.gitignore new file mode 100644 index 00000000000..3280cb17888 --- /dev/null +++ b/vespalib/src/tests/benchmark/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +benchmark_test +vespalib_benchmark_test_app diff --git a/vespalib/src/tests/benchmark/CMakeLists.txt b/vespalib/src/tests/benchmark/CMakeLists.txt new file mode 100644 index 00000000000..7003a5c4183 --- /dev/null +++ b/vespalib/src/tests/benchmark/CMakeLists.txt @@ -0,0 +1,12 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_benchmark_test_app + SOURCES + benchmark.cpp + testbase.cpp + DEPENDS + vespalib + EXTERNAL_DEPENDS + ${VESPA_GLIBC_RT_LIB} +) +vespa_add_test(NAME vespalib_benchmark_test NO_VALGRIND COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/benchmark_test.sh BENCHMARK + DEPENDS vespalib_benchmark_test_app) diff --git a/vespalib/src/tests/benchmark/benchmark.cpp b/vespalib/src/tests/benchmark/benchmark.cpp new file mode 100644 index 00000000000..f1e69758c8c --- /dev/null +++ b/vespalib/src/tests/benchmark/benchmark.cpp @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include "testbase.h" + +#include <vespa/log/log.h> +LOG_SETUP("benchmark_test"); + +using namespace vespalib; + +TEST_SETUP(Test) + +int +Test::Main() +{ + TEST_INIT("benchmark_test"); + + if (_argc > 1) { + size_t concurrency(1); + size_t numRuns(1000); + if (_argc > 2) { + numRuns = strtoul(_argv[2], NULL, 0); + if (_argc > 3) { + concurrency = strtoul(_argv[3], NULL, 0); + } + } + Benchmark::run(_argv[1], numRuns, concurrency); + } + + TEST_DONE(); +} diff --git a/vespalib/src/tests/benchmark/benchmark_test.sh b/vespalib/src/tests/benchmark/benchmark_test.sh new file mode 100755 index 00000000000..28dc6b518be --- /dev/null +++ b/vespalib/src/tests/benchmark/benchmark_test.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +set -e +TIME=time + +$TIME ./vespalib_benchmark_test_app vespalib::ParamByReferenceVectorInt 200000 1 +$TIME ./vespalib_benchmark_test_app vespalib::ParamByValueVectorInt 4000 1 +$TIME ./vespalib_benchmark_test_app vespalib::ParamByReferenceVectorString 30000 1 +$TIME ./vespalib_benchmark_test_app vespalib::ParamByValueVectorString 40 1 +$TIME ./vespalib_benchmark_test_app vespalib::ReturnByReferenceVectorString 10 1 +$TIME ./vespalib_benchmark_test_app vespalib::ReturnByValueVectorString 10 1 +$TIME ./vespalib_benchmark_test_app vespalib::ReturnByValueMultiVectorString 10 1 +$TIME ./vespalib_benchmark_test_app vespalib::ClockSystem 1000 1 +$TIME ./vespalib_benchmark_test_app vespalib::ClockGToD 1000 1 +$TIME ./vespalib_benchmark_test_app vespalib::ClockGToD 20000 1 +$TIME ./vespalib_benchmark_test_app vespalib::ClockREALTIME 1000 1 +$TIME ./vespalib_benchmark_test_app vespalib::ClockMONOTONIC 1000 1 +$TIME ./vespalib_benchmark_test_app vespalib::ClockMONOTONIC_RAW 1000 1 +$TIME ./vespalib_benchmark_test_app vespalib::ClockPROCESS_CPUTIME_ID 2500 1 +$TIME ./vespalib_benchmark_test_app vespalib::ClockTHREAD_CPUTIME_ID 2500 1 +$TIME ./vespalib_benchmark_test_app vespalib::CreateVespalibString 20000 1 diff --git a/vespalib/src/tests/benchmark/testbase.cpp b/vespalib/src/tests/benchmark/testbase.cpp new file mode 100644 index 00000000000..6b5f8d7d627 --- /dev/null +++ b/vespalib/src/tests/benchmark/testbase.cpp @@ -0,0 +1,279 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "testbase.h" +#include <vespa/vespalib/util/time.h> +#include <cassert> + +#include <vespa/log/log.h> +LOG_SETUP(".testbase"); + +using namespace std::chrono; + +namespace vespalib { + +IMPLEMENT_IDENTIFIABLE_ABSTRACT_NS(vespalib, Benchmark, Identifiable); + +IMPLEMENT_BENCHMARK(ParamByReferenceVectorInt, Benchmark); +IMPLEMENT_BENCHMARK(ParamByValueVectorInt, Benchmark); +IMPLEMENT_BENCHMARK(ParamByReferenceVectorString, Benchmark); +IMPLEMENT_BENCHMARK(ParamByValueVectorString, Benchmark); +IMPLEMENT_BENCHMARK(ReturnByReferenceVectorString, Benchmark); +IMPLEMENT_BENCHMARK(ReturnByValueVectorString, Benchmark); +IMPLEMENT_BENCHMARK(ReturnByValueMultiVectorString, Benchmark); +IMPLEMENT_BENCHMARK(ClockSystem, Benchmark); +IMPLEMENT_BENCHMARK(ClockREALTIME, Benchmark); +IMPLEMENT_BENCHMARK(ClockMONOTONIC, Benchmark); +IMPLEMENT_BENCHMARK(ClockMONOTONIC_RAW, Benchmark); +IMPLEMENT_BENCHMARK(ClockPROCESS_CPUTIME_ID, Benchmark); +IMPLEMENT_BENCHMARK(ClockTHREAD_CPUTIME_ID, Benchmark); +IMPLEMENT_BENCHMARK(CreateVespalibString, Benchmark); + +void Benchmark::run(const char *name, size_t numRuns, size_t concurrency) +{ + const Identifiable::RuntimeClass * cInfo = Identifiable::classFromName(name); + if (cInfo) { + std::unique_ptr<Benchmark> test(static_cast<Benchmark *>(cInfo->create())); + test->run(numRuns, concurrency); + } else { + LOG(warning, "Could not find any test with the name %s", name); + } +} +void Benchmark::run(size_t numRuns, size_t concurrency) +{ + LOG(info, "Starting benchmark %s with %ld threads and %ld rep", getClass().name(), concurrency, numRuns); + for (size_t i(0); i < numRuns; i++) { + onRun(); + } + LOG(info, "Stopping benchmark %s", getClass().name()); +} + +size_t ParamByReferenceVectorInt::callByReference(const Vector & values) const +{ + return values.size(); +} + +size_t ParamByReferenceVectorInt::onRun() +{ + Vector values(1000); + size_t sum(0); + for (size_t i=0; i < 1000; i++) { + sum += callByReference(values); + } + return sum; +} + +size_t ParamByValueVectorInt::callByValue(Vector values) const +{ + return values.size(); +} + +size_t ParamByValueVectorInt::onRun() +{ + Vector values(1000); + size_t sum(0); + for (size_t i=0; i < 1000; i++) { + sum += callByValue(values); + } + return sum; +} + +size_t ParamByReferenceVectorString::callByReference(const Vector & values) const +{ + return values.size(); +} + +size_t ParamByReferenceVectorString::onRun() +{ + Vector values(1000, "This is a simple string copy test"); + size_t sum(0); + for (size_t i=0; i < 1000; i++) { + sum += callByReference(values); + } + return sum; +} + +size_t ParamByValueVectorString::callByValue(Vector values) const +{ + return values.size(); +} + +size_t ParamByValueVectorString::onRun() +{ + Vector values(1000, "This is a simple string copy test"); + size_t sum(0); + for (size_t i=0; i < 1000; i++) { + sum += callByValue(values); + } + return sum; +} + +const ReturnByReferenceVectorString::Vector & ReturnByReferenceVectorString::returnByReference(Vector & param) const +{ + Vector values(1000, "return by value"); + param.swap(values); + return param; +} + +size_t ReturnByReferenceVectorString::onRun() +{ + size_t sum(0); + for (size_t i=0; i < 1000; i++) { + Vector values; + sum += returnByReference(values).size(); + } + return sum; +} + +ReturnByValueVectorString::Vector ReturnByValueVectorString::returnByValue() const +{ + Vector values; + if (rand() % 7) { + Vector tmp(1000, "return by value"); + values.swap(tmp); + } else { + Vector tmp(1000, "Return by value"); + values.swap(tmp); + } + return values; +} + +size_t ReturnByValueVectorString::onRun() +{ + size_t sum(0); + for (size_t i=0; i < 1000; i++) { + sum += returnByValue().size(); + } + return sum; +} + +ReturnByValueMultiVectorString::Vector ReturnByValueMultiVectorString::returnByValue() const +{ + if (rand() % 7) { + Vector values(1000, "return by value"); + return values; + } else { + Vector values(1000, "Return by value"); + return values; + } +} + +size_t ReturnByValueMultiVectorString::onRun() +{ + size_t sum(0); + for (size_t i=0; i < 1000; i++) { + sum += returnByValue().size(); + } + return sum; +} + +size_t ClockSystem::onRun() +{ + vespalib::system_time start(vespalib::system_clock::now()); + vespalib::system_time end(start); + for (size_t i=0; i < 1000; i++) { + end = vespalib::system_clock::now(); + } + return std::chrono::duration_cast<std::chrono::nanoseconds>(start - end).count(); +} + +size_t ClockREALTIME::onRun() +{ + struct timespec ts; + int foo = clock_gettime(CLOCK_REALTIME, &ts); + assert(foo == 0); + (void) foo; + nanoseconds start(ts.tv_sec*1000L*1000L*1000L + ts.tv_nsec); + nanoseconds end(start); + for (size_t i=0; i < 1000; i++) { + clock_gettime(CLOCK_REALTIME, &ts); + end = nanoseconds(ts.tv_sec*1000L*1000L*1000L + ts.tv_nsec); + } + return count_ns(start - end); +} + +size_t ClockMONOTONIC::onRun() +{ + struct timespec ts; + int foo = clock_gettime(CLOCK_MONOTONIC, &ts); + assert(foo == 0); + (void) foo; + nanoseconds start(ts.tv_sec*1000L*1000L*1000L + ts.tv_nsec); + nanoseconds end(start); + for (size_t i=0; i < 1000; i++) { + clock_gettime(CLOCK_MONOTONIC, &ts); + end = nanoseconds(ts.tv_sec*1000L*1000L*1000L + ts.tv_nsec); + } + return count_ns(start - end);; +} + +ClockMONOTONIC_RAW::ClockMONOTONIC_RAW() +{ +#ifndef CLOCK_MONOTONIC_RAW + LOG(warning, "CLOCK_MONOTONIC_RAW is not defined, using CLOCK_MONOTONIC instead."); +#endif +} + +#ifndef CLOCK_MONOTONIC_RAW + #define CLOCK_MONOTONIC_RAW CLOCK_MONOTONIC +#endif + +size_t ClockMONOTONIC_RAW::onRun() +{ + struct timespec ts; + int foo = clock_gettime(CLOCK_MONOTONIC_RAW, &ts); + assert(foo == 0); + (void) foo; + nanoseconds start(ts.tv_sec*1000L*1000L*1000L + ts.tv_nsec); + nanoseconds end(start); + for (size_t i=0; i < 1000; i++) { + clock_gettime(CLOCK_MONOTONIC_RAW, &ts); + end = nanoseconds(ts.tv_sec*1000L*1000L*1000L + ts.tv_nsec); + } + return count_ns(start - end); +} + +size_t ClockPROCESS_CPUTIME_ID::onRun() +{ + struct timespec ts; + int foo = clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts); + assert(foo == 0); + (void) foo; + nanoseconds start(ts.tv_sec*1000L*1000L*1000L + ts.tv_nsec); + nanoseconds end(start); + for (size_t i=0; i < 1000; i++) { + clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts); + end =nanoseconds(ts.tv_sec*1000L*1000L*1000L + ts.tv_nsec); + } + return count_ns(start - end); +} + +size_t ClockTHREAD_CPUTIME_ID::onRun() +{ + struct timespec ts; + int foo = clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts); + assert(foo == 0); + (void) foo; + nanoseconds start(ts.tv_sec*1000L*1000L*1000L + ts.tv_nsec); + nanoseconds end(start); + for (size_t i=0; i < 1000; i++) { + clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts); + end = nanoseconds(ts.tv_sec*1000L*1000L*1000L + ts.tv_nsec); + } + return count_ns(start - end); +} + +size_t CreateVespalibString::onRun() +{ + size_t sum(0); + const char * text1("Dette er en passe"); + const char * text2(" kort streng som passer paa stacken"); + char text[100]; + strcpy(text, text1); + strcat(text, text2); + for (size_t i=0; i < 1000; i++) { + string s(text); + sum += s.size(); + } + return sum; +} + +} diff --git a/vespalib/src/tests/benchmark/testbase.h b/vespalib/src/tests/benchmark/testbase.h new file mode 100644 index 00000000000..95621f52471 --- /dev/null +++ b/vespalib/src/tests/benchmark/testbase.h @@ -0,0 +1,168 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/objects/identifiable.h> +#include <vector> +#include <string> + +#define BENCHMARK_BASE_CID(n) (0x1000000 + n) + +#define CID_vespalib_Benchmark BENCHMARK_BASE_CID(0) +#define CID_vespalib_ParamByReferenceVectorInt BENCHMARK_BASE_CID(1) +#define CID_vespalib_ParamByReferenceVectorString BENCHMARK_BASE_CID(2) +#define CID_vespalib_ParamByValueVectorInt BENCHMARK_BASE_CID(3) +#define CID_vespalib_ParamByValueVectorString BENCHMARK_BASE_CID(4) +#define CID_vespalib_ReturnByReferenceVectorString BENCHMARK_BASE_CID(5) +#define CID_vespalib_ReturnByValueVectorString BENCHMARK_BASE_CID(6) +#define CID_vespalib_ReturnByValueMultiVectorString BENCHMARK_BASE_CID(7) +#define CID_vespalib_ClockSystem BENCHMARK_BASE_CID(8) +#define CID_vespalib_ClockREALTIME BENCHMARK_BASE_CID(10) +#define CID_vespalib_ClockMONOTONIC BENCHMARK_BASE_CID(11) +#define CID_vespalib_ClockMONOTONIC_RAW BENCHMARK_BASE_CID(12) +#define CID_vespalib_ClockPROCESS_CPUTIME_ID BENCHMARK_BASE_CID(13) +#define CID_vespalib_ClockTHREAD_CPUTIME_ID BENCHMARK_BASE_CID(14) +#define CID_vespalib_CreateVespalibString BENCHMARK_BASE_CID(15) + +#define DECLARE_BENCHMARK(a) DECLARE_IDENTIFIABLE_NS(vespalib, a) +#define IMPLEMENT_BENCHMARK(a, d) IMPLEMENT_IDENTIFIABLE_NS(vespalib, a, d) + +namespace vespalib { + +class Benchmark : public Identifiable +{ +public: + DECLARE_IDENTIFIABLE_ABSTRACT_NS(vespalib, Benchmark); + void run(size_t numRuns, size_t concurrency=1); + static void run(const char * testName, size_t numRuns, size_t concurrency); +private: + virtual size_t onRun() = 0; +}; + +class ParamByReferenceVectorInt : public Benchmark +{ +public: + DECLARE_BENCHMARK(ParamByReferenceVectorInt); +private: + typedef std::vector<int> Vector; + size_t callByReference(const Vector & values) const __attribute__((noinline)); + size_t onRun() override; +}; + +class ParamByValueVectorInt : public Benchmark +{ +public: + DECLARE_BENCHMARK(ParamByValueVectorInt); +private: + typedef std::vector<int> Vector; + size_t callByValue(Vector values) const __attribute__((noinline)); + size_t onRun() override; +}; + +class ParamByReferenceVectorString : public Benchmark +{ +public: + DECLARE_BENCHMARK(ParamByReferenceVectorString); +private: + typedef std::vector<std::string> Vector; + size_t callByReference(const Vector & values) const __attribute__((noinline)); + size_t onRun() override; +}; + +class ParamByValueVectorString : public Benchmark +{ +public: + DECLARE_BENCHMARK(ParamByValueVectorString); +private: + typedef std::vector<std::string> Vector; + size_t callByValue(Vector values) const __attribute__((noinline)); + size_t onRun() override; +}; + +class ReturnByReferenceVectorString : public Benchmark +{ +public: + DECLARE_BENCHMARK(ReturnByReferenceVectorString); +private: + typedef std::vector<std::string> Vector; + const Vector & returnByReference(Vector & values) const __attribute__((noinline)); + size_t onRun() override; +}; + +class ReturnByValueVectorString : public Benchmark +{ +public: + DECLARE_BENCHMARK(ReturnByValueVectorString); +private: + typedef std::vector<std::string> Vector; + Vector returnByValue() const __attribute__((noinline)); + size_t onRun() override; +}; + +class ReturnByValueMultiVectorString : public Benchmark +{ +public: + DECLARE_BENCHMARK(ReturnByValueMultiVectorString); +private: + typedef std::vector<std::string> Vector; + Vector returnByValue() const __attribute__((noinline)); + size_t onRun() override; +}; + +class CreateVespalibString : public Benchmark +{ +public: + DECLARE_BENCHMARK(CreateVespalibString); +private: + size_t onRun() override; +}; + +class ClockSystem : public Benchmark +{ +public: + DECLARE_BENCHMARK(ClockSystem); +private: + size_t onRun() override; +}; + +class ClockREALTIME : public Benchmark +{ +public: + DECLARE_BENCHMARK(ClockREALTIME); +private: + size_t onRun() override; +}; + +class ClockMONOTONIC : public Benchmark +{ +public: + DECLARE_BENCHMARK(ClockMONOTONIC); +private: + size_t onRun() override; +}; + +class ClockMONOTONIC_RAW : public Benchmark +{ +public: + DECLARE_BENCHMARK(ClockMONOTONIC_RAW); + ClockMONOTONIC_RAW(); +private: + size_t onRun() override; +}; + +class ClockPROCESS_CPUTIME_ID : public Benchmark +{ +public: + DECLARE_BENCHMARK(ClockPROCESS_CPUTIME_ID); +private: + size_t onRun() override; +}; + +class ClockTHREAD_CPUTIME_ID : public Benchmark +{ +public: + DECLARE_BENCHMARK(ClockTHREAD_CPUTIME_ID); +private: + size_t onRun() override; +}; + +} diff --git a/vespalib/src/tests/bits/.gitignore b/vespalib/src/tests/bits/.gitignore new file mode 100644 index 00000000000..b5330fc2580 --- /dev/null +++ b/vespalib/src/tests/bits/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +bits_test +vespalib_bits_test_app diff --git a/vespalib/src/tests/bits/CMakeLists.txt b/vespalib/src/tests/bits/CMakeLists.txt new file mode 100644 index 00000000000..f63196bc489 --- /dev/null +++ b/vespalib/src/tests/bits/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_bits_test_app TEST + SOURCES + bits_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_bits_test_app COMMAND vespalib_bits_test_app) diff --git a/vespalib/src/tests/bits/bits_test.cpp b/vespalib/src/tests/bits/bits_test.cpp new file mode 100644 index 00000000000..47d691c739d --- /dev/null +++ b/vespalib/src/tests/bits/bits_test.cpp @@ -0,0 +1,63 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/bits.h> +#include <boost/crc.hpp> +#include <boost/version.hpp> + +#if BOOST_VERSION < 106900 + #define REFLECT reflect +#else + #define REFLECT reflect_q +#endif + +using namespace vespalib; + +class Test : public TestApp +{ +public: + int Main() override; + template <typename T> + void testFixed(const T * v, size_t sz); + void testBuffer(); +}; + +int +Test::Main() +{ + TEST_INIT("bits_test"); + uint8_t u8[5] = { 0, 1, 127, 135, 255 }; + testFixed(u8, sizeof(u8)/sizeof(u8[0])); + uint16_t u16[5] = { 0, 1, 127, 135, 255 }; + testFixed(u16, sizeof(u16)/sizeof(u16[0])); + uint32_t u32[5] = { 0, 1, 127, 135, 255 }; + testFixed(u32, sizeof(u32)/sizeof(u32[0])); + uint64_t u64[5] = { 0, 1, 127, 135, 255 }; + testFixed(u64, sizeof(u64)/sizeof(u64[0])); + testBuffer(); + TEST_DONE(); +} + +template <typename T> +void Test::testFixed(const T * v, size_t sz) +{ + EXPECT_EQUAL(0u, Bits::reverse(static_cast<T>(0))); + EXPECT_EQUAL(1ul << (sizeof(T)*8 - 1), Bits::reverse(static_cast<T>(1))); + EXPECT_EQUAL(static_cast<T>(-1), Bits::reverse(static_cast<T>(-1))); + for (size_t i(0); i < sz; i++) { + EXPECT_EQUAL(Bits::reverse(v[i]), boost::detail::reflector<sizeof(T)*8>::REFLECT(v[i])); + EXPECT_EQUAL(Bits::reverse(Bits::reverse(v[i])), v[i]); + } +} + +void Test::testBuffer() +{ + uint64_t a(0x0102040810204080ul); + uint64_t b(a); + Bits::reverse(&a, sizeof(a)); + EXPECT_EQUAL(a, Bits::reverse(b)); + Bits::reverse(&a, sizeof(a)); + EXPECT_EQUAL(a, b); +} + +TEST_APPHOOK(Test) diff --git a/vespalib/src/tests/clock/.gitignore b/vespalib/src/tests/clock/.gitignore new file mode 100644 index 00000000000..96861fcc5d3 --- /dev/null +++ b/vespalib/src/tests/clock/.gitignore @@ -0,0 +1,5 @@ +.depend +Makefile +clock_test +vespalib_clock_test_app +vespalib_clock_benchmark_app diff --git a/vespalib/src/tests/clock/CMakeLists.txt b/vespalib/src/tests/clock/CMakeLists.txt new file mode 100644 index 00000000000..d3ee3178163 --- /dev/null +++ b/vespalib/src/tests/clock/CMakeLists.txt @@ -0,0 +1,14 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_clock_benchmark_app TEST + SOURCES + clock_benchmark.cpp + DEPENDS + vespalib +) +vespa_add_executable(vespalib_clock_test_app TEST + SOURCES + clock_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_clock_test_app COMMAND vespalib_clock_test_app) diff --git a/vespalib/src/tests/clock/clock_benchmark.cpp b/vespalib/src/tests/clock/clock_benchmark.cpp new file mode 100644 index 00000000000..249add4bc1a --- /dev/null +++ b/vespalib/src/tests/clock/clock_benchmark.cpp @@ -0,0 +1,178 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/util/clock.h> +#include <vespa/vespalib/util/invokeserviceimpl.h> +#include <vespa/fastos/thread.h> +#include <cassert> +#include <vector> +#include <atomic> +#include <cinttypes> +#include <cstring> +#include <condition_variable> +#include <mutex> + +using vespalib::Clock; +using vespalib::steady_time; +using vespalib::steady_clock; +using vespalib::duration; +using vespalib::to_s; + +struct UpdateClock { + virtual ~UpdateClock() {} + virtual void update() = 0; +}; + +struct NSValue : public UpdateClock { + void update() override { _value = std::chrono::steady_clock::now().time_since_epoch().count(); } + int64_t _value; +}; + +struct NSVolatile : public UpdateClock { + void update() override { _value = std::chrono::steady_clock::now().time_since_epoch().count(); } + volatile int64_t _value; +}; +struct NSAtomic : public UpdateClock { + void update() override { _value.store(std::chrono::steady_clock::now().time_since_epoch().count()); } + std::atomic<int64_t> _value; +}; + +class TestClock : public FastOS_Runnable +{ +private: + int _timePeriodMS; + std::mutex _lock; + std::condition_variable _cond; + UpdateClock &_clock; + bool _stop; + + void Run(FastOS_ThreadInterface *thisThread, void *arguments) override; + +public: + TestClock(UpdateClock & clock, double timePeriod) + : _timePeriodMS(static_cast<uint32_t>(timePeriod*1000)), + _lock(), + _cond(), + _clock(clock), + _stop(false) + { } + ~TestClock() { + std::lock_guard<std::mutex> guard(_lock); + _stop = true; + _cond.notify_all(); + } +}; + +void TestClock::Run(FastOS_ThreadInterface *thread, void *) +{ + std::unique_lock<std::mutex> guard(_lock); + while ( ! thread->GetBreakFlag() && !_stop) { + _clock.update(); + _cond.wait_for(guard, std::chrono::milliseconds(_timePeriodMS)); + } +} + +struct SamplerBase : public FastOS_Runnable { + SamplerBase(uint32_t threadId) + : _thread(nullptr), + _threadId(threadId), + _samples(0), + _count() + { + memset(_count, 0, sizeof(_count)); + } + FastOS_ThreadInterface * _thread; + uint32_t _threadId; + uint64_t _samples; + uint64_t _count[3]; +}; + +template<typename Func> +struct Sampler : public SamplerBase { + Sampler(Func func, uint32_t threadId) + : SamplerBase(threadId), + _func(func) + { } + void Run(FastOS_ThreadInterface *, void *) override { + uint64_t samples; + steady_time prev = _func(); + for (samples = 0; (samples < _samples); samples++) { + steady_time now = _func(); + duration diff = now - prev; + if (diff > duration::zero()) prev = now; + _count[1 + ((diff == duration::zero()) ? 0 : (diff > duration::zero()) ? 1 : -1)]++; + } + + } + Func _func; +}; + +template<typename Func> +void benchmark(const char * desc, FastOS_ThreadPool & pool, uint64_t samples, uint32_t numThreads, Func func) { + std::vector<std::unique_ptr<SamplerBase>> threads; + threads.reserve(numThreads); + steady_time start = steady_clock::now(); + for (uint32_t i(0); i < numThreads; i++) { + SamplerBase * sampler = new Sampler<Func>(func, i); + sampler->_samples = samples; + sampler->_thread = pool.NewThread(sampler, nullptr); + threads.emplace_back(sampler); + } + uint64_t count[3]; + memset(count, 0, sizeof(count)); + for (const auto & sampler : threads) { + sampler->_thread->Join(); + for (uint32_t i(0); i < 3; i++) { + count[i] += sampler->_count[i]; + } + } + printf("%s: Took %" PRId64 " clock samples in %2.3f with [%" PRId64 ", %" PRId64 ", %" PRId64 "] counts\n", desc, samples, to_s(steady_clock::now() - start), count[0], count[1], count[2]); +} + +int +main(int , char *argv[]) +{ + uint64_t frequency = atoll(argv[1]); + uint32_t numThreads = atoi(argv[2]); + uint64_t samples = atoll(argv[3]); + FastOS_ThreadPool pool(0x10000); + NSValue nsValue; + NSVolatile nsVolatile; + NSAtomic nsAtomic; + vespalib::InvokeServiceImpl invoker(vespalib::from_s(1.0/frequency)); + Clock clock(invoker.nowRef()); + TestClock nsClock(nsValue, 1.0/frequency); + TestClock nsVolatileClock(nsVolatile, 1.0/frequency); + TestClock nsAtomicClock(nsAtomic, 1.0/frequency); + assert(pool.NewThread(&nsClock, nullptr) != nullptr); + assert(pool.NewThread(&nsVolatileClock, nullptr) != nullptr); + assert(pool.NewThread(&nsAtomicClock, nullptr) != nullptr); + + benchmark("vespalib::Clock", pool, samples, numThreads, [&clock]() { + return clock.getTimeNS(); + }); + benchmark("uint64_t", pool, samples, numThreads, [&nsValue]() { + return steady_time (duration(nsValue._value)); + }); + benchmark("volatile uint64_t", pool, samples, numThreads, [&nsVolatile]() { + return steady_time(duration(nsVolatile._value)); + }); + benchmark("memory_order_relaxed", pool, samples, numThreads, [&nsAtomic]() { + return steady_time(duration(nsAtomic._value.load(std::memory_order_relaxed))); + }); + benchmark("memory_order_consume", pool, samples, numThreads, [&nsAtomic]() { + return steady_time(duration(nsAtomic._value.load(std::memory_order_consume))); + }); + benchmark("memory_order_acquire", pool, samples, numThreads, [&nsAtomic]() { + return steady_time(duration(nsAtomic._value.load(std::memory_order_acquire))); + }); + benchmark("memory_order_seq_cst", pool, samples, numThreads, [&nsAtomic]() { + return steady_time(duration(nsAtomic._value.load(std::memory_order_seq_cst))); + }); + + benchmark("vespalib::steady_time::now()", pool, samples, numThreads, []() { + return steady_clock::now(); + }); + + pool.Close(); + return 0; +} diff --git a/vespalib/src/tests/clock/clock_test.cpp b/vespalib/src/tests/clock/clock_test.cpp new file mode 100644 index 00000000000..f2de085da84 --- /dev/null +++ b/vespalib/src/tests/clock/clock_test.cpp @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/clock.h> +#include <vespa/vespalib/util/invokeserviceimpl.h> +#include <thread> + +using vespalib::Clock; +using vespalib::duration; +using vespalib::steady_time; +using vespalib::steady_clock; + +void waitForMovement(steady_time start, Clock & clock, vespalib::duration timeout) { + steady_time startOsClock = steady_clock::now(); + while ((clock.getTimeNS() <= start) && ((steady_clock::now() - startOsClock) < timeout)) { + std::this_thread::sleep_for(1ms); + } +} + +TEST("Test that clock is ticking forward") { + vespalib::InvokeServiceImpl invoker(50ms); + Clock clock(invoker.nowRef()); + steady_time start = clock.getTimeNS(); + waitForMovement(start, clock, 10s); + steady_time stop = clock.getTimeNS(); + EXPECT_TRUE(stop > start); +} + +TEST_MAIN() { TEST_RUN_ALL(); }
\ No newline at end of file diff --git a/vespalib/src/tests/crc/.gitignore b/vespalib/src/tests/crc/.gitignore new file mode 100644 index 00000000000..cd64c20e0a4 --- /dev/null +++ b/vespalib/src/tests/crc/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +crc_test +vespalib_crc_test_app diff --git a/vespalib/src/tests/crc/CMakeLists.txt b/vespalib/src/tests/crc/CMakeLists.txt new file mode 100644 index 00000000000..30adfd131f1 --- /dev/null +++ b/vespalib/src/tests/crc/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_crc_test_app TEST + SOURCES + crc_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_crc_test_app COMMAND vespalib_crc_test_app boost) diff --git a/vespalib/src/tests/crc/crc_test.cpp b/vespalib/src/tests/crc/crc_test.cpp new file mode 100644 index 00000000000..8afeed487ee --- /dev/null +++ b/vespalib/src/tests/crc/crc_test.cpp @@ -0,0 +1,78 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/crc.h> +#include <boost/crc.hpp> +#include <vector> + +using namespace vespalib; + +class Test : public TestApp +{ +public: + int Main() override; + void testCorrectNess(); + void testBenchmark(bool our, size_t bufSz, size_t numRep); +}; + +int +Test::Main() +{ + TEST_INIT("crc_test"); + testCorrectNess(); + if (_argc >= 2) { + testBenchmark(false, 1024, 1000*1000); + } else { + testBenchmark(true, 1024, 1000*1000); + } + TEST_DONE(); +} + +void Test::testCorrectNess() +{ + const char *a[7] = { "", "a", "ab", "abc", "abcd", "abcde", "doc:crawler:http://www.ntnu.no/" }; + for (size_t i(0); i < sizeof(a)/sizeof(a[0]); i++) { + uint32_t vespaCrc32 = crc_32_type::crc(a[i], strlen(a[i])); + boost::crc_32_type calculator; + calculator.process_bytes(a[i], strlen(a[i])); + EXPECT_EQUAL(vespaCrc32, calculator.checksum()); + vespalib::crc_32_type calculator2; + calculator2.process_bytes(a[i], strlen(a[i])); + EXPECT_EQUAL(vespaCrc32, calculator2.checksum()); + EXPECT_EQUAL(calculator.checksum(), calculator2.checksum()); + } + vespalib::crc_32_type calculator2; + boost::crc_32_type calculator; + for (size_t i(0); i < sizeof(a)/sizeof(a[0]); i++) { + calculator.process_bytes(a[i], strlen(a[i])); + calculator2.process_bytes(a[i], strlen(a[i])); + EXPECT_EQUAL(calculator.checksum(), calculator2.checksum()); + } + EXPECT_EQUAL(calculator.checksum(), calculator2.checksum()); +} + +void Test::testBenchmark(bool our, size_t bufSz, size_t numRep) +{ + std::vector<char> a(numRep+bufSz); + for(size_t i(0), m(a.size()); i < m; i++) { + a[i] = i&0xff; + } + uint32_t sum(0); + if (our) { + for (size_t i(0); i < (numRep); i++) { + //sum ^= crc_32_type::crc(&a[i], bufSz); + vespalib::crc_32_type calculator; + calculator.process_bytes(&a[i], bufSz); + sum ^=calculator.checksum(); + } + } else { + for (size_t i(0); i < (numRep); i++) { + boost::crc_32_type calculator; + calculator.process_bytes(&a[i], bufSz); + sum ^=calculator.checksum(); + } + } + printf("sum = %x\n", sum); +} + +TEST_APPHOOK(Test) diff --git a/vespalib/src/tests/directio/.gitignore b/vespalib/src/tests/directio/.gitignore new file mode 100644 index 00000000000..ad19022dfc3 --- /dev/null +++ b/vespalib/src/tests/directio/.gitignore @@ -0,0 +1 @@ +vespalib_directio_test_app diff --git a/vespalib/src/tests/directio/CMakeLists.txt b/vespalib/src/tests/directio/CMakeLists.txt new file mode 100644 index 00000000000..41a8dca85b9 --- /dev/null +++ b/vespalib/src/tests/directio/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_directio_test_app TEST + SOURCES + directio.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_directio_test_app COMMAND vespalib_directio_test_app) diff --git a/vespalib/src/tests/directio/directio.cpp b/vespalib/src/tests/directio/directio.cpp new file mode 100644 index 00000000000..77374f6f926 --- /dev/null +++ b/vespalib/src/tests/directio/directio.cpp @@ -0,0 +1,57 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/data/databuffer.h> +#include <vespa/fastos/file.h> + +using namespace vespalib; + +TEST("that DirectIOException propagates the correct information.") { + const char *msg("The buffer"); + DirectIOException e("file.a", msg, 10, 3); + EXPECT_EQUAL(10u, e.getLength()); + EXPECT_EQUAL(3u, e.getOffset()); + EXPECT_EQUAL(msg, e.getBuffer()); + EXPECT_EQUAL(0u, string(e.what()).find("DirectIO failed for file 'file.a' buffer=")); + EXPECT_EQUAL(string("file.a"), e.getFileName()); +} + +TEST("that DirectIOException is thrown on unaligned buf.") { + FastOS_File f("vespalib_directio_test_app"); + f.EnableDirectIO(); + EXPECT_TRUE(f.OpenReadOnly()); + DataBuffer buf(10000, 4_Ki); + bool caught(false); + try { + f.ReadBuf(buf.getFree()+1, 4_Ki, 0); + } catch (const DirectIOException & e) { + EXPECT_EQUAL(4_Ki, e.getLength()); + EXPECT_EQUAL(0u, e.getOffset()); + EXPECT_EQUAL(buf.getFree()+1, e.getBuffer()); + EXPECT_EQUAL(string(f.GetFileName()), e.getFileName()); + caught = true; + } + EXPECT_TRUE(caught); +} + +TEST("that DirectIOException is thrown on unaligned offset.") { + FastOS_File f("vespalib_directio_test_app"); + f.EnableDirectIO(); + EXPECT_TRUE(f.OpenReadOnly()); + DataBuffer buf(10000, 4_Ki); + bool caught(false); + try { + f.ReadBuf(buf.getFree(), 4_Ki, 1); + } catch (const DirectIOException & e) { + EXPECT_EQUAL(4_Ki, e.getLength()); + EXPECT_EQUAL(1u, e.getOffset()); + EXPECT_EQUAL(buf.getFree(), e.getBuffer()); + EXPECT_EQUAL(string(f.GetFileName()), e.getFileName()); + caught = true; + } + EXPECT_TRUE(caught); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/floatingpointtype/.gitignore b/vespalib/src/tests/floatingpointtype/.gitignore new file mode 100644 index 00000000000..abe8249f33a --- /dev/null +++ b/vespalib/src/tests/floatingpointtype/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +floatingpointtype_test +vespalib_floatingpointtype_test_app diff --git a/vespalib/src/tests/floatingpointtype/CMakeLists.txt b/vespalib/src/tests/floatingpointtype/CMakeLists.txt new file mode 100644 index 00000000000..3f0ec8eab69 --- /dev/null +++ b/vespalib/src/tests/floatingpointtype/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_floatingpointtype_test_app TEST + SOURCES + floatingpointtypetest.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_floatingpointtype_test_app COMMAND vespalib_floatingpointtype_test_app) diff --git a/vespalib/src/tests/floatingpointtype/floatingpointtypetest.cpp b/vespalib/src/tests/floatingpointtype/floatingpointtypetest.cpp new file mode 100644 index 00000000000..d26385f23bf --- /dev/null +++ b/vespalib/src/tests/floatingpointtype/floatingpointtypetest.cpp @@ -0,0 +1,73 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/objects/floatingpointtype.h> + +class Test : public vespalib::TestApp +{ +public: + void testFloatingPoint(); + int Main() override; +}; + +void +Test::testFloatingPoint() +{ + vespalib::Double d1(1.0); + vespalib::Double d2(1.000000000000001); + vespalib::Double d3(-1.00000000000001); + vespalib::Double d4(4.0); + + EXPECT_TRUE(d1.getValue() != d2.getValue()); + + EXPECT_EQUAL(d1, d2); + EXPECT_EQUAL(d2, d1); + + EXPECT_NOT_EQUAL(d1, d3); + EXPECT_NOT_EQUAL(d1, d4); + + EXPECT_TRUE(d1 - d2 == 0); + EXPECT_TRUE(d2 - d1 == 0); + + EXPECT_TRUE(d1 - 1 == 0); + EXPECT_TRUE(d1 + 1 != 0); + + EXPECT_TRUE(d2 * d4 == 4.0); + EXPECT_TRUE(d2 / d4 == 0.25); + + EXPECT_TRUE(d1 >= 1); + EXPECT_TRUE(d1 <= 1); + EXPECT_TRUE(!(d1 < 1)); + EXPECT_TRUE(!(d1 > 1)); + + EXPECT_EQUAL(d2 * 4, d4); + + EXPECT_EQUAL(++d4, 5.0); + EXPECT_EQUAL(d4++, 5.0); + EXPECT_EQUAL(d4, 6.0); + + d4 /= 3; + EXPECT_EQUAL(d4, 2.00000000001); + d4 *= 2; + EXPECT_EQUAL(d4, 4.000000000001); + + EXPECT_EQUAL(--d4, 3.0); + EXPECT_EQUAL(d4--, 3.0); + EXPECT_EQUAL(d4, 2.0); + d4 /= 0.50000000001; + + EXPECT_EQUAL(d4, 4.0); + + EXPECT_TRUE(!(d3 + 1 > 0)); + EXPECT_TRUE(!(d3 + 1 < 0)); +} + +int +Test::Main() +{ + TEST_INIT("floatingpointtype_test"); + testFloatingPoint(); + TEST_DONE(); +} + +TEST_APPHOOK(Test) diff --git a/vespalib/src/tests/growablebytebuffer/.gitignore b/vespalib/src/tests/growablebytebuffer/.gitignore new file mode 100644 index 00000000000..76218df9168 --- /dev/null +++ b/vespalib/src/tests/growablebytebuffer/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +growablebytebuffer_test +vespalib_growablebytebuffer_test_app diff --git a/vespalib/src/tests/growablebytebuffer/CMakeLists.txt b/vespalib/src/tests/growablebytebuffer/CMakeLists.txt new file mode 100644 index 00000000000..b518206ae56 --- /dev/null +++ b/vespalib/src/tests/growablebytebuffer/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_growablebytebuffer_test_app TEST + SOURCES + growablebytebuffer_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_growablebytebuffer_test_app COMMAND vespalib_growablebytebuffer_test_app) diff --git a/vespalib/src/tests/growablebytebuffer/growablebytebuffer_test.cpp b/vespalib/src/tests/growablebytebuffer/growablebytebuffer_test.cpp new file mode 100644 index 00000000000..0a616745023 --- /dev/null +++ b/vespalib/src/tests/growablebytebuffer/growablebytebuffer_test.cpp @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/growablebytebuffer.h> + +using namespace vespalib; + +class Test : public TestApp +{ +public: + void testGrowing(); + int Main() override; +}; + +void +Test::testGrowing() +{ + GrowableByteBuffer buf(10); + + buf.putInt(3); + buf.putInt(7); + buf.putLong(1234); + buf.putDouble(1234); + buf.putString("hei der"); + + EXPECT_EQUAL(35u, buf.position()); +} + +int +Test::Main() +{ + TEST_INIT("guard_test"); + testGrowing(); + TEST_DONE(); +} + +TEST_APPHOOK(Test) diff --git a/vespalib/src/tests/json/.gitignore b/vespalib/src/tests/json/.gitignore new file mode 100644 index 00000000000..9918fbce6e8 --- /dev/null +++ b/vespalib/src/tests/json/.gitignore @@ -0,0 +1 @@ +vespalib_json_test_app diff --git a/vespalib/src/tests/json/CMakeLists.txt b/vespalib/src/tests/json/CMakeLists.txt new file mode 100644 index 00000000000..0ea216b189b --- /dev/null +++ b/vespalib/src/tests/json/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_json_test_app TEST + SOURCES + json.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_json_test_app COMMAND vespalib_json_test_app boost) diff --git a/vespalib/src/tests/json/json.cpp b/vespalib/src/tests/json/json.cpp new file mode 100644 index 00000000000..1a707ae1776 --- /dev/null +++ b/vespalib/src/tests/json/json.cpp @@ -0,0 +1,470 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> + +#include <vespa/vespalib/util/jsonstream.h> +#include <vespa/vespalib/util/jsonexception.h> +#include <vespa/vespalib/stllike/asciistream.h> + +using namespace vespalib; + +class JSONTest : public vespalib::TestApp +{ +private: + void testJSONWriterValues(); + void testJSONWriterObject(); + void testJSONWriterArray(); + void testJSONWriterComplex(); + void testJsonStream(); + void testJsonStreamErrors(); + void testJsonStreamStateReporting(); + +public: + int Main() override; +}; + +void +JSONTest::testJSONWriterValues() +{ + JSONStringer js; + + { // bool + js.appendBool(true); + EXPECT_EQUAL(js.toString(), "true"); + js.clear().appendBool(false); + EXPECT_EQUAL(js.toString(), "false"); + } + { // double + js.clear().appendDouble(1234.5678); + EXPECT_EQUAL(js.toString(), "1234.5678"); + js.clear().appendDouble(-1234.5678); + EXPECT_EQUAL(js.toString(), "-1234.5678"); + js.clear().appendDouble(0.0); + EXPECT_EQUAL(js.toString(), "0.0"); + js.clear().appendDouble(0.00000000012345678912356789123456789); + EXPECT_EQUAL(js.toString(), "1.234567891235679e-10"); + js.clear().appendDouble(std::numeric_limits<double>::max()); + EXPECT_EQUAL(js.toString(), "1.797693134862316e+308"); + js.clear().appendDouble(std::numeric_limits<double>::min()); + EXPECT_EQUAL(js.toString(), "2.225073858507201e-308"); + js.clear().appendDouble(1.0 * (uint64_t(1) << 53)); + EXPECT_EQUAL(js.toString(), "9007199254740992.0"); + js.clear().appendDouble(1000); + EXPECT_EQUAL(js.toString(), "1000.0"); + } + { // float + js.clear().appendFloat(1234.5678f); + EXPECT_EQUAL(js.toString(), "1234.5677"); + js.clear().appendFloat(-1234.5678f); + EXPECT_EQUAL(js.toString(), "-1234.5677"); + js.clear().appendFloat(0.0f); + EXPECT_EQUAL(js.toString(), "0.0"); + js.clear().appendFloat(0.00000000012345678912356789123456789f); + EXPECT_EQUAL(js.toString(), "1.2345679e-10"); + js.clear().appendFloat(std::numeric_limits<float>::max()); + EXPECT_EQUAL(js.toString(), "3.4028235e+38"); + js.clear().appendFloat(std::numeric_limits<float>::min()); + EXPECT_EQUAL(js.toString(), "1.1754944e-38"); + js.clear().appendFloat(1.0 * (uint64_t(1) << 24)); + EXPECT_EQUAL(js.toString(), "16777216.0"); + js.clear().appendFloat(1000); + EXPECT_EQUAL(js.toString(), "1000.0"); + } + { // long + js.clear().appendInt64(4294967296ll); + EXPECT_EQUAL(js.toString(), "4294967296"); + js.clear().appendInt64(-4294967296ll); + EXPECT_EQUAL(js.toString(), "-4294967296"); + } + { // string + js.clear().appendString("string"); + EXPECT_EQUAL(js.toString(), "\"string\""); + } + { // NULL + js.clear().appendNull(); + EXPECT_EQUAL(js.toString(), "null"); + } + { // quote + js.clear().appendString("x\"y"); + EXPECT_EQUAL(js.toString(), "\"x\\\"y\""); + js.clear().appendString("x\\y"); + EXPECT_EQUAL(js.toString(), "\"x\\\\y\""); + js.clear().appendString("x/y"); + EXPECT_EQUAL(js.toString(), "\"x/y\""); + js.clear().appendString("x\by"); + EXPECT_EQUAL(js.toString(), "\"x\\by\""); + js.clear().appendString("x\fy"); + EXPECT_EQUAL(js.toString(), "\"x\\fy\""); + js.clear().appendString("x\ny"); + EXPECT_EQUAL(js.toString(), "\"x\\ny\""); + js.clear().appendString("x\ry"); + EXPECT_EQUAL(js.toString(), "\"x\\ry\""); + js.clear().appendString("x\ty"); + EXPECT_EQUAL(js.toString(), "\"x\\ty\""); + } +} + +void +JSONTest::testJSONWriterObject() +{ + JSONStringer js; + + { // single pair + js.beginObject().appendKey("k1").appendInt64(1l).endObject(); + EXPECT_EQUAL(js.toString(), "{\"k1\":1}"); + } + { // multiple pairs + js.clear().beginObject().appendKey("k1").appendInt64(1l).appendKey("k2").appendInt64(2l).endObject(); + EXPECT_EQUAL(js.toString(), "{\"k1\":1,\"k2\":2}"); + } + { // object in object + js.clear().beginObject().appendKey("k1").beginObject().appendKey("k1.1").appendInt64(11l).endObject().endObject(); + EXPECT_EQUAL(js.toString(), "{\"k1\":{\"k1.1\":11}}"); + } + { // object in object (multiple pairs) + js.clear().beginObject(). + appendKey("k1"). + beginObject(). + appendKey("k1.1").appendInt64(11l). + appendKey("k1.2").appendInt64(12l). + endObject(). + appendKey("k2"). + beginObject(). + appendKey("k2.1").appendInt64(21l). + appendKey("k2.2").appendInt64(22l). + endObject(). + endObject(); + EXPECT_EQUAL(js.toString(), "{\"k1\":{\"k1.1\":11,\"k1.2\":12},\"k2\":{\"k2.1\":21,\"k2.2\":22}}"); + } + { // array in object + js.clear().beginObject().appendKey("k1"). + beginArray().appendInt64(1l).appendInt64(2l).endArray().endObject(); + EXPECT_EQUAL(js.toString(), "{\"k1\":[1,2]}"); + } + { // array in object (multiple pairs) + js.clear().beginObject(). + appendKey("k1").beginArray().appendInt64(1l).appendInt64(2l).endArray(). + appendKey("k2").beginArray().appendInt64(3l).appendInt64(4l).endArray(). + endObject(); + EXPECT_EQUAL(js.toString(), "{\"k1\":[1,2],\"k2\":[3,4]}"); + } +} + + +void +JSONTest::testJSONWriterArray() +{ + JSONStringer js; + + { // single element + js.beginArray().appendInt64(1l).endArray(); + EXPECT_EQUAL(js.toString(), "[1]"); + } + { // multiple elements + js.clear().beginArray().appendInt64(1l).appendInt64(2l).endArray(); + EXPECT_EQUAL(js.toString(), "[1,2]"); + } + { // array in array + js.clear().beginArray().beginArray().appendInt64(1l).endArray().endArray(); + EXPECT_EQUAL(js.toString(), "[[1]]"); + } + { // array in array (multiple elements) + js.clear().beginArray(). + beginArray().appendInt64(1l).appendInt64(2l).endArray(). + beginArray().appendInt64(3l).appendInt64(4l).endArray(). + endArray(); + EXPECT_EQUAL(js.toString(), "[[1,2],[3,4]]"); + } + { // object in array + js.clear().beginArray(). + beginObject().appendKey("k1").appendInt64(1l).endObject(). + endArray(); + EXPECT_EQUAL(js.toString(), "[{\"k1\":1}]"); + } + { // object in array (multiple elements) + js.clear().beginArray(). + beginObject().appendKey("k1").appendInt64(1l).appendKey("k2").appendInt64(2l).endObject(). + beginObject().appendKey("k3").appendInt64(3l).appendKey("k4").appendInt64(4l).endObject(). + endArray(); + EXPECT_EQUAL(js.toString(), "[{\"k1\":1,\"k2\":2},{\"k3\":3,\"k4\":4}]"); + } +} + + +void +JSONTest::testJSONWriterComplex() +{ + JSONStringer js; + + js.beginObject(); + { // object + js.appendKey("k1"); + js.beginObject(); + { + js.appendKey("k1.1"); + js.appendInt64(1l); + } + { + js.appendKey("k1.2"); + js.beginArray(); + js.appendInt64(2l); + js.appendInt64(3l); + js.endArray(); + } + js.endObject(); + } + { // object of object + js.appendKey("k2"); + js.beginObject(); + { + js.appendKey("k2.1"); + js.beginObject(); + { + js.appendKey("k2.1.1"); + js.appendInt64(4l); + } + { + js.appendKey("k2.1.2"); + js.beginArray(); + js.appendInt64(5l); + js.appendInt64(6l); + js.endArray(); + } + js.endObject(); + } + js.endObject(); + } + { // array of object + js.appendKey("k3"); + js.beginArray(); + { + js.beginObject(); + { + js.appendKey("k3.1"); + js.appendInt64(7l); + } + { + js.appendKey("k3.2"); + js.beginArray(); + js.appendInt64(8l); + js.appendInt64(9l); + js.endArray(); + } + js.endObject(); + } + { + js.beginObject(); + { + js.appendKey("k3.1"); + js.appendInt64(10l); + } + { + js.appendKey("k3.2"); + js.beginArray(); + js.appendInt64(11l); + js.appendInt64(12l); + js.endArray(); + } + js.endObject(); + } + js.endArray(); + } + js.endObject(); + EXPECT_EQUAL(js.toString(), "{\"k1\":{\"k1.1\":1,\"k1.2\":[2,3]},\"k2\":{\"k2.1\":{\"k2.1.1\":4,\"k2.1.2\":[5,6]}},\"k3\":[{\"k3.1\":7,\"k3.2\":[8,9]},{\"k3.1\":10,\"k3.2\":[11,12]}]}"); +} + +namespace { + struct Builder : public vespalib::JsonStreamTypes { + void build(JsonStream& s) { + s << Object() << "k1" << Object() + << "k1.1" << 1 + << "k1.2" << Array() + << 2l << 3ll << End() + << End() + << "k2" << Object() + << "k2.1" << Object() + << "k2.1.1" << 4u + << "k2.1.2" << Array() + << 5ul << 6ull << End() + << End() + << End() + << "k3" << Array() + << Object() + << "k3.1" << -7 + << "k3.2" << Array() + << -8l << -9ll << End() + << End() + << Object() + << "k3.1" << 10l + << "k3.2" << Array() + << 11l << 12l << End() + << End() + << End() + << End(); + } + }; +} + +void +JSONTest::testJsonStream() +{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + Builder b; + b.build(stream); + stream.finalize(); + EXPECT_EQUAL(as.str(), "{\"k1\":{\"k1.1\":1,\"k1.2\":[2,3]},\"k2\":{\"k2.1\":{\"k2.1.1\":4,\"k2.1.2\":[5,6]}},\"k3\":[{\"k3.1\":-7,\"k3.2\":[-8,-9]},{\"k3.1\":10,\"k3.2\":[11,12]}]}"); +} + +void +JSONTest::testJsonStreamErrors() +{ + using namespace vespalib::jsonstream; + // Unsupported object keys + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << Object(); + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: An object value cannot be an object key ({}(ObjectExpectingKey))", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << true; + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: A bool value cannot be an object key ({}(ObjectExpectingKey))", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << 13; + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: An int64_t value cannot be an object key ({}(ObjectExpectingKey))", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << uint64_t(13); + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: A uint64_t value cannot be an object key ({}(ObjectExpectingKey))", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << 0.5; + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: A double value cannot be an object key ({}(ObjectExpectingKey))", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << jsonstream::Array(); + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: An array value cannot be an object key ({}(ObjectExpectingKey))", e.getReason()); + } + // Invalid points to add End() + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << "foo" << End(); + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: Object got key but not value. Cannot end it now ({foo}(ObjectExpectingValue))", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << End(); + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: No tag to end. At root ((RootExpectingArrayOrObjectStart))", e.getReason()); + } + // Adding to finalized stream + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << End() << "foo"; + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: Stream already finalized. Can't add a string value. (Finalized)", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << End() << false; + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: Stream already finalized. Can't add a bool value. (Finalized)", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << End() << 13; + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: Stream already finalized. Can't add a long long value. (Finalized)", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << End() << 13u; + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: Stream already finalized. Can't add an unsigned long long value. (Finalized)", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << End() << 0.2; + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: Stream already finalized. Can't add a double value. (Finalized)", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << End() << Object(); + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: Stream already finalized. Can't start a new object. (Finalized)", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << End() << jsonstream::Array(); + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: Stream already finalized. Can't start a new array. (Finalized)", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << End() << End(); + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: Stream already finalized. Can't end it. (Finalized)", e.getReason()); + } +} + +void +JSONTest::testJsonStreamStateReporting() +{ + using namespace vespalib::jsonstream; + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << jsonstream::Array() << 13 + << "foo" + << Object() << "key" << "value" << End() + << false + << End(); + EXPECT_EQUAL("Current: Finalized", stream.getJsonStreamState()); +} + +int +JSONTest::Main() +{ + TEST_INIT("json_test"); + + testJSONWriterValues(); + testJSONWriterObject(); + testJSONWriterArray(); + testJSONWriterComplex(); + testJsonStream(); + testJsonStreamErrors(); + testJsonStreamStateReporting(); + + TEST_DONE(); +} + +TEST_APPHOOK(JSONTest); + diff --git a/vespalib/src/tests/memorydatastore/.gitignore b/vespalib/src/tests/memorydatastore/.gitignore new file mode 100644 index 00000000000..6809bff5d3d --- /dev/null +++ b/vespalib/src/tests/memorydatastore/.gitignore @@ -0,0 +1 @@ +vespalib_memorydatastore_test_app diff --git a/vespalib/src/tests/memorydatastore/CMakeLists.txt b/vespalib/src/tests/memorydatastore/CMakeLists.txt new file mode 100644 index 00000000000..65d9231455a --- /dev/null +++ b/vespalib/src/tests/memorydatastore/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_memorydatastore_test_app TEST + SOURCES + memorydatastore.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_memorydatastore_test_app COMMAND vespalib_memorydatastore_test_app) diff --git a/vespalib/src/tests/memorydatastore/memorydatastore.cpp b/vespalib/src/tests/memorydatastore/memorydatastore.cpp new file mode 100644 index 00000000000..1d49b0af91b --- /dev/null +++ b/vespalib/src/tests/memorydatastore/memorydatastore.cpp @@ -0,0 +1,72 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/data/memorydatastore.h> +#include <vespa/vespalib/stllike/asciistream.h> + +using namespace vespalib; + +class MemoryDataStoreTest : public vespalib::TestApp +{ +private: + void testMemoryDataStore(); + void testVariableSizeVector(); +public: + int Main() override; +}; + +void +MemoryDataStoreTest::testMemoryDataStore() +{ + MemoryDataStore s(alloc::Alloc::alloc(256)); + std::vector<MemoryDataStore::Reference> v; + v.push_back(s.push_back("mumbo", 5)); + for (size_t i(0); i < 50; i++) { + v.push_back(s.push_back("mumbo", 5)); + EXPECT_EQUAL(static_cast<const char *>(v[i].data()) + 5, v[i+1].data()); + } + v.push_back(s.push_back("mumbo", 5)); + EXPECT_EQUAL(52ul, v.size()); + EXPECT_NOT_EQUAL(static_cast<const char *>(v[50].data()) + 5, v[51].data()); + for (size_t i(0); i < v.size(); i++) { + EXPECT_EQUAL(0, memcmp("mumbo", v[i].data(), 5)); + } +} + +void +MemoryDataStoreTest::testVariableSizeVector() +{ + VariableSizeVector v(20000, 5*20000); + for (size_t i(0); i < 10000; i++) { + asciistream os; + os << i; + v.push_back(os.str().data(), os.str().size()); + } + for (size_t i(0); i < v.size(); i++) { + asciistream os; + os << i; + EXPECT_EQUAL(os.str().size(), v[i].size()); + EXPECT_EQUAL(0, memcmp(os.str().data(), v[i].data(), os.str().size())); + } + size_t i(0); + for (auto it(v.begin()), mt(v.end()); it != mt; it++, i++) { + asciistream os; + os << i; + EXPECT_EQUAL(os.str().size(), it->size()); + EXPECT_EQUAL(0, memcmp(os.str().data(), (*it).data(), os.str().size())); + } + +} + +int +MemoryDataStoreTest::Main() +{ + TEST_INIT("data_test"); + testMemoryDataStore(); + testVariableSizeVector(); + + TEST_DONE(); +} + +TEST_APPHOOK(MemoryDataStoreTest); + diff --git a/vespalib/src/tests/objects/identifiable/.gitignore b/vespalib/src/tests/objects/identifiable/.gitignore new file mode 100644 index 00000000000..a547ace8ee4 --- /dev/null +++ b/vespalib/src/tests/objects/identifiable/.gitignore @@ -0,0 +1,5 @@ +.depend +Makefile +asciistream_test +identifiable_test +vespalib_identifiable_test_app diff --git a/vespalib/src/tests/objects/identifiable/CMakeLists.txt b/vespalib/src/tests/objects/identifiable/CMakeLists.txt new file mode 100644 index 00000000000..c4aefa44350 --- /dev/null +++ b/vespalib/src/tests/objects/identifiable/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_identifiable_test_app TEST + SOURCES + identifiable_test.cpp + namedobject.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_identifiable_test_app COMMAND vespalib_identifiable_test_app) diff --git a/vespalib/src/tests/objects/identifiable/identifiable_test.cpp b/vespalib/src/tests/objects/identifiable/identifiable_test.cpp new file mode 100644 index 00000000000..b3adfbfa9e2 --- /dev/null +++ b/vespalib/src/tests/objects/identifiable/identifiable_test.cpp @@ -0,0 +1,338 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "namedobject.h" +#include <vespa/vespalib/objects/identifiable.hpp> +#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/testkit/testapp.h> + +using namespace vespalib; + +class IdentifiableTest : public TestApp { + void requireThatIdentifiableCastCanCastPointers(); + void requireThatIdentifiableCastCanCastReferences(); + void testNamedObject(); + void testNboStream(); + template <typename T> + void testStream(const T & a); + void testNboSerializer(); + template <typename T> + void testSerializer(const T & a); +public: + int Main() override; +}; + +#define CID_Abstract 0x700000 +#define CID_A 0x700001 +#define CID_B 0x700002 +#define CID_C 0x700003 + +class Abstract : public Identifiable +{ +public: + DECLARE_IDENTIFIABLE_ABSTRACT(Abstract); + virtual ~Abstract() { } + virtual void someAbstractVirtualMethod() = 0; +}; + +class A : public Abstract +{ +public: + DECLARE_IDENTIFIABLE(A); + A() { } + void someAbstractVirtualMethod() override { }; +}; + +class B : public A +{ +public: + DECLARE_IDENTIFIABLE(B); + B() { } +}; + +class C : public Identifiable +{ +private: + int _value; + +public: + DECLARE_IDENTIFIABLE(C); + C() : _value(0) {} + C(int value) : _value(value) {} + C *clone() const { return new C(*this); } + virtual int cmp(const Identifiable &rhs) const { + int result(cmpClassId(rhs)); + if (result == 0) { + result = _value - static_cast<const C &>(rhs)._value; + } + return result; + } +}; + +IMPLEMENT_IDENTIFIABLE_ABSTRACT(Abstract, Identifiable); +IMPLEMENT_IDENTIFIABLE(A, Abstract); +IMPLEMENT_IDENTIFIABLE(B, A); +IMPLEMENT_IDENTIFIABLE(C, Identifiable); + +void +IdentifiableTest::testNamedObject() +{ + NamedObject a("first"), b("second");; + nbostream os; + NBOSerializer nos(os); + nos << a << b; + EXPECT_EQUAL(27u,os.size()); + Identifiable::UP o1; + o1 = Identifiable::create(nos); + EXPECT_EQUAL(14u, os.size()); + ASSERT_TRUE(o1->inherits(NamedObject::classId)); + ASSERT_TRUE(o1->getClass().id() == NamedObject::classId); + EXPECT_TRUE(static_cast<const NamedObject &>(*o1).getName() == "first"); + o1 = Identifiable::create(nos); + EXPECT_EQUAL(0u, os.size()); + ASSERT_TRUE(o1->inherits(NamedObject::classId)); + ASSERT_TRUE(o1->getClass().id() == NamedObject::classId); + EXPECT_TRUE(static_cast<const NamedObject &>(*o1).getName() == "second"); +} + +template <typename T> +void IdentifiableTest::testStream(const T & a) +{ + nbostream s; + s << a; + T b; + s >> b; + EXPECT_TRUE(s.empty()); + EXPECT_EQUAL(a, b); + EXPECT_EQUAL(nbostream::ok, s.state()); + EXPECT_TRUE(s.good()); +} + +template <typename T> +void IdentifiableTest::testSerializer(const T & a) +{ + nbostream t; + NBOSerializer s(t); + s << a; + T b; + s >> b; + EXPECT_TRUE(s.getStream().empty()); + EXPECT_EQUAL(a, b); + EXPECT_EQUAL(nbostream::ok, s.getStream().state()); +} + +void IdentifiableTest::testNboSerializer() +{ + testSerializer(true); + testSerializer(false); + testSerializer(static_cast<int8_t>('a')); + testSerializer(static_cast<uint8_t>(156)); + testSerializer(static_cast<int16_t>(156)); + testSerializer(static_cast<int32_t>(156)); + testSerializer(static_cast<int64_t>(156)); + testSerializer(static_cast<uint16_t>(156)); + testSerializer(static_cast<uint32_t>(156)); + testSerializer(static_cast<uint64_t>(156)); + testSerializer(static_cast<float>(156)); + testSerializer(static_cast<double>(156)); + testSerializer(vespalib::string("abcdefgh")); +} + +void IdentifiableTest::testNboStream() +{ + testStream(true); + testStream(false); + testStream('a'); + testStream(static_cast<unsigned char>(156)); + testStream(static_cast<int16_t>(156)); + testStream(static_cast<int32_t>(156)); + testStream(static_cast<int64_t>(156)); + testStream(static_cast<uint16_t>(156)); + testStream(static_cast<uint32_t>(156)); + testStream(static_cast<uint64_t>(156)); + testStream(static_cast<float>(156)); + testStream(static_cast<double>(156)); + testStream(std::string("abcdefgh")); + testStream(vespalib::string("abcdefgh")); + { + nbostream s(4); + EXPECT_EQUAL(4u, s.capacity()); + s << "abcdef"; + EXPECT_EQUAL(nbostream::ok, s.state()); + EXPECT_EQUAL(10u, s.size()); + EXPECT_EQUAL(16u, s.capacity()); + EXPECT_EQUAL(0, strncmp(s.data() + 4, "abcdef", 6)); + } + { + nbostream s(8); + EXPECT_EQUAL(0u, s.size()); + EXPECT_EQUAL(8u, s.capacity()); + const char * prev = s.data(); + s << "ABCD"; + EXPECT_EQUAL(8u, s.size()); + EXPECT_EQUAL(8u, s.capacity()); + EXPECT_EQUAL(prev, s.data()); + s << "A long string that will cause resizing"; + EXPECT_EQUAL(50u, s.size()); + EXPECT_EQUAL(64u, s.capacity()); + EXPECT_NOT_EQUAL(prev, s.data()); + } + { + nbostream s(8); + EXPECT_EQUAL(0u, s.size()); + EXPECT_EQUAL(8u, s.capacity()); + const char * prev = s.data(); + s << "ABCD"; + EXPECT_EQUAL(8u, s.size()); + EXPECT_EQUAL(8u, s.capacity()); + EXPECT_EQUAL(prev, s.data()); + s.reserve(50); + EXPECT_NOT_EQUAL(prev, s.data()); + EXPECT_EQUAL(8u, s.size()); + EXPECT_EQUAL(64u, s.capacity()); + prev = s.data(); + s << "A long string that will cause resizing"; + EXPECT_EQUAL(50u, s.size()); + EXPECT_EQUAL(64u, s.capacity()); + EXPECT_EQUAL(prev, s.data()); + } + { + nbostream s; + s << int64_t(9); + EXPECT_EQUAL(8u, s.size()); + EXPECT_EQUAL(0u, s.rp()); + int64_t a(7), b(1); + s >> a; + EXPECT_EQUAL(0u, s.size()); + EXPECT_EQUAL(8u, s.rp()); + EXPECT_TRUE(s.empty()); + EXPECT_TRUE(s.good()); + EXPECT_EQUAL(9, a); + try { + s >> b; + EXPECT_TRUE(false); + } catch (const IllegalStateException & e) { + EXPECT_EQUAL("Stream failed bufsize(1024), readp(8), writep(8)", e.getMessage()); + } + EXPECT_EQUAL(0u, s.size()); + EXPECT_EQUAL(8u, s.rp()); + EXPECT_TRUE(s.empty()); + EXPECT_FALSE(s.good()); + EXPECT_EQUAL(1, b); + EXPECT_EQUAL(nbostream::eof, s.state()); + } +} + +int +IdentifiableTest::Main() +{ + TEST_INIT("identifiable_test"); + + TEST_DO(requireThatIdentifiableCastCanCastPointers()); + TEST_DO(requireThatIdentifiableCastCanCastReferences()); + testNamedObject(); + testNboStream(); + testNboSerializer(); + + A a; + B b; + + const Identifiable::RuntimeClass & rtcA = a.getClass(); + EXPECT_EQUAL(rtcA.id(), static_cast<unsigned int>(A::classId)); + EXPECT_EQUAL(strcmp(rtcA.name(), "A"), 0); + + const Identifiable::RuntimeClass & rtcB = b.getClass(); + EXPECT_EQUAL(rtcB.id(), static_cast<unsigned int>(B::classId)); + EXPECT_EQUAL(strcmp(rtcB.name(), "B"), 0); + + const Identifiable::RuntimeClass * rt(Identifiable::classFromId(0x1ab76245)); + ASSERT_TRUE(rt == NULL); + rt = Identifiable::classFromId(Abstract::classId); + ASSERT_TRUE(rt != NULL); + Identifiable * u = rt->create(); + ASSERT_TRUE(u == NULL); + rt = Identifiable::classFromId(A::classId); + ASSERT_TRUE(rt != NULL); + rt = Identifiable::classFromId(B::classId); + ASSERT_TRUE(rt != NULL); + + Identifiable * o = rt->create(); + ASSERT_TRUE(o != NULL); + + const Identifiable::RuntimeClass & rtc = o->getClass(); + ASSERT_TRUE(rtc.id() == B::classId); + ASSERT_TRUE(strcmp(rtc.name(), "B") == 0); + ASSERT_TRUE(o->inherits(B::classId)); + ASSERT_TRUE(o->inherits(A::classId)); + ASSERT_TRUE(o->inherits(Abstract::classId)); + ASSERT_TRUE(o->inherits(Identifiable::classId)); + ASSERT_TRUE(o->getClass().id() == B::classId); + nbostream os; + NBOSerializer nos(os); + nos << *o; + EXPECT_EQUAL(os.size(), 4u); + Identifiable::UP o2 = Identifiable::create(nos); + EXPECT_TRUE(os.empty()); + ASSERT_TRUE(o->inherits(B::classId)); + ASSERT_TRUE(o->getClass().id() == B::classId); + delete o; + + rt = Identifiable::classFromName("NotBNorA"); + ASSERT_TRUE(rt == NULL); + rt = Identifiable::classFromName("B"); + ASSERT_TRUE(rt != NULL); + o = rt->create(); + ASSERT_TRUE(o != NULL); + const Identifiable::RuntimeClass & rtc2 = o->getClass(); + ASSERT_TRUE(rtc2.id() == B::classId); + ASSERT_TRUE(strcmp(rtc2.name(), "B") == 0); + ASSERT_TRUE(o->inherits(B::classId)); + ASSERT_TRUE(o->inherits(A::classId)); + ASSERT_TRUE(o->inherits(Abstract::classId)); + ASSERT_TRUE(o->inherits(Identifiable::classId)); + ASSERT_TRUE(o->getClass().id() == B::classId); + delete o; + + IdentifiablePtr<C> c0(NULL); + IdentifiablePtr<C> c1(new C(10)); + IdentifiablePtr<C> c2(new C(20)); + + EXPECT_LESS(c0.cmp(c1), 0); + EXPECT_EQUAL(c0.cmp(c0), 0); + EXPECT_GREATER(c1.cmp(c0), 0); + + EXPECT_LESS(c1.cmp(c2), 0); + EXPECT_EQUAL(c1.cmp(c1), 0); + EXPECT_GREATER(c2.cmp(c1), 0); + + TEST_DONE(); +} + +void IdentifiableTest::requireThatIdentifiableCastCanCastPointers() { + A a; + B b; + EXPECT_TRUE(Identifiable::cast<A *>(&a)); + EXPECT_TRUE(Identifiable::cast<A *>(&b)); + EXPECT_TRUE(!Identifiable::cast<B *>(&a)); + EXPECT_TRUE(Identifiable::cast<B *>(&b)); + EXPECT_TRUE(Identifiable::cast<Abstract *>(&a)); + EXPECT_TRUE(Identifiable::cast<Abstract *>(&b)); +} + +void IdentifiableTest::requireThatIdentifiableCastCanCastReferences() { + A a; + B b; + try { + // These should not throw. + Identifiable::cast<A &>(a); + Identifiable::cast<A &>(b); + Identifiable::cast<B &>(b); + Identifiable::cast<Abstract &>(a); + Identifiable::cast<Abstract &>(b); + } catch (std::bad_cast &e) { + TEST_FATAL(e.what()); + } + EXPECT_EXCEPTION(Identifiable::cast<B &>(a), std::bad_cast, "bad_cast"); +} + +TEST_APPHOOK(IdentifiableTest) diff --git a/vespalib/src/tests/objects/identifiable/namedobject.cpp b/vespalib/src/tests/objects/identifiable/namedobject.cpp new file mode 100644 index 00000000000..3e8d3291177 --- /dev/null +++ b/vespalib/src/tests/objects/identifiable/namedobject.cpp @@ -0,0 +1,19 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "namedobject.h" + +namespace vespalib { + +IMPLEMENT_IDENTIFIABLE_NS(vespalib, NamedObject, Identifiable); + + +Serializer & NamedObject::onSerialize(Serializer & os) const +{ + return os.put(_name); +} + +Deserializer & NamedObject::onDeserialize(Deserializer & is) +{ + return is.get(_name); +} + +} diff --git a/vespalib/src/tests/objects/identifiable/namedobject.h b/vespalib/src/tests/objects/identifiable/namedobject.h new file mode 100644 index 00000000000..784715a66f6 --- /dev/null +++ b/vespalib/src/tests/objects/identifiable/namedobject.h @@ -0,0 +1,23 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/objects/identifiable.h> +#include <string> + +namespace vespalib +{ + +class NamedObject : public Identifiable +{ +public: + DECLARE_IDENTIFIABLE_NS(vespalib, NamedObject); + DECLARE_NBO_SERIALIZE; + NamedObject() : _name() { } + NamedObject(const string & name) : _name(name) { } + const string & getName() const { return _name; } +private: + string _name; +}; + +} + diff --git a/vespalib/src/tests/objects/objectdump/.gitignore b/vespalib/src/tests/objects/objectdump/.gitignore new file mode 100644 index 00000000000..6ddd515391d --- /dev/null +++ b/vespalib/src/tests/objects/objectdump/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +objectdump_test +vespalib_objectdump_test_app diff --git a/vespalib/src/tests/objects/objectdump/CMakeLists.txt b/vespalib/src/tests/objects/objectdump/CMakeLists.txt new file mode 100644 index 00000000000..67395998b39 --- /dev/null +++ b/vespalib/src/tests/objects/objectdump/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_objectdump_test_app TEST + SOURCES + objectdump.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_objectdump_test_app COMMAND vespalib_objectdump_test_app) diff --git a/vespalib/src/tests/objects/objectdump/objectdump.cpp b/vespalib/src/tests/objects/objectdump/objectdump.cpp new file mode 100644 index 00000000000..812b1e79e17 --- /dev/null +++ b/vespalib/src/tests/objects/objectdump/objectdump.cpp @@ -0,0 +1,115 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/objects/identifiable.h> +#include <vespa/vespalib/objects/visit.hpp> + +#define CID_Base 10000000 +#define CID_Foo 10000001 +#define CID_Bar 10000002 +#define CID_Baz 10000003 + +using vespalib::ObjectVisitor; +using vespalib::IdentifiablePtr; + +struct Base : public vespalib::Identifiable +{ + DECLARE_IDENTIFIABLE(Base); + virtual Base *clone() const { return new Base(*this); } +}; +IMPLEMENT_IDENTIFIABLE(Base, vespalib::Identifiable); + +struct Baz : public Base +{ + DECLARE_IDENTIFIABLE(Baz); + Baz *clone() const override { return new Baz(*this); } +}; +IMPLEMENT_IDENTIFIABLE(Baz, Base); + +struct Bar : public Base +{ + DECLARE_IDENTIFIABLE(Bar); + bool _bool; + int8_t _int8; + uint8_t _uint8; + int16_t _int16; + uint16_t _uint16; + int32_t _int32; + uint32_t _uint32; + int64_t _int64; + uint64_t _uint64; + float _float; + double _double; + vespalib::string _string; + Bar() : _bool(true), _int8(-1), _uint8(1), _int16(-2), _uint16(2), + _int32(-4), _uint32(4), _int64(-8), _uint64(8), + _float(2.5), _double(2.75), _string("bla bla") {} + + Bar *clone() const override { return new Bar(*this); } + + void visitMembers(ObjectVisitor &v) const override { + visit(v, "_bool", _bool); + visit(v, "_int8", _int8); + visit(v, "_uint8", _uint8); + visit(v, "_int16", _int16); + visit(v, "_uint16", _uint16); + visit(v, "_int32", _int32); + visit(v, "_uint32", _uint32); + visit(v, "_int64", _int64); + visit(v, "_uint64", _uint64); + visit(v, "_float", _float); + visit(v, "_double", _double); + visit(v, "_string", _string); + visit(v, "info", "a dummy string"); + visit(v, "(const char*)0", (const char*)0); + } +}; +IMPLEMENT_IDENTIFIABLE(Bar, Base); + +struct Foo : public Base +{ + DECLARE_IDENTIFIABLE(Foo); + Bar _objMember; + Baz _objMember2; + Baz *_objPtr; + std::vector<Bar> _list; + std::vector<IdentifiablePtr<Base> > _list2; + + Foo(); + ~Foo(); + Foo *clone() const override { return new Foo(*this); } + void visitMembers(ObjectVisitor &v) const override; +}; + +Foo::~Foo() { } +Foo::Foo() + : _objMember(), _objMember2(), _objPtr(0), _list(), _list2() +{ + _list.push_back(Bar()); + _list.push_back(Bar()); + _list.push_back(Bar()); + _list2.push_back(Bar()); + _list2.push_back(Baz()); +} + +void +Foo::visitMembers(ObjectVisitor &v) const { + visit(v, "_objMember", _objMember); + visit(v, "_objMember2", _objMember2); + visit(v, "_objPtr", _objPtr); + visit(v, "_list", _list); + visit(v, "_list2", _list2); +} + +IMPLEMENT_IDENTIFIABLE(Foo, Base); + +TEST_SETUP(Test); + +int +Test::Main() +{ + TEST_INIT("objectdump_test"); + Foo foo; + fprintf(stderr, "%s", foo.asString().c_str()); + TEST_DONE(); +} diff --git a/vespalib/src/tests/objects/objectselection/.gitignore b/vespalib/src/tests/objects/objectselection/.gitignore new file mode 100644 index 00000000000..f6aefd07270 --- /dev/null +++ b/vespalib/src/tests/objects/objectselection/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +objectselection_test +vespalib_objectselection_test_app diff --git a/vespalib/src/tests/objects/objectselection/CMakeLists.txt b/vespalib/src/tests/objects/objectselection/CMakeLists.txt new file mode 100644 index 00000000000..94f43078820 --- /dev/null +++ b/vespalib/src/tests/objects/objectselection/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_objectselection_test_app TEST + SOURCES + objectselection.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_objectselection_test_app COMMAND vespalib_objectselection_test_app) diff --git a/vespalib/src/tests/objects/objectselection/objectselection.cpp b/vespalib/src/tests/objects/objectselection/objectselection.cpp new file mode 100644 index 00000000000..aa9c841f2dc --- /dev/null +++ b/vespalib/src/tests/objects/objectselection/objectselection.cpp @@ -0,0 +1,94 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/objects/identifiable.hpp> +#include <vespa/vespalib/objects/objectpredicate.h> +#include <vespa/vespalib/objects/objectoperation.h> + +using namespace vespalib; + +#define CID_Foo 60000005 +#define CID_Bar 60000010 + +struct Foo : public Identifiable +{ + typedef IdentifiablePtr<Foo> CP; + std::vector<CP> nodes; + + DECLARE_IDENTIFIABLE(Foo); + virtual Foo *clone() const { return new Foo(*this); } + void selectMembers(const ObjectPredicate &p, ObjectOperation &o) override { + for (uint32_t i = 0; i < nodes.size(); ++i) { + nodes[i]->select(p, o); + } + } +}; +IMPLEMENT_IDENTIFIABLE(Foo, Identifiable); + +struct Bar : public Foo +{ + int value; + + DECLARE_IDENTIFIABLE(Bar); + Bar() : value(0) {} + Bar(int v) { value = v; } + Bar *clone() const override { return new Bar(*this); } +}; +IMPLEMENT_IDENTIFIABLE(Bar, Identifiable); + +struct ObjectType : public ObjectPredicate +{ + uint32_t cid; + ObjectType(uint32_t id) : cid(id) {} + bool check(const Identifiable &obj) const override { + return (obj.getClass().id() == cid); + } +}; + +struct ObjectCollect : public ObjectOperation +{ + std::vector<Identifiable*> nodes; + ~ObjectCollect() override; + void execute(Identifiable &obj) override { + nodes.push_back(&obj); + } +}; + +ObjectCollect::~ObjectCollect() = default; + +TEST_SETUP(Test); + +int +Test::Main() +{ + TEST_INIT("objectselection_test"); + { + Foo f1; + Foo f2; + Foo f3; + Bar b1(1); + Bar b2(2); + Bar b3(3); + Bar b4(4); + f2.nodes.push_back(b1); + f2.nodes.push_back(b2); + f3.nodes.push_back(b3); + f3.nodes.push_back(b4); + f1.nodes.push_back(f2); + f1.nodes.push_back(f3); + + ObjectType predicate(Bar::classId); + ObjectCollect operation; + f1.select(predicate, operation); + ASSERT_TRUE(operation.nodes.size() == 4); + ASSERT_TRUE(operation.nodes[0]->getClass().id() == Bar::classId); + ASSERT_TRUE(operation.nodes[1]->getClass().id() == Bar::classId); + ASSERT_TRUE(operation.nodes[2]->getClass().id() == Bar::classId); + ASSERT_TRUE(operation.nodes[3]->getClass().id() == Bar::classId); + ASSERT_TRUE(((Bar*)operation.nodes[0])->value == 1); + ASSERT_TRUE(((Bar*)operation.nodes[1])->value == 2); + ASSERT_TRUE(((Bar*)operation.nodes[2])->value == 3); + ASSERT_TRUE(((Bar*)operation.nodes[3])->value == 4); + } + TEST_DONE(); +} diff --git a/vespalib/src/tests/polymorphicarray/.gitignore b/vespalib/src/tests/polymorphicarray/.gitignore new file mode 100644 index 00000000000..e0decc87f2c --- /dev/null +++ b/vespalib/src/tests/polymorphicarray/.gitignore @@ -0,0 +1 @@ +vespalib_polymorphicarray_test_app diff --git a/vespalib/src/tests/polymorphicarray/CMakeLists.txt b/vespalib/src/tests/polymorphicarray/CMakeLists.txt new file mode 100644 index 00000000000..14edfbec4b4 --- /dev/null +++ b/vespalib/src/tests/polymorphicarray/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_polymorphicarray_test_app TEST + SOURCES + polymorphicarray_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_polymorphicarray_test_app COMMAND vespalib_polymorphicarray_test_app) diff --git a/vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp b/vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp new file mode 100644 index 00000000000..d4ec8f3ed7c --- /dev/null +++ b/vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp @@ -0,0 +1,123 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/util/polymorphicarrays.h> + +using namespace vespalib; + +class A { +public: + virtual ~A() = default; + virtual void assign(const A & rhs) { (void) rhs; assert(false); } // Required by the primitive array. + virtual A * clone() const { assert(false); return nullptr; } // Required for the complex array. + + // For testing + virtual bool operator == (const A & rhs) const = 0; + virtual void print(std::ostream & os) const = 0; +}; + +class Primitive : public A +{ +public: + Primitive(size_t v=11) noexcept : _v(v) { } + size_t value() const { return _v; } + bool operator == (const A & rhs) const override { + return dynamic_cast<const Primitive &>(rhs).value() == value(); + } + void assign(const A & rhs) override { + _v = dynamic_cast<const Primitive &>(rhs).value(); + } + void print(std::ostream & os) const override { + os << _v; + } +private: + size_t _v; +}; + + +class Complex : public A +{ +public: + Complex(size_t v=11) noexcept : _v(v) { } + size_t value() const { return _v; } + bool operator == (const A & rhs) const override { + return dynamic_cast<const Complex &>(rhs).value() == value(); + } + Complex * clone() const override { + return new Complex(_v); + } + void print(std::ostream & os) const override { + os << _v; + } +private: + size_t _v; +}; + +std::ostream & operator << (std::ostream & os, const A & v) { + v.print(os); + return os; +} + + +template <typename T> +void +verifyArray(IArrayT<A> & array) +{ + EXPECT_EQUAL(0u, array.size()); + for (size_t i(0); i < 10; i++) { + array.push_back(T(i)); + } + EXPECT_EQUAL(10u, array.size()); + for (size_t i(0); i < 10; i++) { + EXPECT_EQUAL(T(i), array[i]); + } + IArrayT<A>::UP copy(array.clone()); + array.clear(); + EXPECT_EQUAL(0u, array.size()); + + for (size_t i(0); i < copy->size(); i++) { + array.push_back((*copy)[i]); + } + + array.resize(19); + EXPECT_EQUAL(19u, array.size()); + for (size_t i(0); i < 10; i++) { + EXPECT_EQUAL(T(i), array[i]); + } + for (size_t i(10); i < array.size(); i++) { + EXPECT_EQUAL(T(11), array[i]); + } + array.resize(13); + EXPECT_EQUAL(13u, array.size()); + for (size_t i(0); i < 10; i++) { + EXPECT_EQUAL(T(i), array[i]); + } + for (size_t i(10); i < array.size(); i++) { + EXPECT_EQUAL(T(11), array[i]); + } + dynamic_cast<T &>(array[1]) = T(17); + EXPECT_EQUAL(T(0), array[0]); + EXPECT_EQUAL(T(17), array[1]); + EXPECT_EQUAL(T(2), array[2]); +} + + +TEST("require that primitive arrays conforms") { + PrimitiveArrayT<Primitive, A> a; + verifyArray<Primitive>(a); + EXPECT_EQUAL(7u, a[7].value()); +} + +class Factory : public ComplexArrayT<A>::Factory +{ +public: + A * create() override { return new Complex(); } + Factory * clone() const override { return new Factory(*this); } +}; + +TEST("require that complex arrays conforms") { + ComplexArrayT<A> a(Factory::UP(new Factory())); + verifyArray<Complex>(a); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/programoptions/.gitignore b/vespalib/src/tests/programoptions/.gitignore new file mode 100644 index 00000000000..f083a1e093d --- /dev/null +++ b/vespalib/src/tests/programoptions/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +programoptions_test +vespalib_programoptions_test_app diff --git a/vespalib/src/tests/programoptions/CMakeLists.txt b/vespalib/src/tests/programoptions/CMakeLists.txt new file mode 100644 index 00000000000..fb2fdb48dd7 --- /dev/null +++ b/vespalib/src/tests/programoptions/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_programoptions_test_app TEST + SOURCES + programoptions_test.cpp + programoptions_testutils.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_programoptions_test_app COMMAND vespalib_programoptions_test_app) diff --git a/vespalib/src/tests/programoptions/programoptions_test.cpp b/vespalib/src/tests/programoptions/programoptions_test.cpp new file mode 100644 index 00000000000..4b63eae949b --- /dev/null +++ b/vespalib/src/tests/programoptions/programoptions_test.cpp @@ -0,0 +1,361 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "programoptions_testutils.h" +#include <vespa/vespalib/util/programoptions.h> +#include <vespa/vespalib/testkit/testapp.h> +#include <iostream> + +namespace vespalib { + +class Test : public vespalib::TestApp +{ +public: + void testSyntaxPage(); + void testNormalUsage(); + void testFailures(); + void testVectorArgument(); + void testAllHiddenOption(); + void testOptionsAfterArguments(); + int Main() override; +}; + +int +Test::Main() +{ + TEST_INIT("programoptions_test"); + srandom(1); + testSyntaxPage(); + testNormalUsage(); + testFailures(); + testVectorArgument(); + testAllHiddenOption(); + // Currently not supported + // testOptionsAfterArguments(); + TEST_DONE(); +} + +struct MyOptions : public ProgramOptions { + bool boolOpt; + bool boolWithDefOpt; + int intOpt; + uint32_t uintOpt; + float floatOpt; + std::string stringOpt; + std::string argString; + int argInt; + std::string argOptionalString; + std::map<std::string, std::string> properties; + int anotherOptionalArg; + + MyOptions(int argc, const char *const *argv); + ~MyOptions(); +}; + +MyOptions::MyOptions(int argc, const char* const* argv) + : ProgramOptions(argc, argv) +{ + // Required options + addOption("uintopt u", uintOpt, "Sets an unsigned int"); + // Optional options + addOption("b bool", boolOpt, "Enables a flag"); + addOption("boolwithdef", boolWithDefOpt, true, "If set turns to false"); + + addOption("intopt i", intOpt, 5, "Sets a signed int"); + addOption("floatopt", floatOpt, 4.0f, "Sets a float\nMultiline baby"); + addOption("string s", stringOpt, std::string("ballalaika"), + "Sets a string value. This is a very long description that " + "should be broken down into multiple lines in some sensible " + "way."); + addOptionHeader("Advanced options"); + addOption("p properties", properties, "Property map"); + addHiddenIdentifiers("prop"); + setArgumentTypeName("key"); + setArgumentTypeName("value", 1); + + addArgument("argString", argString, "Required string argument."); + addArgument("argInt", argInt, "Required int argument."); + addArgument("argOptionalString", argOptionalString, std::string("foo"), + "Optional string argument with a long description so we " + "can see that it will be broken correctly."); + addArgument("argSecondOptional", anotherOptionalArg, 3, + "Yet another optional argument"); + + setSyntaxMessage("A test program to see if this utility works."); + setSyntaxPageMaxLeftColumnSize(25); +} + +MyOptions::~MyOptions() { } + +void Test::testSyntaxPage() { + AppOptions opts("myapp"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + std::ostringstream actual; + options.writeSyntaxPage(actual); + + std::string expected( +"\nA test program to see if this utility works.\n\n" +"Usage: myapp [options] <argString> <argInt> [argOptionalString] [argSecondOptional]\n\n" +"Arguments:\n" +" argString (string) : Required string argument.\n" +" argInt (int) : Required int argument.\n" +" argOptionalString (string)\n" +" : Optional string argument with a long description so\n" +" we can see that it will be broken correctly.\n" +" (optional)\n" +" argSecondOptional (int) : Yet another optional argument (optional)\n\n" +"Options:\n" +" --uintopt -u <uint> : Sets an unsigned int (required)\n" +" -b --bool : Enables a flag\n" +" --boolwithdef : If set turns to false\n" +" --intopt -i <int> : Sets a signed int (default 5)\n" +" --floatopt <float> : Sets a float\n" +" Multiline baby (default 4)\n" +" --string -s <string> : Sets a string value. This is a very long description\n" +" that should be broken down into multiple lines in some\n" +" sensible way. (default \"ballalaika\")\n\n" +"Advanced options:\n" +" -p --properties <key> <value> : Property map (default empty)\n" + ); + EXPECT_EQUAL(expected, actual.str()); +} + +void Test::testNormalUsage() { + { + AppOptions opts("myapp -b --uintopt 4 -s foo tit 1 tei 6"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + options.parse(); + EXPECT_EQUAL(true, options.boolOpt); + EXPECT_EQUAL(true, options.boolWithDefOpt); + EXPECT_EQUAL(5, options.intOpt); + EXPECT_EQUAL(4u, options.uintOpt); + EXPECT_APPROX(4, options.floatOpt, 0.00001); + EXPECT_EQUAL("foo", options.stringOpt); + EXPECT_EQUAL("tit", options.argString); + EXPECT_EQUAL(1, options.argInt); + EXPECT_EQUAL("tei", options.argOptionalString); + EXPECT_EQUAL(0u, options.properties.size()); + EXPECT_EQUAL(6, options.anotherOptionalArg); + } + { + AppOptions opts("myapp --uintopt 6 tit 1"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + options.parse(); + EXPECT_EQUAL(false, options.boolOpt); + EXPECT_EQUAL(true, options.boolWithDefOpt); + EXPECT_EQUAL(5, options.intOpt); + EXPECT_EQUAL(6u, options.uintOpt); + EXPECT_APPROX(4, options.floatOpt, 0.00001); + EXPECT_EQUAL("ballalaika", options.stringOpt); + EXPECT_EQUAL("tit", options.argString); + EXPECT_EQUAL(1, options.argInt); + EXPECT_EQUAL("foo", options.argOptionalString); + EXPECT_EQUAL(0u, options.properties.size()); + EXPECT_EQUAL(3, options.anotherOptionalArg); + } + // Arguments coming after options. + // (Required for nesting of short options) + { + AppOptions opts("myapp --uintopt --intopt 6 -8 tit 1 tei"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + options.parse(); + EXPECT_EQUAL(false, options.boolOpt); + EXPECT_EQUAL(true, options.boolWithDefOpt); + EXPECT_EQUAL(-8, options.intOpt); + EXPECT_EQUAL(6u, options.uintOpt); + EXPECT_APPROX(4, options.floatOpt, 0.00001); + EXPECT_EQUAL("ballalaika", options.stringOpt); + EXPECT_EQUAL("tit", options.argString); + EXPECT_EQUAL(1, options.argInt); + EXPECT_EQUAL("tei", options.argOptionalString); + EXPECT_EQUAL(0u, options.properties.size()); + } + { + AppOptions opts( "myapp -uib 6 -8 --boolwithdef tit 1 tei"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + options.parse(); + EXPECT_EQUAL(true, options.boolOpt); + EXPECT_EQUAL(false, options.boolWithDefOpt); + EXPECT_EQUAL(-8, options.intOpt); + EXPECT_EQUAL(6u, options.uintOpt); + EXPECT_APPROX(4, options.floatOpt, 0.00001); + EXPECT_EQUAL("ballalaika", options.stringOpt); + EXPECT_EQUAL("tit", options.argString); + EXPECT_EQUAL(1, options.argInt); + EXPECT_EQUAL("tei", options.argOptionalString); + EXPECT_EQUAL(0u, options.properties.size()); + } + // Properties + { + AppOptions opts("myapp -u 6 -p foo bar --prop hmm brr tit 1 tei"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + options.parse(); + EXPECT_EQUAL(false, options.boolOpt); + EXPECT_EQUAL(true, options.boolWithDefOpt); + EXPECT_EQUAL(5, options.intOpt); + EXPECT_EQUAL(6u, options.uintOpt); + EXPECT_APPROX(4, options.floatOpt, 0.00001); + EXPECT_EQUAL("ballalaika", options.stringOpt); + EXPECT_EQUAL("tit", options.argString); + EXPECT_EQUAL(1, options.argInt); + EXPECT_EQUAL("tei", options.argOptionalString); + EXPECT_EQUAL(2u, options.properties.size()); + EXPECT_EQUAL("bar", options.properties["foo"]); + EXPECT_EQUAL("brr", options.properties["hmm"]); + } +} + +void Test::testFailures() { + // Non-existing long option + { + AppOptions opts("myapp -b --uintopt 4 -s foo --none"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + try{ + options.parse(); + TEST_FATAL("Expected exception"); + } catch (InvalidCommandLineArgumentsException& e) { + EXPECT_EQUAL("Invalid option 'none'.", e.getMessage()); + } + } + // Non-existing short option + { + AppOptions opts("myapp -b --uintopt 4 -s foo -q"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + try{ + options.parse(); + TEST_FATAL("Expected exception"); + } catch (InvalidCommandLineArgumentsException& e) { + EXPECT_EQUAL("Invalid option 'q'.", e.getMessage()); + } + } + // Lacking option argument + { + AppOptions opts("myapp -b --uintopt 4 -s"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + try{ + options.parse(); + TEST_FATAL("Expected exception"); + } catch (InvalidCommandLineArgumentsException& e) { + EXPECT_EQUAL("Option 's' needs 1 arguments. Only 0 available.", + e.getMessage()); + } + } + // Out of signed ranged + { + AppOptions opts("myapp -b --uintopt 4 -intopt 3000000000"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + try{ + options.parse(); + TEST_FATAL("Expected exception"); + } catch (InvalidCommandLineArgumentsException& e) { + EXPECT_EQUAL("The argument '3000000000' can not be interpreted as a " + "number of type int.", e.getMessage()); + } + } + // Negative value to unsigned var (Currently doesnt fail) +/* + { + AppOptions opts("myapp -b --uintopt -1 foo 0"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + try{ + options.parse(); + TEST_FATAL("Expected exception"); + } catch (InvalidCommandLineArgumentsException& e) { + EXPECT_EQUAL("The argument '-1' can not be interpreted as a " + "number of type uint.", e.getMessage()); + } + } + */ + // Lacking required option + { + AppOptions opts("myapp -b"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + try{ + options.parse(); + TEST_FATAL("Expected exception"); + } catch (InvalidCommandLineArgumentsException& e) { + EXPECT_EQUAL("Option 'uintopt' has no default and must be set.", + e.getMessage()); + } + } + // Lacking required argument + { + AppOptions opts("myapp --uintopt 1 tit"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + try{ + options.parse(); + TEST_FATAL("Expected exception"); + } catch (InvalidCommandLineArgumentsException& e) { + EXPECT_EQUAL("Insufficient data is given to set required argument " + "'argInt'.", + e.getMessage()); + } + } + // Argument of wrong type + { + AppOptions opts("myapp --uintopt 1 tit en"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + try{ + options.parse(); + TEST_FATAL("Expected exception"); + } catch (InvalidCommandLineArgumentsException& e) { + EXPECT_EQUAL("The argument 'en' can not be interpreted as a number " + "of type int.", + e.getMessage()); + } + } +} + +void Test::testVectorArgument() +{ + AppOptions opts("myapp foo bar baz"); + std::vector<std::string> args; + ProgramOptions options(opts.getArgCount(), opts.getArguments()); + options.addListArgument("ids", args, "Vector element"); + std::ostringstream actual; + options.writeSyntaxPage(actual); + std::string expected( +"\nUsage: myapp [ids...]\n\n" +"Arguments:\n" +" ids (string[]) : Vector element\n" + ); + EXPECT_EQUAL(expected, actual.str()); + + options.parse(); + EXPECT_EQUAL(3u, args.size()); + EXPECT_EQUAL("foo", args[0]); + EXPECT_EQUAL("bar", args[1]); + EXPECT_EQUAL("baz", args[2]); +} + +void Test::testAllHiddenOption() +{ + AppOptions opts("myapp --foo bar"); + std::string option; + ProgramOptions options(opts.getArgCount(), opts.getArguments()); + options.addOption("", option, "Description"); + options.addHiddenIdentifiers("foo"); + std::ostringstream actual; + options.writeSyntaxPage(actual); + std::string expected("\nUsage: myapp\n"); + EXPECT_EQUAL(expected, actual.str()); + + options.parse(); + EXPECT_EQUAL("bar", option); +} + +void Test::testOptionsAfterArguments() +{ + AppOptions opts("myapp bar --foo baz"); + std::string option; + std::string argument; + ProgramOptions options(opts.getArgCount(), opts.getArguments()); + options.addOption("foo", option, "Description"); + options.addArgument("arg", argument, "Description"); + options.parse(); + EXPECT_EQUAL("baz", option); + EXPECT_EQUAL("bar", argument); +} + +} // vespalib + +TEST_APPHOOK(vespalib::Test) diff --git a/vespalib/src/tests/programoptions/programoptions_testutils.cpp b/vespalib/src/tests/programoptions/programoptions_testutils.cpp new file mode 100644 index 00000000000..948413c36db --- /dev/null +++ b/vespalib/src/tests/programoptions/programoptions_testutils.cpp @@ -0,0 +1,48 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "programoptions_testutils.h" + +namespace vespalib { + +namespace { + std::vector<std::string> splitString(const std::string& source) { + std::vector<std::string> target; + std::string::size_type start = 0; + std::string::size_type stop = source.find(' '); + while (stop != std::string::npos) { + target.push_back(source.substr(start, stop - start)); + start = stop + 1; + stop = source.find(' ', start); + } + target.push_back(source.substr(start)); + return target; + } +} // anonymous + +AppOptions::AppOptions(const std::string& optString) + : _argc(0), _argv(0), _source() +{ + _source = splitString(optString); + _argc = _source.size(); + _argv = new const char*[_source.size()]; + for (int i=0; i<_argc; ++i) { + if (_source[i].size() > 1 + && _source[i][0] == _source[i][_source[i].size() - 1] + && (_source[i][0] == '\'' || _source[i][0] == '"')) + { + if (_source[i].size() == 2) { + _source[i] = ""; + } else { + _source[i] = _source[i].substr(1, _source.size() - 2); + } + } + _argv[i] = _source[i].c_str(); + } +} + +AppOptions::~AppOptions() +{ + delete[] _argv; +} + +} // vespalib diff --git a/vespalib/src/tests/programoptions/programoptions_testutils.h b/vespalib/src/tests/programoptions/programoptions_testutils.h new file mode 100644 index 00000000000..a6f103f3e95 --- /dev/null +++ b/vespalib/src/tests/programoptions/programoptions_testutils.h @@ -0,0 +1,32 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * This class contains some test utilities, to create argc/argv inputs for + * application tests. + */ + +#pragma once + +#include <string> +#include <vector> + +namespace vespalib { + +class AppOptions { + int _argc; + const char** _argv; + std::vector<std::string> _source; + + AppOptions(const AppOptions&); + AppOptions& operator=(const AppOptions&); + +public: + AppOptions(const std::string& optString); + ~AppOptions(); + + int getArgCount() const { return _argc; } + const char* const* getArguments() const { return _argv; } + +}; + +} // vespalib + diff --git a/vespalib/src/tests/rusage/.gitignore b/vespalib/src/tests/rusage/.gitignore new file mode 100644 index 00000000000..c01c01ed328 --- /dev/null +++ b/vespalib/src/tests/rusage/.gitignore @@ -0,0 +1 @@ +vespalib_rusage_test_app diff --git a/vespalib/src/tests/rusage/CMakeLists.txt b/vespalib/src/tests/rusage/CMakeLists.txt new file mode 100644 index 00000000000..1c1ab85facd --- /dev/null +++ b/vespalib/src/tests/rusage/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_rusage_test_app TEST + SOURCES + rusage_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_rusage_test_app COMMAND vespalib_rusage_test_app) diff --git a/vespalib/src/tests/rusage/rusage_test.cpp b/vespalib/src/tests/rusage/rusage_test.cpp new file mode 100644 index 00000000000..7e30f3b968b --- /dev/null +++ b/vespalib/src/tests/rusage/rusage_test.cpp @@ -0,0 +1,57 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/rusage.h> + +using namespace vespalib; + +TEST("testRUsage") +{ + RUsage r1; + EXPECT_EQUAL("", r1.toString()); + RUsage r2; + EXPECT_EQUAL(r2.toString(), r1.toString()); + RUsage diff = r2-r1; + EXPECT_EQUAL(diff.toString(), r2.toString()); + { + RUsage then = RUsage::createSelf(steady_time(7ns)); + RUsage now = RUsage::createSelf(); + EXPECT_NOT_EQUAL(now.toString(), then.toString()); + } + { + RUsage then = RUsage::createChildren(steady_time(1337583ns)); + RUsage now = RUsage::createChildren(); + EXPECT_NOT_EQUAL(now.toString(), then.toString()); + } + { + timeval a, b, c, d, r; + a.tv_usec = 7; + a.tv_sec = 7; + b.tv_usec = 7; + b.tv_sec = 7; + c.tv_usec = 1; + c.tv_sec = 8; + d.tv_usec = 9; + d.tv_sec = 4; + r = a - b; + EXPECT_EQUAL(0, r.tv_sec); + EXPECT_EQUAL(0, r.tv_usec); + r = b - a; + EXPECT_EQUAL(0, r.tv_sec); + EXPECT_EQUAL(0, r.tv_usec); + r = a - c; + EXPECT_EQUAL(-1, r.tv_sec); + EXPECT_EQUAL( 6, r.tv_usec); + r = c - a; + EXPECT_EQUAL(0, r.tv_sec); + EXPECT_EQUAL(999994, r.tv_usec); + r = a - d; + EXPECT_EQUAL(2, r.tv_sec); + EXPECT_EQUAL(999998, r.tv_usec); + r = d - a; + EXPECT_EQUAL(-3, r.tv_sec); + EXPECT_EQUAL( 2, r.tv_usec); + } +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/shutdownguard/.gitignore b/vespalib/src/tests/shutdownguard/.gitignore new file mode 100644 index 00000000000..c167d4784ca --- /dev/null +++ b/vespalib/src/tests/shutdownguard/.gitignore @@ -0,0 +1 @@ +vespalib_shutdownguard_test_app diff --git a/vespalib/src/tests/shutdownguard/CMakeLists.txt b/vespalib/src/tests/shutdownguard/CMakeLists.txt new file mode 100644 index 00000000000..6714842fbbf --- /dev/null +++ b/vespalib/src/tests/shutdownguard/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_shutdownguard_test_app TEST + SOURCES + shutdownguard_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_shutdownguard_test_app NO_VALGRIND COMMAND vespalib_shutdownguard_test_app) diff --git a/vespalib/src/tests/shutdownguard/shutdownguard_test.cpp b/vespalib/src/tests/shutdownguard/shutdownguard_test.cpp new file mode 100644 index 00000000000..348e9bbd503 --- /dev/null +++ b/vespalib/src/tests/shutdownguard/shutdownguard_test.cpp @@ -0,0 +1,43 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/shutdownguard.h> +#include <vespa/vespalib/util/malloc_mmap_guard.h> +#include <thread> +#include <unistd.h> +#include <sys/wait.h> +#include <cstdlib> + +using namespace vespalib; + +TEST("test shutdown guard") +{ + { + ShutdownGuard farFuture(1000000s); + std::this_thread::sleep_for(20ms); + } + EXPECT_TRUE(true); + pid_t child = fork(); + if (child == 0) { + ShutdownGuard soon(30ms); + for (int i = 0; i < 1000; ++i) { + std::this_thread::sleep_for(20ms); + } + std::_Exit(0); + } + for (int i = 0; i < 1000; ++i) { + std::this_thread::sleep_for(20ms); + int stat = 0; + if (waitpid(child, &stat, WNOHANG) == child) { + EXPECT_TRUE(WIFEXITED(stat)); + EXPECT_EQUAL(1, WEXITSTATUS(stat)); + break; + } + EXPECT_TRUE(i < 800); + } +} + +TEST("test malloc mmap guard") { + MallocMmapGuard guard(0x100000); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/stllike/CMakeLists.txt b/vespalib/src/tests/stllike/CMakeLists.txt index 80509c565c6..005ef3d1ed0 100644 --- a/vespalib/src/tests/stllike/CMakeLists.txt +++ b/vespalib/src/tests/stllike/CMakeLists.txt @@ -55,3 +55,17 @@ vespa_add_executable(vespalib_replace_variable_test_app TEST GTest::GTest ) vespa_add_test(NAME vespalib_replace_variable_test_app COMMAND vespalib_replace_variable_test_app) +vespa_add_executable(vespalib_lrucache_test_app TEST + SOURCES + lrucache.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_lrucache_test_app COMMAND vespalib_lrucache_test_app) +vespa_add_executable(vespalib_cache_test_app TEST + SOURCES + cache_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_cache_test_app COMMAND vespalib_cache_test_app) diff --git a/vespalib/src/tests/stllike/cache_test.cpp b/vespalib/src/tests/stllike/cache_test.cpp new file mode 100644 index 00000000000..35f04d91510 --- /dev/null +++ b/vespalib/src/tests/stllike/cache_test.cpp @@ -0,0 +1,139 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/stllike/cache.hpp> +#include <map> + +using namespace vespalib; + +template<typename K, typename V> +class Map : public std::map<K, V> { + typedef typename std::map<K, V>::const_iterator const_iterator; + typedef std::map<K, V> M; +public: + bool read(const K & k, V & v) const { + const_iterator found = M::find(k); + bool ok(found != this->end()); + if (ok) { + v = found->second; + } + return ok; + } + void write(const K & k, const V & v) { + (*this)[k] = v; + } + void erase(const K & k) { + M::erase(k); + } +}; + +using P = LruParam<uint32_t, string>; +using B = Map<uint32_t, string>; + +TEST("testCache") { + B m; + cache< CacheParam<P, B> > cache(m, -1); + // Verfify start conditions. + EXPECT_TRUE(cache.size() == 0); + EXPECT_TRUE( ! cache.hasKey(1) ); + cache.write(1, "First inserted string"); + EXPECT_TRUE( cache.hasKey(1) ); + m[2] = "String inserted beneath"; + EXPECT_TRUE( ! cache.hasKey(2) ); + EXPECT_EQUAL( cache.read(2), "String inserted beneath"); + EXPECT_TRUE( cache.hasKey(2) ); + cache.erase(1); + EXPECT_TRUE( ! cache.hasKey(1) ); + EXPECT_TRUE(cache.size() == 1); +} + +TEST("testCacheSize") +{ + B m; + cache< CacheParam<P, B> > cache(m, -1); + cache.write(1, "10 bytes string"); + EXPECT_EQUAL(80u, cache.sizeBytes()); + cache.write(1, "10 bytes string"); // Still the same size + EXPECT_EQUAL(80u, cache.sizeBytes()); +} + +TEST("testCacheSizeDeep") +{ + B m; + cache< CacheParam<P, B, zero<uint32_t>, size<string> > > cache(m, -1); + cache.write(1, "15 bytes string"); + EXPECT_EQUAL(95u, cache.sizeBytes()); + cache.write(1, "10 bytes s"); + EXPECT_EQUAL(90u, cache.sizeBytes()); + cache.write(1, "20 bytes string ssss"); + EXPECT_EQUAL(100u, cache.sizeBytes()); +} + +TEST("testCacheEntriesHonoured") { + B m; + cache< CacheParam<P, B, zero<uint32_t>, size<string> > > cache(m, -1); + cache.maxElements(1); + cache.write(1, "15 bytes string"); + EXPECT_EQUAL(1u, cache.size()); + EXPECT_EQUAL(95u, cache.sizeBytes()); + cache.write(2, "16 bytes stringg"); + EXPECT_EQUAL(1u, cache.size()); + EXPECT_TRUE( cache.hasKey(2) ); + EXPECT_FALSE( cache.hasKey(1) ); + EXPECT_EQUAL(96u, cache.sizeBytes()); +} + +TEST("testCacheMaxSizeHonoured") { + B m; + cache< CacheParam<P, B, zero<uint32_t>, size<string> > > cache(m, 200); + cache.write(1, "15 bytes string"); + EXPECT_EQUAL(1u, cache.size()); + EXPECT_EQUAL(95u, cache.sizeBytes()); + cache.write(2, "16 bytes stringg"); + EXPECT_EQUAL(2u, cache.size()); + EXPECT_EQUAL(191u, cache.sizeBytes()); + cache.write(3, "17 bytes stringgg"); + EXPECT_EQUAL(3u, cache.size()); + EXPECT_EQUAL(288u, cache.sizeBytes()); + cache.write(4, "18 bytes stringggg"); + EXPECT_EQUAL(3u, cache.size()); + EXPECT_EQUAL(291u, cache.sizeBytes()); +} + +TEST("testThatMultipleRemoveOnOverflowIsFine") { + B m; + cache< CacheParam<P, B, zero<uint32_t>, size<string> > > cache(m, 2000); + + for (size_t j(0); j < 5; j++) { + for (size_t i(0); cache.size() == i; i++) { + cache.write(j*53+i, "a"); + } + } + EXPECT_EQUAL(25u, cache.size()); + EXPECT_EQUAL(2025u, cache.sizeBytes()); + EXPECT_FALSE( cache.hasKey(0) ); + string ls("long string aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + string vls=ls+ls+ls+ls+ls+ls; + cache.write(53+5, ls); + EXPECT_EQUAL(25u, cache.size()); + EXPECT_EQUAL(2498u, cache.sizeBytes()); + EXPECT_FALSE( cache.hasKey(1) ); + cache.write(53*7+5, ls); + EXPECT_EQUAL(19u, cache.size()); + EXPECT_EQUAL(2485u, cache.sizeBytes()); + EXPECT_FALSE( cache.hasKey(2) ); + cache.write(53*8+5, vls); + EXPECT_EQUAL(14u, cache.size()); + EXPECT_EQUAL(4923u, cache.sizeBytes()); + cache.write(53*9+6, vls); + EXPECT_EQUAL(1u, cache.size()); + EXPECT_EQUAL(2924u, cache.sizeBytes()); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/stllike/lrucache.cpp b/vespalib/src/tests/stllike/lrucache.cpp new file mode 100644 index 00000000000..2cc6f2b4ee8 --- /dev/null +++ b/vespalib/src/tests/stllike/lrucache.cpp @@ -0,0 +1,190 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/stllike/lrucache_map.hpp> + +using namespace vespalib; + +TEST("testCache") { + lrucache_map< LruParam<int, string> > cache(7); + // Verfify start conditions. + EXPECT_TRUE(cache.size() == 0); + cache.insert(1, "First inserted string"); + EXPECT_TRUE(cache.verifyInternals()); + EXPECT_TRUE(cache.size() == 1); + EXPECT_TRUE(cache.hasKey(1)); + cache.insert(2, "Second inserted string"); + EXPECT_TRUE(cache.verifyInternals()); + EXPECT_TRUE(cache.size() == 2); + EXPECT_TRUE(cache.hasKey(1)); + EXPECT_TRUE(cache.hasKey(2)); + cache.insert(3, "Third inserted string"); + EXPECT_TRUE(cache.verifyInternals()); + EXPECT_TRUE(cache.size() == 3); + EXPECT_TRUE(cache.hasKey(1)); + EXPECT_TRUE(cache.hasKey(2)); + EXPECT_TRUE(cache.hasKey(3)); + cache.insert(4, "Fourth inserted string"); + EXPECT_TRUE(cache.verifyInternals()); + EXPECT_TRUE(cache.size() == 4); + EXPECT_TRUE(cache.hasKey(1)); + EXPECT_TRUE(cache.hasKey(2)); + EXPECT_TRUE(cache.hasKey(3)); + EXPECT_TRUE(cache.hasKey(4)); + cache.insert(5, "Fifth inserted string"); + EXPECT_TRUE(cache.verifyInternals()); + EXPECT_TRUE(cache.size() == 5); + EXPECT_TRUE(cache.hasKey(1)); + EXPECT_TRUE(cache.hasKey(2)); + EXPECT_TRUE(cache.hasKey(3)); + EXPECT_TRUE(cache.hasKey(4)); + EXPECT_TRUE(cache.hasKey(5)); + cache.insert(6, "Sixt inserted string"); + EXPECT_TRUE(cache.verifyInternals()); + EXPECT_TRUE(cache.size() == 6); + EXPECT_TRUE(cache.hasKey(1)); + EXPECT_TRUE(cache.hasKey(2)); + EXPECT_TRUE(cache.hasKey(3)); + EXPECT_TRUE(cache.hasKey(4)); + EXPECT_TRUE(cache.hasKey(5)); + EXPECT_TRUE(cache.hasKey(6)); + cache.insert(7, "Seventh inserted string"); + EXPECT_TRUE(cache.verifyInternals()); + EXPECT_EQUAL(cache.size(), 7u); + EXPECT_TRUE(cache.hasKey(1)); + EXPECT_TRUE(cache.hasKey(2)); + EXPECT_TRUE(cache.hasKey(3)); + EXPECT_TRUE(cache.hasKey(4)); + EXPECT_TRUE(cache.hasKey(5)); + EXPECT_TRUE(cache.hasKey(6)); + EXPECT_TRUE(cache.hasKey(7)); + cache.insert(8, "Eighth inserted string"); + EXPECT_TRUE(cache.verifyInternals()); + EXPECT_EQUAL(cache.size(), 7u); + EXPECT_TRUE(cache.hasKey(2)); + EXPECT_TRUE(cache.hasKey(3)); + EXPECT_TRUE(cache.hasKey(4)); + EXPECT_TRUE(cache.hasKey(5)); + EXPECT_TRUE(cache.hasKey(6)); + EXPECT_TRUE(cache.hasKey(7)); + EXPECT_TRUE(cache.hasKey(8)); + cache.insert(15, "Eighth inserted string"); + EXPECT_TRUE(cache.verifyInternals()); + EXPECT_EQUAL(cache.size(), 7u); + EXPECT_TRUE(cache.hasKey(3)); + EXPECT_TRUE(cache.hasKey(4)); + EXPECT_TRUE(cache.hasKey(5)); + EXPECT_TRUE(cache.hasKey(6)); + EXPECT_TRUE(cache.hasKey(7)); + EXPECT_TRUE(cache.hasKey(8)); + EXPECT_TRUE(cache.hasKey(15)); + // Test get and erase + cache.get(3); + EXPECT_TRUE(cache.verifyInternals()); + cache.erase(3); + EXPECT_TRUE(cache.verifyInternals()); + EXPECT_TRUE(!cache.hasKey(3)); +} + +typedef std::shared_ptr<std::string> MyKey; +typedef std::shared_ptr<std::string> MyData; + +struct SharedEqual { + bool operator()(const MyKey & a, const MyKey & b) { + return ((*a) == (*b)); + } +}; + +struct SharedHash { + size_t operator() (const MyKey & arg) const { return arg->size(); } +}; + + +TEST("testCacheInsertOverResize") { + using LS = std::shared_ptr<std::string>; + using Cache = lrucache_map< LruParam<int, LS> >; + + Cache cache(100); + size_t sum(0); + for (size_t i(0); i < cache.capacity()*10; i++) { + LS s(new std::string("abc")); + cache[random()] = s; + sum += strlen(s->c_str()); + EXPECT_EQUAL(strlen(s->c_str()), s->size()); + } + EXPECT_EQUAL(sum, cache.capacity()*10*3); +} + +TEST("testCacheErase") { + lrucache_map< LruParam<MyKey, MyData, SharedHash, SharedEqual> > cache(4); + + MyData d(new std::string("foo")); + MyKey k(new std::string("barlol")); + // Verfify start conditions. + EXPECT_TRUE(cache.size() == 0); + EXPECT_TRUE(d.use_count() == 1); + EXPECT_TRUE(k.use_count() == 1); + cache.insert(k, d); + EXPECT_TRUE(d.use_count() == 2); + EXPECT_TRUE(k.use_count() == 2); + cache.erase(k); + EXPECT_TRUE(d.use_count() == 1); + EXPECT_TRUE(k.use_count() == 1); +} + +TEST("testCacheIterator") { + typedef lrucache_map< LruParam<int, string> > Cache; + Cache cache(3); + cache.insert(1, "first"); + cache.insert(2, "second"); + cache.insert(3, "third"); + Cache::iterator it(cache.begin()); + Cache::iterator mt(cache.end()); + ASSERT_TRUE(it != mt); + ASSERT_EQUAL("third", *it); + ASSERT_TRUE(it != mt); + ASSERT_EQUAL("second", *(++it)); + ASSERT_TRUE(it != mt); + ASSERT_EQUAL("second", *it++); + ASSERT_TRUE(it != mt); + ASSERT_EQUAL("first", *it); + ASSERT_TRUE(it != mt); + it++; + ASSERT_TRUE(it == mt); + cache.insert(4, "fourth"); + Cache::iterator it2(cache.begin()); + Cache::iterator it3(cache.begin()); + ASSERT_EQUAL("fourth", *it2); + ASSERT_TRUE(it2 == it3); + it2++; + ASSERT_TRUE(it2 != it3); + it2++; + it2++; + ASSERT_TRUE(it2 == mt); + Cache::iterator it4 = cache.erase(it3); + ASSERT_EQUAL("third", *it4); + ASSERT_EQUAL("third", *cache.begin()); + Cache::iterator it5(cache.erase(cache.end())); + ASSERT_TRUE(it5 == cache.end()); +} + +TEST("testCacheIteratorErase") { + typedef lrucache_map< LruParam<int, string> > Cache; + Cache cache(3); + cache.insert(1, "first"); + cache.insert(8, "second"); + cache.insert(15, "third"); + cache.insert(15, "third"); + cache.insert(8, "second"); + cache.insert(1, "first"); + Cache::iterator it(cache.begin()); + ASSERT_EQUAL("first", *it); + it++; + ASSERT_EQUAL("second", *it); + it = cache.erase(it); + ASSERT_EQUAL("third", *it); + cache.erase(it); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/util/process_memory_stats/.gitignore b/vespalib/src/tests/util/process_memory_stats/.gitignore new file mode 100644 index 00000000000..81af04ee64f --- /dev/null +++ b/vespalib/src/tests/util/process_memory_stats/.gitignore @@ -0,0 +1,2 @@ +mapfile +vespalib_process_memory_stats_test_app diff --git a/vespalib/src/tests/util/process_memory_stats/CMakeLists.txt b/vespalib/src/tests/util/process_memory_stats/CMakeLists.txt new file mode 100644 index 00000000000..30a0f90d952 --- /dev/null +++ b/vespalib/src/tests/util/process_memory_stats/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_process_memory_stats_test_app TEST + SOURCES + process_memory_stats_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_process_memory_stats_test_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/process_memory_stats_test.sh + DEPENDS vespalib_process_memory_stats_test_app) diff --git a/vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.cpp b/vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.cpp new file mode 100644 index 00000000000..6d0917e6d15 --- /dev/null +++ b/vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.cpp @@ -0,0 +1,92 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/util/process_memory_stats.h> +#include <vespa/vespalib/util/size_literals.h> +#include <iostream> +#include <fstream> +#include <sys/mman.h> +#include <fcntl.h> +#include <unistd.h> + + +using namespace vespalib; + +namespace { + +constexpr uint64_t SIZE_EPSILON = 4095; + +std::string toString(const ProcessMemoryStats &stats) +{ + std::ostringstream os; + os << "Mapped(" + << stats.getMappedVirt() << "," << stats.getMappedRss() << + "), Anonymous(" + << stats.getAnonymousVirt() << "," << stats.getAnonymousRss() << ")"; + return os.str(); +} + +} + +TEST("Simple stats") +{ + ProcessMemoryStats stats(ProcessMemoryStats::create(SIZE_EPSILON)); + std::cout << toString(stats) << std::endl; + EXPECT_LESS(0u, stats.getMappedVirt()); + EXPECT_LESS(0u, stats.getMappedRss()); + EXPECT_LESS(0u, stats.getAnonymousVirt()); + EXPECT_LESS(0u, stats.getAnonymousRss()); +} + +TEST("grow anonymous memory") +{ + ProcessMemoryStats stats1(ProcessMemoryStats::create(SIZE_EPSILON)); + std::cout << toString(stats1) << std::endl; + size_t mapLen = 64_Ki; + void *mapAddr = mmap(nullptr, mapLen, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + EXPECT_NOT_EQUAL(reinterpret_cast<void *>(-1), mapAddr); + ProcessMemoryStats stats2(ProcessMemoryStats::create()); + std::cout << toString(stats2) << std::endl; + EXPECT_LESS_EQUAL(stats1.getAnonymousVirt() + mapLen, + stats2.getAnonymousVirt()); + memset(mapAddr, 1, mapLen); + ProcessMemoryStats stats3(ProcessMemoryStats::create(SIZE_EPSILON)); + std::cout << toString(stats3) << std::endl; + // Cannot check that resident grows if swap is enabled and system loaded + munmap(mapAddr, mapLen); +} + +TEST("grow mapped memory") +{ + std::ofstream of("mapfile"); + size_t mapLen = 64_Ki; + std::vector<char> buf(mapLen, 4); + of.write(&buf[0], buf.size()); + of.close(); + int mapfileFileDescriptor = open("mapfile", O_RDONLY, 0666); + EXPECT_LESS_EQUAL(0, mapfileFileDescriptor); + ProcessMemoryStats stats1(ProcessMemoryStats::create(SIZE_EPSILON)); + std::cout << toString(stats1) << std::endl; + void *mapAddr = mmap(nullptr, mapLen, PROT_READ, MAP_SHARED, + mapfileFileDescriptor, 0); + EXPECT_NOT_EQUAL(reinterpret_cast<void *>(-1), mapAddr); + ProcessMemoryStats stats2(ProcessMemoryStats::create(SIZE_EPSILON)); + std::cout << toString(stats2) << std::endl; + EXPECT_LESS_EQUAL(stats1.getMappedVirt() + mapLen, stats2.getMappedVirt()); + EXPECT_EQUAL(0, memcmp(mapAddr, &buf[0], mapLen)); + ProcessMemoryStats stats3(ProcessMemoryStats::create(SIZE_EPSILON)); + std::cout << toString(stats3) << std::endl; + // Cannot check that resident grows if swap is enabled and system loaded + munmap(mapAddr, mapLen); +} + +TEST("order samples") +{ + ProcessMemoryStats a(0,0,0,7,0); + ProcessMemoryStats b(0,0,0,8,0); + EXPECT_TRUE(a < b); + EXPECT_FALSE(b < a); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.sh b/vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.sh new file mode 100755 index 00000000000..7fe5261ab2d --- /dev/null +++ b/vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +set -e +rm -f mapfile +$VALGRIND ./vespalib_process_memory_stats_test_app +rm -f mapfile diff --git a/vespalib/src/tests/xmlserializable/.gitignore b/vespalib/src/tests/xmlserializable/.gitignore new file mode 100644 index 00000000000..8573da1cd93 --- /dev/null +++ b/vespalib/src/tests/xmlserializable/.gitignore @@ -0,0 +1,4 @@ +*_test +.depend +Makefile +vespalib_xmlserializable_test_app diff --git a/vespalib/src/tests/xmlserializable/CMakeLists.txt b/vespalib/src/tests/xmlserializable/CMakeLists.txt new file mode 100644 index 00000000000..119a1294253 --- /dev/null +++ b/vespalib/src/tests/xmlserializable/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_xmlserializable_test_app TEST + SOURCES + xmlserializabletest.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_xmlserializable_test_app COMMAND vespalib_xmlserializable_test_app) diff --git a/vespalib/src/tests/xmlserializable/xmlserializabletest.cpp b/vespalib/src/tests/xmlserializable/xmlserializabletest.cpp new file mode 100644 index 00000000000..cc8d61cb7c2 --- /dev/null +++ b/vespalib/src/tests/xmlserializable/xmlserializabletest.cpp @@ -0,0 +1,163 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/xmlstream.h> + +namespace vespalib { + +class Test : public vespalib::TestApp +{ +public: + void testNormalUsage(); + void testEscaping(); + void testNesting(); + void testIndent(); + + int Main() override; +}; + +int +Test::Main() +{ + TEST_INIT("xmlserializables_test"); + srandom(1); + testNormalUsage(); + testEscaping(); + testNesting(); + testIndent(); + TEST_DONE(); +} + +void +Test::testNormalUsage() +{ + std::ostringstream ost; + XmlOutputStream xos(ost); + using namespace vespalib::xml; + xos << XmlTag("car") + << XmlTag("door") + << XmlAttribute("windowstate", "up") + << XmlEndTag() + << XmlTag("description") + << "This is a car description used to test" + << XmlEndTag() + << XmlEndTag(); + std::string expected = + "<car>\n" + "<door windowstate=\"up\"/>\n" + "<description>This is a car description used to test</description>\n" + "</car>"; + EXPECT_EQUAL(expected, ost.str()); +} + +void +Test::testEscaping() +{ + std::ostringstream ost; + XmlOutputStream xos(ost); + using namespace vespalib::xml; + xos << XmlTag("!#trash%-", XmlTagFlags::CONVERT_ILLEGAL_CHARACTERS) + << XmlTag("foo") + << XmlAttribute("bar", "<100%\" &\n>") + << XmlEndTag() + << XmlTag("escaped") + << XmlEscapedContent() + << XmlContentWrapper("<>&\"'% \r\n\t\f\0", 12) + << XmlEndTag() + << XmlTag("encoded") + << XmlBase64Content() + << XmlContentWrapper("<>&\"'% \t\f\0", 10) + << XmlEndTag() + << XmlTag("auto1") + << XmlContentWrapper("<>&\t\f\r\nfoo", 10) + << XmlEndTag() + << XmlTag("auto2") + << XmlContentWrapper("<>&\t\0\r\nfoo", 10) + << XmlEndTag() + << XmlEndTag(); + std::string expected = + "<__trash_->\n" + "<foo bar=\"<100%" & >\"/>\n" + "<escaped><>&\"'% \n	�</escaped>\n" + "<encoded binaryencoding=\"base64\">PD4mIiclIAkMAA==</encoded>\n" + "<auto1><>&	 \nfoo</auto1>\n" + "<auto2 binaryencoding=\"base64\">PD4mCQANCmZvbw==</auto2>\n" + "</__trash_->"; + EXPECT_EQUAL(expected, ost.str()); +} + +namespace { + struct LookAndFeel : public XmlSerializable { + + LookAndFeel() {} + + void printXml(XmlOutputStream& out) const override { + using namespace vespalib::xml; + out << XmlAttribute("color", "blue") + << XmlTag("other") + << XmlAttribute("count", 5) + << XmlTag("something") << "foo" << XmlEndTag() + << XmlTag("else") << "bar" << XmlEndTag() + << XmlEndTag(); + } + }; +} + +void +Test::testNesting() +{ + std::ostringstream ost; + XmlOutputStream xos(ost); + using namespace vespalib::xml; + xos << XmlTag("car") + << XmlTag("door") + << LookAndFeel() + << XmlEndTag() + << XmlTag("description") + << "This is a car description used to test" + << XmlEndTag() + << XmlEndTag(); + std::string expected = + "<car>\n" + "<door color=\"blue\">\n" + "<other count=\"5\">\n" + "<something>foo</something>\n" + "<else>bar</else>\n" + "</other>\n" + "</door>\n" + "<description>This is a car description used to test</description>\n" + "</car>"; + EXPECT_EQUAL(expected, ost.str()); +} + +void +Test::testIndent() +{ + std::ostringstream ost; + XmlOutputStream xos(ost, " "); + using namespace vespalib::xml; + xos << XmlTag("foo") + << XmlTag("bar") << 2.14 << XmlEndTag() + << "Litt innhold" + << XmlTag("nytag") + << "Mer innhold" + << XmlTag("base") + << XmlBase64Content() << "foobar" + << XmlEndTag() + << XmlEndTag() + << XmlEndTag(); + std::string expected = + "<foo>\n" + " <bar>2.14</bar>\n" + " Litt innhold\n" + " <nytag>\n" + " Mer innhold\n" + " <base binaryencoding=\"base64\">Zm9vYmFy</base>\n" + " </nytag>\n" + "</foo>"; + EXPECT_EQUAL(expected, ost.str()); +} + +} // vespalib + +TEST_APPHOOK(vespalib::Test) diff --git a/vespalib/src/vespa/vespalib/objects/CMakeLists.txt b/vespalib/src/vespa/vespalib/objects/CMakeLists.txt index a4b2192d98f..ddf01d5a935 100644 --- a/vespalib/src/vespa/vespalib/objects/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/objects/CMakeLists.txt @@ -1,7 +1,19 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(vespalib_vespalib_objects OBJECT SOURCES - nbostream.cpp + asciiserializer.cpp + deserializer.cpp + floatingpointtype.cpp hexdump.cpp + identifiable.cpp + nboserializer.cpp + nbostream.cpp + object2slime.cpp + objectdumper.cpp + objectoperation.cpp + objectpredicate.cpp + objectvisitor.cpp + serializer.cpp + visit.cpp DEPENDS ) diff --git a/vespalib/src/vespa/vespalib/objects/asciiserializer.cpp b/vespalib/src/vespa/vespalib/objects/asciiserializer.cpp new file mode 100644 index 00000000000..fadaea9b054 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/asciiserializer.cpp @@ -0,0 +1,47 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "asciiserializer.h" +#include <vespa/vespalib/stllike/asciistream.h> + +namespace vespalib { + +AsciiSerializer &AsciiSerializer::put(bool value) { + _stream << value; + return *this; +} + +AsciiSerializer &AsciiSerializer::put(uint8_t value) { + _stream << value; + return *this; +} + +AsciiSerializer &AsciiSerializer::put(uint16_t value) { + _stream << value; + return *this; +} + +AsciiSerializer &AsciiSerializer::put(uint32_t value) { + _stream << value; + return *this; +} + +AsciiSerializer &AsciiSerializer::put(uint64_t value) { + _stream << value; + return *this; +} + +AsciiSerializer &AsciiSerializer::put(float value) { + _stream << value; + return *this; +} + +AsciiSerializer &AsciiSerializer::put(double value) { + _stream << value; + return *this; +} + +AsciiSerializer &AsciiSerializer::put(stringref value) { + _stream << value; + return *this; +} + +} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/objects/asciiserializer.h b/vespalib/src/vespa/vespalib/objects/asciiserializer.h new file mode 100644 index 00000000000..3964c52949d --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/asciiserializer.h @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "serializer.h" +#include "deserializer.h" + +namespace vespalib { + +class asciistream; + +class AsciiSerializer : public Serializer { +public: + AsciiSerializer(asciistream &stream) : _stream(stream) { } + AsciiSerializer &put(bool value) override; + AsciiSerializer &put(uint8_t value) override; + AsciiSerializer &put(uint16_t value) override; + AsciiSerializer &put(uint32_t value) override; + AsciiSerializer &put(uint64_t value) override; + AsciiSerializer &put(float value) override; + AsciiSerializer &put(double value) override; + AsciiSerializer &put(stringref val) override; + + const asciistream &getStream() const { return _stream; } + asciistream &getStream() { return _stream; } +private: + asciistream &_stream; +}; + +} // namespace vespalib + diff --git a/vespalib/src/vespa/vespalib/objects/deserializer.cpp b/vespalib/src/vespa/vespalib/objects/deserializer.cpp new file mode 100644 index 00000000000..f320b2d2489 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/deserializer.cpp @@ -0,0 +1,46 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "deserializer.h" +#include "identifiable.h" + +namespace vespalib { + +Deserializer & Deserializer::get(Identifiable & value) +{ + return value.deserializeDirect(*this); +} +Deserializer & Deserializer::get(int8_t & value) +{ + uint8_t v(0); + get(v); + value = v; + return *this; +} +Deserializer & Deserializer::get(int16_t & value) +{ + uint16_t v(0); + get(v); + value = v; + return *this; +} +Deserializer & Deserializer::get(int32_t & value) +{ + uint32_t v(0); + get(v); + value = v; + return *this; +} +Deserializer & Deserializer::get(int64_t & value) +{ + uint64_t v(0); + get(v); + value = v; + return *this; +} +Deserializer & Deserializer::get(std::string & value) +{ + string v; + get(v); + value = v; + return *this; +} +} diff --git a/vespalib/src/vespa/vespalib/objects/deserializer.h b/vespalib/src/vespa/vespalib/objects/deserializer.h new file mode 100644 index 00000000000..e59d3e07581 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/deserializer.h @@ -0,0 +1,54 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/util/array.h> +#include <vespa/vespalib/stllike/string.h> +#include <vector> +#include <cstdint> + +namespace vespalib { + +class Identifiable; + +class Deserializer +{ +public: + virtual ~Deserializer() = default; + virtual Deserializer & get(bool & value) = 0; + virtual Deserializer & get(uint8_t & value) = 0; + virtual Deserializer & get(uint16_t & value) = 0; + virtual Deserializer & get(uint32_t & value) = 0; + virtual Deserializer & get(uint64_t & value) = 0; + virtual Deserializer & get(double & value) = 0; + virtual Deserializer & get(float & value) = 0; + virtual Deserializer & get(string & value) = 0; + + virtual Deserializer & get(Identifiable & value); + virtual Deserializer & get(int8_t & value); + virtual Deserializer & get(int16_t & value); + virtual Deserializer & get(int32_t & value); + virtual Deserializer & get(int64_t & value); + + + Deserializer & get(std::string & value); + Deserializer & operator >> (bool & value) { return get(value); } + Deserializer & operator >> (uint8_t & value) { return get(value); } + Deserializer & operator >> (int8_t & value) { return get(value); } + Deserializer & operator >> (uint16_t & value) { return get(value); } + Deserializer & operator >> (int16_t & value) { return get(value); } + Deserializer & operator >> (uint32_t & value) { return get(value); } + Deserializer & operator >> (int32_t & value) { return get(value); } + Deserializer & operator >> (uint64_t & value) { return get(value); } + Deserializer & operator >> (int64_t & value) { return get(value); } + Deserializer & operator >> (float & value) { return get(value); } + Deserializer & operator >> (double & value) { return get(value); } + Deserializer & operator >> (string & value) { return get(value); } + template <typename T> + Deserializer & operator >> (vespalib::Array<T> & v); + template <typename T> + Deserializer & operator >> (std::vector<T> & v); + +}; + +} + diff --git a/vespalib/src/vespa/vespalib/objects/deserializer.hpp b/vespalib/src/vespa/vespalib/objects/deserializer.hpp new file mode 100644 index 00000000000..b90867fa153 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/deserializer.hpp @@ -0,0 +1,33 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "deserializer.h" + +namespace vespalib { + +template <typename T> +Deserializer & +Deserializer::operator >> (vespalib::Array<T> & v) { + uint32_t sz; + get(sz); + v.resize(sz); + for(size_t i(0); i < sz; i++) { + (*this) >> v[i]; + } + return *this; +} + +template <typename T> +Deserializer & +Deserializer::operator >> (std::vector<T> & v) { + uint32_t sz; + get(sz); + v.resize(sz); + for(size_t i(0); i < sz; i++) { + (*this) >> v[i]; + } + return *this; +} + +} + diff --git a/vespalib/src/vespa/vespalib/objects/floatingpointtype.cpp b/vespalib/src/vespa/vespalib/objects/floatingpointtype.cpp new file mode 100644 index 00000000000..b011dbdd23e --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/floatingpointtype.cpp @@ -0,0 +1,26 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/objects/floatingpointtype.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <ostream> + +namespace vespalib { + +template<typename Number> +std::ostream& operator<<(std::ostream& out, FloatingPointType<Number> number) +{ + return out << number.getValue(); +} + +template<typename Number> +vespalib::asciistream & operator<<(vespalib::asciistream & out, FloatingPointType<Number> number) +{ + return out << number.getValue(); +} + +template std::ostream& operator<<(std::ostream& out, FloatingPointType<float> number); +template std::ostream& operator<<(std::ostream& out, FloatingPointType<double> number); + +template vespalib::asciistream& operator<<(vespalib::asciistream& out, FloatingPointType<float> number); +template vespalib::asciistream& operator<<(vespalib::asciistream& out, FloatingPointType<double> number); + +} diff --git a/vespalib/src/vespa/vespalib/objects/floatingpointtype.h b/vespalib/src/vespa/vespalib/objects/floatingpointtype.h new file mode 100644 index 00000000000..d1fd80cdcd6 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/floatingpointtype.h @@ -0,0 +1,79 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +/** + * \class vespalib::FloatingPointType + * \ingroup object + * + * \brief Wrapper class for floating point types taking care of comparisons. + * + * Due to floating point values not being able to represent a lot of numbers + * exactly, one should always compare floating point values, allowing slight + * variations. + * + * To avoid having to handle this everywhere in the code, some wrapper classes + * exist here to take care of this for you. They will automatically convert + * primitive values to the wrapper class, so you can still use primitive + * constants in your code. + * + * Node that epsilon is currently just set to 10^(-6). We can reduce it or + * adjust it based on type or value later if we see the need. But the class + * should define it, at least by default, to make the interface easy to use as + * most use cases don't really care that much. + */ + +#pragma once + +#include <iosfwd> // To get std::ostream for output operator + +namespace vespalib { + +class asciistream; + +template<typename Number> +class FloatingPointType { + Number _value; + +public: + typedef FloatingPointType<Number> Type; + + FloatingPointType() : _value(0.0) {} + FloatingPointType(Number n) : _value(n) {} + + Number getValue() const { return _value; } + + Number abs() const { return (_value < 0 ? -1 * _value : _value); } + + bool operator==(Type n) const { return ((*this - n).abs() < 0.000001); } + bool operator!=(Type n) const { return ((*this - n).abs() > 0.000001); } + bool operator<(Type n) const { return (n._value - 0.000001 > _value); } + bool operator>(Type n) const { return (n._value + 0.000001 < _value); } + bool operator<=(Type n) const { return (n._value + 0.000001 > _value); } + bool operator>=(Type n) const { return (n._value - 0.000001 < _value); } + + Type operator-(Type n) const { return Type(_value - n._value); } + Type operator+(Type n) const { return Type(_value + n._value); } + Type operator*(Type n) const { return Type(_value * n._value); } + Type operator/(Type n) const { return Type(_value / n._value); } + + Type& operator+=(Type n) { _value += n._value; return *this; } + Type& operator-=(Type n) { _value -= n._value; return *this; } + Type& operator*=(Type n) { _value *= n._value; return *this; } + Type& operator/=(Type n) { _value /= n._value; return *this; } + + Type& operator++() { ++_value; return *this; } + Type operator++(int) { Type t(_value); ++_value; return t; } + Type& operator--() { --_value; return *this; } + Type operator--(int) { Type t(_value); --_value; return t; } +}; + +typedef FloatingPointType<double> Double; +typedef FloatingPointType<double> Float; + +template<typename Number> +std::ostream& operator<<(std::ostream& out, FloatingPointType<Number> number); + +template<typename Number> +vespalib::asciistream & operator<<(vespalib::asciistream & out, FloatingPointType<Number> number); + +} // vespalib + diff --git a/vespalib/src/vespa/vespalib/objects/identifiable.cpp b/vespalib/src/vespa/vespalib/objects/identifiable.cpp new file mode 100644 index 00000000000..224fda1a0a4 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/identifiable.cpp @@ -0,0 +1,294 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "identifiable.hpp" +#include <cassert> +#include <vespa/vespalib/util/stringfmt.h> +#include <stdexcept> +#include <algorithm> +#include <vespa/vespalib/objects/nbostream.h> +#include "objectdumper.h" +#include "visit.h" +#include "objectpredicate.h" +#include "objectoperation.h" +#include <vespa/vespalib/util/classname.h> +#include <vespa/vespalib/stllike/hash_set.hpp> + + +namespace vespalib { + +namespace { + +class Register { +public: + using RuntimeClass = Identifiable::RuntimeClass; + Register(); + ~Register(); + bool append(RuntimeClass * c); + bool erase(RuntimeClass * c); + const RuntimeClass * classFromId(unsigned id) const; + const RuntimeClass * classFromName(const char * name) const; + bool empty() const { return _listById.empty(); } +private: + struct HashId { + uint32_t operator() (const RuntimeClass * f) const { return f->id(); } + uint32_t operator() (uint32_t id) const { return id; } + }; + struct EqualId { + bool operator() (const RuntimeClass * a, const RuntimeClass * b) const { return a->id() == b->id(); } + bool operator() (const RuntimeClass * a, uint32_t b) const { return a->id() == b; } + bool operator() (uint32_t a, const RuntimeClass * b) const { return a == b->id(); } + }; + struct HashName { + uint32_t operator() (const RuntimeClass * f) const { return hashValue(f->name()); } + uint32_t operator() (const char * name) const { return hashValue(name); } + }; + struct EqualName { + bool operator() (const RuntimeClass * a, const RuntimeClass * b) const { return strcmp(a->name(), b->name()) == 0; } + bool operator() (const RuntimeClass * a, const char * b) const { return strcmp(a->name(), b) == 0; } + bool operator() (const char * a, const RuntimeClass * b) const { return strcmp(a, b->name()) == 0; } + }; + using IdList = hash_set<RuntimeClass *, HashId, EqualId>; + using NameList = hash_set<RuntimeClass *, HashName, EqualName>; + IdList _listById; + NameList _listByName; +}; + +Register::Register() : + _listById(), + _listByName() +{ } + +Register::~Register() = default; + +bool Register::erase(Identifiable::RuntimeClass * c) +{ + _listById.erase(c); + _listByName.erase(c); + return true; +} + +bool Register::append(Identifiable::RuntimeClass * c) +{ + bool ok((_listById.find(c) == _listById.end()) && ((_listByName.find(c) == _listByName.end()))); + if (ok) { + _listById.insert(c); + _listByName.insert(c); + } + return ok; +} + +const Identifiable::RuntimeClass * Register::classFromId(unsigned id) const +{ + IdList::const_iterator it(_listById.find<uint32_t>(id)); + return (it != _listById.end()) ? *it : nullptr; +} + +const Identifiable::RuntimeClass * Register::classFromName(const char *name) const +{ + NameList::const_iterator it(_listByName.find<const char *>(name)); + return (it != _listByName.end()) ? *it : nullptr; +} + +Register * _register = nullptr; + +} + +Identifiable::ILoader * Identifiable::_classLoader = nullptr; + +IMPLEMENT_IDENTIFIABLE(Identifiable, Identifiable); + +const Identifiable::RuntimeClass * +Identifiable::classFromId(unsigned id) { + return _register->classFromId(id); +} + +const Identifiable::RuntimeClass * +Identifiable::classFromName(const char * name) { + return _register->classFromName(name); +} + +Identifiable::RuntimeClass::RuntimeClass(RuntimeInfo * info_) : + _rt(info_) +{ + if (_rt->_factory) { + Identifiable::UP tmp(create()); + Identifiable &tmpref = *tmp; + assert(id() == tmp->getClass().id()); + //printf("Class %s has typeinfo %s\n", name(), typeid(*tmp).name()); + for (const RuntimeInfo * curr = _rt; curr && curr != curr->_base; curr = curr->_base) { + //printf("\tinherits %s : typeinfo = %s\n", curr->_name, curr->_typeId().name()); + if ( ! curr->_tryCast(tmp.get()) ) { + throw std::runtime_error(make_string("(%s, %s) is not a baseclass of (%s, %s)", curr->_name, curr->_typeId().name(), name(), typeid(tmpref).name())); + } + } + } + if (_register == nullptr) { + _register = new Register(); + } + if (! _register->append(this)) { + const RuntimeClass * old = _register->classFromId(id()); + throw std::runtime_error(make_string("Duplicate Identifiable object(%s, %s, %d) being registered. Choose a unique id. Object (%s, %s, %d) is using it.", name(), info(), id(), old->name(), old->info(), old->id())); + } +} + +Identifiable::RuntimeClass::~RuntimeClass() +{ + if ( ! _register->erase(this) ) { + assert(0); + } + if (_register->empty()) { + delete _register; + _register = nullptr; + } +} + +bool Identifiable::RuntimeClass::inherits(unsigned cid) const +{ + const RuntimeInfo *cur; + for (cur = _rt; (cur != &Identifiable::_RTInfo) && cid != cur->_id; cur = cur->_base) { } + return (cid == cur->_id); +} + +Serializer & operator << (Serializer & os, const Identifiable & obj) +{ + os.put(obj.getClass().id()); + obj.serialize(os); + return os; +} + +nbostream & operator << (nbostream & os, const Identifiable & obj) +{ + NBOSerializer nos(os); + nos << obj; + return os; +} + +nbostream & operator >> (nbostream & is, Identifiable & obj) +{ + NBOSerializer nis(is); + nis >> obj; + return is; +} + +Deserializer & operator >> (Deserializer & os, Identifiable & obj) +{ + uint32_t cid(0); + os.get(cid); + if (cid == obj.getClass().id()) { + obj.deserialize(os); + } else { + throw std::runtime_error(make_string("Failed deserializing %s : Received cid %d(%0x) != %d(%0x)", + obj.getClass().name(), + cid, cid, + obj.getClass().id(), obj.getClass().id())); + //Should mark as failed + } + return os; +} + +Identifiable::UP Identifiable::create(Deserializer & is) +{ + uint32_t cid(0); + is.get(cid); + UP obj; + const Identifiable::RuntimeClass *rtc = Identifiable::classFromId(cid); + if (rtc == nullptr) { + if ((_classLoader != nullptr) && _classLoader->hasClass(cid)) { + _classLoader->loadClass(cid); + rtc = Identifiable::classFromId(cid); + if (rtc == nullptr) { + throw std::runtime_error(make_string("Failed loading class for Identifiable with classId %d(%0x)", cid, cid)); + } + } + } + if (rtc != nullptr) { + obj.reset(rtc->create()); + if (obj.get()) { + obj->deserialize(is); + } else { + throw std::runtime_error( + make_string("Failed deserializing an Identifiable for classId %d(%0x). " + "It is abstract, so it can not be instantiated. Does it need to be abstract ?", + cid, cid)); + } + } else { + throw std::runtime_error(make_string("Failed deserializing an Identifiable with unknown classId %d(%0x)", cid, cid)); + } + return obj; +} + +string +Identifiable::getNativeClassName() const +{ + return vespalib::getClassName(*this); +} + +string +Identifiable::asString() const +{ + ObjectDumper dumper; + visit(dumper, "", this); + return dumper.toString(); +} + +int Identifiable::onCmp(const Identifiable& b) const +{ + int diff(0); + nbostream as, bs; + NBOSerializer nas(as), nbs(bs); + nas << *this; + nbs << b; + size_t minLength(std::min(as.size(), bs.size())); + if (minLength > 0) { + diff = memcmp(as.data(), bs.data(), minLength); + } + if (diff == 0) { + diff = as.size() - bs.size(); + } + return diff; +} + +void +Identifiable::visitMembers(ObjectVisitor &visitor) const +{ + visitor.visitNotImplemented(); +} + +void +Identifiable::select(const ObjectPredicate &predicate, ObjectOperation &operation) +{ + if (predicate.check(*this)) { + operation.execute(*this); + } else { + selectMembers(predicate, operation); + } +} + +void +Identifiable::selectMembers(const ObjectPredicate &predicate, ObjectOperation &operation) +{ + (void) predicate; + (void) operation; +} + +Serializer & Identifiable::serialize(Serializer & os) const +{ + return os.put(*this); +} + +Deserializer & Identifiable::deserialize(Deserializer & is) +{ + return is.get(*this); +} + +Serializer & Identifiable::onSerialize(Serializer & os) const +{ + return os; +} + +Deserializer & Identifiable::onDeserialize(Deserializer & is) +{ + return is; +} + +} diff --git a/vespalib/src/vespa/vespalib/objects/identifiable.h b/vespalib/src/vespa/vespalib/objects/identifiable.h new file mode 100644 index 00000000000..586fbc7ed0c --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/identifiable.h @@ -0,0 +1,423 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +/** + * \class vespalib::Identifiable + * \ingroup util + * + * \brief Superclass for objects adding some runtime type information. + * + * This class is a superclass used by many other classes to add some runtime + * information. + * + * This can be used to verify type to be able to do cheap static casts + * instead of dynamic casts. It can be used to identify type of object, and + * it can also be used to generate an object of the given type (If it's not + * an identifiable abstract type). + * + */ + +#define CID_Identifiable 1 + +#include "ids.h" +#include "nboserializer.h" +#include "objectvisitor.h" + +#include <vespa/vespalib/util/memory.h> + +#define IDENTIFIABLE_CLASSID(cclass) CID_##cclass +#define IDENTIFIABLE_CLASSID_NS(ns, cclass) CID_##ns##_##cclass +#define IDENTIFIABLE_CLASSID_NS2(ns1, ns2, cclass) CID_##ns1##_##ns2##_##cclass +#define IDENTIFIABLE_CLASSID_NS3(ns1, ns2, ns3, cclass) CID_##ns1##_##ns2##_##ns3##_##cclass + +#define DECLARE_IDENTIFIABLE_STATIC_BASE_COMMON(cclass) \ + static vespalib::Identifiable::RuntimeInfo _RTInfo; \ + static vespalib::Identifiable::RuntimeClass _RTClass; \ + static const std::type_info & typeId() { return typeid(cclass); } \ + static bool tryCast(const vespalib::Identifiable * v) { return dynamic_cast<const cclass *>(v) != NULL; } \ + static cclass *identifyClassAsIdentifiable() { return NULL; } /* no implementation */ + +#define DECLARE_IDENTIFIABLE_BASE_COMMON(cclass) \ + DECLARE_IDENTIFIABLE_STATIC_BASE_COMMON(cclass) \ + const vespalib::Identifiable::RuntimeClass & getClass() const override; + +#define DECLARE_IDENTIFIABLE_BASE_COMMON_ROOT(cclass) \ + DECLARE_IDENTIFIABLE_STATIC_BASE_COMMON(cclass) \ + virtual const vespalib::Identifiable::RuntimeClass & getClass() const; + +#define DECLARE_IDENTIFIABLE_BASE(cclass, classid) \ + public: \ + enum { classId=classid }; \ + DECLARE_IDENTIFIABLE_BASE_COMMON(cclass) + +#define DECLARE_IDENTIFIABLE_BASE_ROOT(cclass, classid) \ + public: \ + enum { classId=classid }; \ + DECLARE_IDENTIFIABLE_BASE_COMMON_ROOT(cclass) + +#define DECLARE_IDENTIFIABLE_ABSTRACT(cclass) DECLARE_IDENTIFIABLE_BASE(cclass, IDENTIFIABLE_CLASSID(cclass)) +#define DECLARE_IDENTIFIABLE_ABSTRACT_NS(ns, cclass) DECLARE_IDENTIFIABLE_BASE(ns::cclass, IDENTIFIABLE_CLASSID_NS(ns, cclass)) +#define DECLARE_IDENTIFIABLE_ABSTRACT_NS2(ns1, ns2, cclass) DECLARE_IDENTIFIABLE_BASE(ns1::ns2::cclass, IDENTIFIABLE_CLASSID_NS2(ns1, ns2, cclass)) +#define DECLARE_IDENTIFIABLE_ABSTRACT_NS3(ns1, ns2, ns3, cclass) DECLARE_IDENTIFIABLE_BASE(ns1::ns2::ns3::cclass, IDENTIFIABLE_CLASSID_NS3(ns1, ns2, ns3, cclass)) + +#define DECLARE_IDENTIFIABLE_STATIC_COMMON(cclass) \ + static cclass * create() { return new cclass(); } \ + static Identifiable * createAsIdentifiable() { return cclass::create(); } + +#define DECLARE_IDENTIFIABLE_COMMON(cclass) \ + void assign(const vespalib::Identifiable & rhs) override; \ + DECLARE_IDENTIFIABLE_STATIC_COMMON(cclass) + +#define DECLARE_IDENTIFIABLE_COMMON_ROOT(cclass) \ + virtual void assign(const vespalib::Identifiable & rhs); \ + DECLARE_IDENTIFIABLE_STATIC_COMMON(cclass) + +#define DECLARE_IDENTIFIABLE(cclass) \ + DECLARE_IDENTIFIABLE_BASE(cclass, IDENTIFIABLE_CLASSID(cclass)) \ + DECLARE_IDENTIFIABLE_COMMON(cclass) + +#define DECLARE_IDENTIFIABLE_ROOT(cclass) \ + DECLARE_IDENTIFIABLE_BASE_ROOT(cclass, IDENTIFIABLE_CLASSID(cclass)) \ + DECLARE_IDENTIFIABLE_COMMON_ROOT(cclass) + +#define DECLARE_IDENTIFIABLE_NS(ns, cclass) \ + DECLARE_IDENTIFIABLE_BASE(ns::cclass, IDENTIFIABLE_CLASSID_NS(ns, cclass)) \ + DECLARE_IDENTIFIABLE_COMMON(ns::cclass) + +#define DECLARE_IDENTIFIABLE_NS2(ns1, ns2, cclass) \ + DECLARE_IDENTIFIABLE_BASE(ns1::ns2::cclass, IDENTIFIABLE_CLASSID_NS2(ns1, ns2, cclass)) \ + DECLARE_IDENTIFIABLE_COMMON(ns1::ns2::cclass) + +#define DECLARE_IDENTIFIABLE_NS3(ns1, ns2, ns3, cclass) \ + DECLARE_IDENTIFIABLE_BASE(ns1::ns2::ns3::cclass, IDENTIFIABLE_CLASSID_NS3(ns1, ns2, ns3, cclass)) \ + DECLARE_IDENTIFIABLE_COMMON(ns1::ns2::ns3::cclass) + +#define IMPLEMENT_IDENTIFIABLE_COMMON(cclass) \ + vespalib::Identifiable::RuntimeClass cclass::_RTClass(&_RTInfo); \ + const vespalib::Identifiable::RuntimeClass & cclass::getClass() const { return _RTClass; } + +#define IMPLEMENT_IDENTIFIABLE_CONCRET(cclass) \ + void cclass::assign(const vespalib::Identifiable & rhs) { \ + if (rhs.inherits(classId)) { \ + *this = static_cast<const cclass &>(rhs); \ + } \ + } + +#define IMPLEMENT_IDENTIFIABLE_BASE(cclass, name, base, id, factory, ctypeInfo, tryCast, typeinfo) \ + vespalib::Identifiable::RuntimeInfo cclass::_RTInfo = {name, typeinfo, id, factory, ctypeInfo, tryCast, &base::_RTInfo }; \ + IMPLEMENT_IDENTIFIABLE_COMMON(cclass) + +#define IMPLEMENT_IDENTIFIABLE_ABSTRACT(cclass, base) \ + IMPLEMENT_IDENTIFIABLE_BASE(cclass, #cclass, base, IDENTIFIABLE_CLASSID(cclass), \ + NULL, cclass::typeId, cclass::tryCast, "") +#define IMPLEMENT_IDENTIFIABLE(cclass, base) \ + IMPLEMENT_IDENTIFIABLE_CONCRET(cclass) \ + IMPLEMENT_IDENTIFIABLE_BASE(cclass, #cclass, base, IDENTIFIABLE_CLASSID(cclass), \ + cclass::createAsIdentifiable, cclass::typeId, cclass::tryCast, "") +#define IMPLEMENT_IDENTIFIABLE_ABSTRACT_NS(ns, cclass, base) \ + IMPLEMENT_IDENTIFIABLE_BASE(ns::cclass, #ns"::"#cclass, base, IDENTIFIABLE_CLASSID_NS(ns, cclass), \ + NULL, cclass::typeId, cclass::tryCast, "") +#define IMPLEMENT_IDENTIFIABLE_NS(ns, cclass, base) \ + IMPLEMENT_IDENTIFIABLE_CONCRET(ns::cclass) \ + IMPLEMENT_IDENTIFIABLE_BASE(ns::cclass, #ns"::"#cclass, base, IDENTIFIABLE_CLASSID_NS(ns, cclass), \ + cclass::createAsIdentifiable, cclass::typeId, cclass::tryCast, "") +#define IMPLEMENT_IDENTIFIABLE_ABSTRACT_NS2(ns1, ns2, cclass, base) \ + IMPLEMENT_IDENTIFIABLE_BASE(ns1::ns2::cclass, #ns1"::"#ns2"::"#cclass, base, IDENTIFIABLE_CLASSID_NS2(ns1, ns2, cclass), \ + NULL, cclass::typeId, cclass::tryCast, "") +#define IMPLEMENT_IDENTIFIABLE_NS2(ns1, ns2, cclass, base) \ + IMPLEMENT_IDENTIFIABLE_CONCRET(ns1::ns2::cclass) \ + IMPLEMENT_IDENTIFIABLE_BASE(ns1::ns2::cclass, #ns1"::"#ns2"::"#cclass, base, IDENTIFIABLE_CLASSID_NS2(ns1, ns2, cclass), \ + cclass::createAsIdentifiable, cclass::typeId, cclass::tryCast, "") +#define IMPLEMENT_IDENTIFIABLE_ABSTRACT_NS3(ns1, ns2, ns3, cclass, base) \ + IMPLEMENT_IDENTIFIABLE_BASE(ns1::ns2::ns3::cclass, #ns1"::"#ns2"::"#ns3"::"#cclass, base, IDENTIFIABLE_CLASSID_NS3(ns1, ns2, ns3, cclass), \ + NULL, cclass::typeId, cclass::tryCast, "") +#define IMPLEMENT_IDENTIFIABLE_NS3(ns1, ns2, ns3, cclass, base) \ + IMPLEMENT_IDENTIFIABLE_CONCRET(ns1::ns2::ns3::cclass) \ + IMPLEMENT_IDENTIFIABLE_BASE(ns1::ns2::ns3::cclass, #ns1"::"#ns2"::"#ns3"::"#cclass, base, IDENTIFIABLE_CLASSID_NS3(ns1, ns2, ns3, cclass), \ + cclass::createAsIdentifiable, cclass::typeId, cclass::tryCast, "") + +#define DECLARE_NBO_SERIALIZE \ + vespalib::Serializer & onSerialize(vespalib::Serializer & os) const override; \ + vespalib::Deserializer & onDeserialize(vespalib::Deserializer & is) override; + + +namespace vespalib { + +class ObjectPredicate; +class ObjectOperation; + +class Identifiable { + protected: + struct RuntimeInfo { + const char * _name; + const char * _info; + unsigned _id; + Identifiable * (* _factory)(); + const std::type_info & (* _typeId)(); + bool (* _tryCast)(const Identifiable *); + const RuntimeInfo * _base; + }; +public: + typedef std::unique_ptr<Identifiable> UP; + class ILoader + { + public: + virtual ~ILoader() { } + virtual bool hasClass(unsigned classId) const = 0; + virtual bool hasClass(const char * className) const = 0; + virtual void loadClass(unsigned classId) = 0; + virtual void loadClass(const char * className) = 0; + }; + struct RuntimeClass { + public: + RuntimeClass(RuntimeInfo * info); + ~RuntimeClass(); + const char * name() const { return _rt->_name; } + const char * info() const { return _rt->_info; } + unsigned id() const { return _rt->_id; } + Identifiable * create() const { return _rt->_factory ? _rt->_factory() : 0; } + const std::type_info & typeId() const { return _rt->_typeId(); } + bool tryCast(const Identifiable *o) const { return _rt->_tryCast(o); } + const RuntimeInfo * base() const { return _rt->_base; } + bool inherits(unsigned id) const; + bool equal(unsigned cid) const { return id() == cid; } + int compare(const RuntimeClass& other) const { return (id() - other.id()); } + private: + RuntimeInfo * _rt; + }; + DECLARE_IDENTIFIABLE_ROOT(Identifiable); + Identifiable() noexcept = default; + Identifiable(Identifiable &&) noexcept = default; + Identifiable & operator = (Identifiable &&) noexcept = default; + Identifiable(const Identifiable &) = default; + Identifiable & operator = (const Identifiable &) = default; + virtual ~Identifiable() noexcept = default; + + /** + * Will produce the full demangled className + */ + string getNativeClassName() const; + + /** + * This returns the innermost class that you represent. Default is that that is yourself. + * However when you are a vector containing other objects, it might be feasible + * to let the world know about them too. + * @return the class info for the innermost object. + */ + virtual const RuntimeClass & getBaseClass() const { return getClass(); } + /** + * Checks if this object inherits from a class with the given id. + * @param id The id of the class to check if is an anchestor. + * @return true if the object does inherit from it. Significantly faster than using dynamic cast. + */ + bool inherits(unsigned id) const { return getClass().inherits(id); } + /** + * Checks if this object inherits from a class with the given name. + * @param name The name of the class to check if is an anchestor. + * @return true if the object does inherit from it. Significantly faster than using dynamic cast. + */ + bool inherits(const char * name) const; + /** + * Identifiable::cast<T> behaves like dynamic_cast<T> when trying + * to cast between Identifiable objects, using the inherits() + * function defined above to check if the cast should succeed. + */ + template <typename T> struct BaseType { typedef T type; }; + template <typename T> struct BaseType<T &> { typedef T type; }; + template <typename T> struct BaseType<T *> { typedef T type; }; + template <typename T> struct BaseType<const T &> { typedef T type; }; + template <typename T> struct BaseType<const T *> { typedef T type; }; + template <typename Type> static void ERROR_Type_is_not_Identifiable() { + Type *(*foo)() = &Type::identifyClassAsIdentifiable; + (void) foo; + } + template <typename T, typename Base> + static T cast(Base &p) { + typedef typename BaseType<T>::type Type; + ERROR_Type_is_not_Identifiable<Type>(); // Help diagnose errors. + if (p.inherits(Type::classId)) { return static_cast<T>(p); } + else { throw std::bad_cast(); } + } + template <typename T, typename Base> + static T cast(Base *p) { + typedef typename BaseType<T>::type Type; + ERROR_Type_is_not_Identifiable<Type>(); // Help diagnose errors. + if (p && p->inherits(Type::classId)) { return static_cast<T>(p); } + else { return 0; } + } + /** + * Given the unique registered id of a class it will look up the object describing it. + * @return object describing the class. + */ + static const RuntimeClass * classFromId(unsigned id); + /** + * Given the unique registered name of a class it will look up the object describing it. + * @return object describing the class. + */ + static const RuntimeClass * classFromName(const char * name); + /** + * Here you can provide an optional classloader. + */ + static void registerClassLoader(ILoader & loader) { _classLoader = &loader; } + static void clearClassLoader() { _classLoader = NULL; } + + /** + * Create a human-readable representation of this object. This + * method will use object visitation internally to capture the + * full structure of this object. + * + * @return structured human-readable representation of this object + **/ + string asString() const; + + /** + * Visit each of the members of this object. This method should be + * overridden by subclasses and should present all appropriate + * internal structure of this object to the given visitor. Note + * that while each level of a class hierarchy may cooperate to + * visit all object members (invoking superclass method within + * method), this method, as implemented in the Identifiable class + * should not be invoked, since its default implementation is + * there to signal about the method not being overridden. + * + * @param visitor the visitor of this object + **/ + virtual void visitMembers(ObjectVisitor &visitor) const; + + /** + * Compares 2 objects. First tests classId, then lives it to the objects and onCmp + * if classes are the same. + * @param b the object to compare with. + * @return <0 if this comes before b, 0 for equality, and >0 fi b comes before *this. + */ + int cmp(const Identifiable& b) const { + int r(cmpClassId(b)); + return (r==0) ? onCmp(b) : r; + } + /** + * Compares 2 objects. This is faster method that cmp as classId tests is not done. + * onCmp is called directly. + * @param b the object to compare with. + * @return <0 if this comes before b, 0 for equality, and >0 fi b comes before *this. + */ + int cmpFast(const Identifiable& b) const { return onCmp(b); } + + /** + * Apply the predicate to this object. If the predicate returns + * true, pass this object to the operation, otherwise invoke the + * @ref selectMemebers method to locate sub-elements that might + * trigger the predicate. + * + * @param predicate component used to select (sub-)objects + * @param operation component performing some operation + * on the selected (sub-)objects + **/ + void select(const ObjectPredicate &predicate, ObjectOperation &operation); + + /** + * Invoke @ref select on any member objects this object wants to + * expose through the selection mechanism. Overriding this method + * is optional, and which objects to expose is determined by the + * application logic of the object itself. + * + * @param predicate component used to select (sub-)objects + * @param operation component performing some operation + * on the selected (sub-)objects + **/ + virtual void selectMembers(const ObjectPredicate &predicate, + ObjectOperation &operation); + + /** + * This will serialize the object by calling the virtual onSerialize. + * The id of the object is not serialized. + * @param os The nbostream used for output. + * @return The nbostream after serialization. + */ + Serializer & serialize(Serializer & os) const; + /** + * This will deserialize the object by calling the virtual onDeserialize. + * It is symetric with serialize. + * @param is The nbostream used for input. + * @return The nbostream after deserialization. + */ + Deserializer & deserialize(Deserializer & is); + /** + * This will read the first 4 bytes containing the classid. It will then create the + * correct object based on that class, and then call deserialize with the rest. + * It is symetric with the << operator, and does the same as >>, except instead of checking the id + * it uses it to construct an object. + * @param is The nbostream used for input. + * @return The object created and constructed by deserialize. NULL if there are any errors. + */ + static UP create(Deserializer & is); + static UP create(nbostream & is) { NBOSerializer nis(is); return create(nis); } + /** + * This will serialize the object by first serializing the 4 byte classid. + * Then the rest is serialized by calling serialize. + * @param os The nbostream used for output. + * @param obj The object to serialize. + * @return The nbostream after serialization. + */ + friend Serializer & operator << (Serializer & os, const Identifiable & obj); + friend nbostream & operator << (nbostream & os, const Identifiable & obj); + /** + * This will read the first 4 bytes containing the classid. It will then check if it matches its own. + * if not it will mark the nbostream as bad. Then it will deserialize he content. + * It is symetric with the << operator + * @param is The nbostream used for input. + * @param obj The object that the stream will be deserialized into. + * @return The object created and constructed by deserialize. NULL if there are any errors. + */ + friend Deserializer & operator >> (Deserializer & os, Identifiable & obj); + friend nbostream & operator >> (nbostream & os, Identifiable & obj); + + /** + * This will serialize a 4 byte num element before it will serialize all elements. + * This is used when you have a vector of objects of known class. + * It is symetric with deserialize template below + * @param v is vector of non polymorf Identifiable. + * @param os is output stream + * @return outputstream + */ + template <typename T> + static Serializer & serialize(const T & v, Serializer & os); + + /** + * This will serialize a 4 byte num element before it will serialize all elements. + * This is used when you have a vector of objects of known class. + * It is symetric with deserialize template below + * @param v is vector of non polymorf Identifiable. + * @param os is output stream + * @return outputstream + */ + template <typename T> + static Deserializer & deserialize(T & v, Deserializer & is); + + Serializer & serializeDirect(Serializer & os) const { return onSerialize(os); } + Deserializer & deserializeDirect(Deserializer & is) { return onDeserialize(is); } +protected: + /** + * Returns the diff of the classid. Used for comparing classes. + * @param b the object to compare with. + * @return getClass().id() - b.getClass().id() + */ + int cmpClassId(const Identifiable& b) const + { return getClass().id() - b.getClass().id(); } + +private: + /** + * Interface for comparing objects to each other. Default implementation + * is to serialise the two objects and use memcmp to verify equality. + * Classes should overide this method if they have special needs. + * It might also be an idea to override for speed, as serialize is 'expensive'. + * @param b the object to compare with. + * @return <0 if this comes before b, 0 for equality, and >0 fi b comes before *this. + */ + virtual int onCmp(const Identifiable& b) const; + virtual Serializer & onSerialize(Serializer & os) const; + virtual Deserializer & onDeserialize(Deserializer & is); + + static ILoader * _classLoader; +}; + +} + diff --git a/vespalib/src/vespa/vespalib/objects/identifiable.hpp b/vespalib/src/vespa/vespalib/objects/identifiable.hpp new file mode 100644 index 00000000000..d86f5490bc9 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/identifiable.hpp @@ -0,0 +1,132 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +/** + * \class vespalib::Identifiable + * \ingroup util + * + * \brief Superclass for objects adding some runtime type information. + * + * This class is a superclass used by many other classes to add some runtime + * information. + * + * This can be used to verify type to be able to do cheap static casts + * instead of dynamic casts. It can be used to identify type of object, and + * it can also be used to generate an object of the given type (If it's not + * an identifiable abstract type). + * + */ + +#include "identifiable.h" + +namespace vespalib { + +template <typename T> +Serializer & Identifiable::serialize(const T & v, Serializer & os) { + uint32_t sz(v.size()); + os.put(sz); + for(size_t i(0); i < sz; i++) { + v[i].serialize(os); + } + return os; +} + +template <typename T> +Deserializer & Identifiable::deserialize(T & v, Deserializer & is) { + uint32_t sz(0); + is.get(sz); + v.resize(sz); + for(size_t i(0); i < sz; i++) { + v[i].deserialize(is); + } + return is; +} + +template <typename T> +class IdentifiablePtr : public CloneablePtr<T> +{ +public: + IdentifiablePtr(const T &t) : CloneablePtr<T>(t.clone()) {} + IdentifiablePtr(IdentifiablePtr &&) noexcept = default; + IdentifiablePtr & operator = (IdentifiablePtr &&) noexcept = default; + IdentifiablePtr(const IdentifiablePtr &) = default; + IdentifiablePtr & operator = (const IdentifiablePtr &) = default; + IdentifiablePtr(T * p=nullptr) noexcept : CloneablePtr<T>(p) { } + IdentifiablePtr(std::unique_ptr<T> &&rhs) noexcept + : CloneablePtr<T>(std::move(rhs)) + { + } + IdentifiablePtr &operator=(std::unique_ptr<T> &&rhs) noexcept + { + CloneablePtr<T>::operator=(std::move(rhs)); + return *this; + } + int cmp(const IdentifiablePtr<T> &rhs) const { + const T *a = this->get(); + const T *b = rhs.get(); + if (a == 0) { + return (b == 0) ? 0 : -1; + } + return (b == 0) ? 1 : a->cmp(*b); + } + bool operator < (const IdentifiablePtr<T> &rhs) const { return (cmp(rhs) < 0); } + bool operator > (const IdentifiablePtr<T> &rhs) const { return (cmp(rhs) > 0); } + bool operator == (const IdentifiablePtr<T> &rhs) const { return (cmp(rhs) == 0); } + bool operator != (const IdentifiablePtr<T> &rhs) const { return (cmp(rhs) != 0); } + Serializer & serialize(Serializer & os) const { + if (this->get()) { + os.put(uint8_t(1)) << *this->get(); + } else { + os.put(uint8_t(0)); + } + return os; + } + Deserializer & deserialize(Deserializer & is) { + uint8_t hasObject; + is.get(hasObject); + if (hasObject) { + this->reset(static_cast<T *>(Identifiable::create(is).release())); + } + return is; + } + friend Serializer & operator << (Serializer & os, const IdentifiablePtr<T> & agg) { return agg.serialize(os); } + friend Deserializer & operator >> (Deserializer & is, IdentifiablePtr<T> & agg) { return agg.deserialize(is); } +}; + +template <typename T> +class IdentifiableSharedPtr : public std::shared_ptr<T> +{ +public: + IdentifiableSharedPtr(const T &t) : std::shared_ptr<T>(t.clone()) {} + IdentifiableSharedPtr(T * p=nullptr) : std::shared_ptr<T>(p) { } + int cmp(const IdentifiableSharedPtr<T> &rhs) const { + const T *a = this->get(); + const T *b = rhs.get(); + if (a == 0) { + return (b == 0) ? 0 : -1; + } + return (b == 0) ? 1 : a->cmp(*b); + } + bool operator < (const IdentifiableSharedPtr<T> &rhs) const { + return (cmp(rhs) < 0); + } + Serializer & serialize(Serializer & os) const { + if (this->get()) { + os.put(uint8_t(1)) << *this->get(); + } else { + os.put(uint8_t(0)); + } + return os; + } + Deserializer & deserialize(Deserializer & is) { + uint8_t hasObject; + is.get(hasObject); + if (hasObject) { + reset(static_cast<T *>(Identifiable::create(is).release())); + } + return is; + } + friend Serializer & operator << (Serializer & os, const IdentifiableSharedPtr<T> & agg) { return agg.serialize(os); } + friend Deserializer & operator >> (Deserializer & is, IdentifiableSharedPtr<T> & agg) { return agg.deserialize(is); } +}; + +} diff --git a/vespalib/src/vespa/vespalib/objects/ids.h b/vespalib/src/vespa/vespalib/objects/ids.h new file mode 100644 index 00000000000..21d4d92aef1 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/ids.h @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +/* + * These are the namespaces of ids used by identifiable classes. + * Should correspond to actual C++ namespace names. + */ + +#define DOCUMENT_CID(v) (0x1000 + v) +#define STORAGEAPI_CID(v) (0x2000 + v) +#define STORAGECLIENT_CID(v) (0x3000 + v) +#define SEARCHLIB_CID(v) (0x4000 + v) +#define VESPALIB_CID(v) (0x5000 + v) +#define CONFIGD_CID(v) (0x6000 + v) +#define VESPA_CONFIGMODEL_CID(v) (0x7000 + v) + +/* + * Here are all the ids in the vespalib namespace: + */ + +#define CID_vespalib_NamedObject VESPALIB_CID(9) + diff --git a/vespalib/src/vespa/vespalib/objects/nboserializer.cpp b/vespalib/src/vespa/vespalib/objects/nboserializer.cpp new file mode 100644 index 00000000000..0508210bd09 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/nboserializer.cpp @@ -0,0 +1,91 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "nboserializer.h" +#include <vespa/vespalib/objects/nbostream.h> + +namespace vespalib { + +const char * NBOSerializer::peek() const { + return _stream.peek(); +} + +NBOSerializer &NBOSerializer::put(bool value) { + _stream << value; + return *this; +} + +NBOSerializer &NBOSerializer::put(uint8_t value) { + _stream << value; + return *this; +} + +NBOSerializer &NBOSerializer::put(uint16_t value) { + _stream << value; + return *this; +} + +NBOSerializer &NBOSerializer::put(uint32_t value) { + _stream << value; + return *this; +} + +NBOSerializer &NBOSerializer::put(uint64_t value) { + _stream << value; + return *this; +} + +NBOSerializer &NBOSerializer::put(float value) { + _stream << value; + return *this; +} + +NBOSerializer &NBOSerializer::put(double value) { + _stream << value; + return *this; +} + +NBOSerializer &NBOSerializer::put(stringref value) { + _stream << value; + return *this; +} + + +NBOSerializer &NBOSerializer::get(bool & value) { + _stream >> value; + return *this; +} + +NBOSerializer &NBOSerializer::get(uint8_t & value) { + _stream >> value; + return *this; +} + +NBOSerializer &NBOSerializer::get(uint16_t & value) { + _stream >> value; + return *this; +} + +NBOSerializer &NBOSerializer::get(uint32_t & value) { + _stream >> value; + return *this; +} + +NBOSerializer &NBOSerializer::get(uint64_t & value) { + _stream >> value; + return *this; +} + +NBOSerializer &NBOSerializer::get(double & value) { + _stream >> value; + return *this; +} + +NBOSerializer &NBOSerializer::get(float & value) { + _stream >> value; + return *this; +} + +NBOSerializer &NBOSerializer::get(string & value) { + _stream >> value; + return *this; +} +} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/objects/nboserializer.h b/vespalib/src/vespa/vespalib/objects/nboserializer.h new file mode 100644 index 00000000000..4e11771d64b --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/nboserializer.h @@ -0,0 +1,40 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "serializer.h" +#include "deserializer.h" + +namespace vespalib { + +class nbostream; + +class NBOSerializer : public Serializer, public Deserializer { +public: + NBOSerializer(nbostream &stream) : _stream(stream) { } + NBOSerializer &put(bool value) override; + NBOSerializer &put(uint8_t value) override; + NBOSerializer &put(uint16_t value) override; + NBOSerializer &put(uint32_t value) override; + NBOSerializer &put(uint64_t value) override; + NBOSerializer &put(float value) override; + NBOSerializer &put(double value) override; + NBOSerializer &put(stringref val) override; + + NBOSerializer &get(bool &value) override; + NBOSerializer &get(uint8_t &value) override; + NBOSerializer &get(uint16_t &value) override; + NBOSerializer &get(uint32_t &value) override; + NBOSerializer &get(uint64_t &value) override; + NBOSerializer &get(double &value) override; + NBOSerializer &get(float &value) override; + NBOSerializer &get(string &value) override; + + const char *peek() const; + + const nbostream &getStream() const { return _stream; } + nbostream &getStream() { return _stream; } +private: + nbostream &_stream; +}; +} // namespace vespalib + diff --git a/vespalib/src/vespa/vespalib/objects/object2slime.cpp b/vespalib/src/vespa/vespalib/objects/object2slime.cpp new file mode 100644 index 00000000000..ee7b0501832 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/object2slime.cpp @@ -0,0 +1,76 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "object2slime.h" +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/data/slime/cursor.h> + +namespace vespalib { + +Object2Slime::Object2Slime(slime::Cursor & cursor) + : _cursor(cursor), + _stack() +{ +} + +Object2Slime::~Object2Slime() = default; + +//----------------------------------------------------------------------------- + +void +Object2Slime::openStruct(const vespalib::string &name, const vespalib::string &type) +{ + if (name.empty()) { + _cursor.get().setString("[type]", type); + } else { + _stack.push_back(_cursor); + _cursor = _cursor.get().setObject(name); + _cursor.get().setString("[type]", type); + } +} + +void +Object2Slime::closeStruct() +{ + if ( ! _stack.empty()) { + _cursor = _stack.back(); + _stack.pop_back(); + } +} + +void +Object2Slime::visitBool(const vespalib::string &name, bool value) +{ + _cursor.get().setBool(name, value); +} + +void +Object2Slime::visitInt(const vespalib::string &name, int64_t value) +{ + _cursor.get().setLong(name, value); +} + +void +Object2Slime::visitFloat(const vespalib::string &name, double value) +{ + _cursor.get().setDouble(name, value); +} + +void +Object2Slime::visitString(const vespalib::string &name, const vespalib::string &value) +{ + _cursor.get().setString(name, value); +} + +void +Object2Slime::visitNull(const vespalib::string &name) +{ + _cursor.get().setNix(name); +} + +void +Object2Slime::visitNotImplemented() +{ + _cursor.get().setNix("not_implemented"); +} + +} diff --git a/vespalib/src/vespa/vespalib/objects/object2slime.h b/vespalib/src/vespa/vespalib/objects/object2slime.h new file mode 100644 index 00000000000..ffab739ed15 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/object2slime.h @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "objectvisitor.h" +#include <vector> +#include <memory> + +namespace vespalib { + +namespace slime { struct Cursor; } + +/** + * This is a concrete object visitor that will build up a structured + * slime representation of an object. + **/ +class Object2Slime : public ObjectVisitor +{ +private: + std::reference_wrapper<slime::Cursor> _cursor; + std::vector<std::reference_wrapper<slime::Cursor>> _stack; + +public: + Object2Slime(slime::Cursor & cursor); + ~Object2Slime(); + + void openStruct(const vespalib::string &name, const vespalib::string &type) override; + void closeStruct() override; + void visitBool(const vespalib::string &name, bool value) override; + void visitInt(const vespalib::string &name, int64_t value) override; + void visitFloat(const vespalib::string &name, double value) override; + void visitString(const vespalib::string &name, const vespalib::string &value) override; + void visitNull(const vespalib::string &name) override; + void visitNotImplemented() override; +}; + +} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/objects/objectdumper.cpp b/vespalib/src/vespa/vespalib/objects/objectdumper.cpp new file mode 100644 index 00000000000..61230ace2f9 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/objectdumper.cpp @@ -0,0 +1,102 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "objectdumper.h" +#include <vespa/vespalib/util/stringfmt.h> + +namespace vespalib { + +void +ObjectDumper::addIndent() +{ + int n = _currIndent; + if (n < 0) { + n = 0; + } + _str.append(vespalib::string(n, ' ')); +} + +void +ObjectDumper::addLine(const vespalib::string &line) +{ + addIndent(); + _str.append(line); + _str.push_back('\n'); +} + +void +ObjectDumper::openScope() +{ + _currIndent += _indent; +} + +void +ObjectDumper::closeScope() +{ + _currIndent -= _indent; +} + +ObjectDumper::ObjectDumper(int indent) + : _str(), + _indent(indent), + _currIndent(0) +{ +} + +ObjectDumper::~ObjectDumper() = default; + +//----------------------------------------------------------------------------- + +void +ObjectDumper::openStruct(const vespalib::string &name, const vespalib::string &type) +{ + if (name.empty()) { + addLine(make_string("%s {", type.c_str())); + } else { + addLine(make_string("%s: %s {", name.c_str(), type.c_str())); + } + openScope(); +} + +void +ObjectDumper::closeStruct() +{ + closeScope(); + addLine("}"); +} + +void +ObjectDumper::visitBool(const vespalib::string &name, bool value) +{ + addLine(make_string("%s: %s", name.c_str(), value? "true" : "false")); +} + +void +ObjectDumper::visitInt(const vespalib::string &name, int64_t value) +{ + addLine(make_string("%s: %" PRId64 "", name.c_str(), value)); +} + +void +ObjectDumper::visitFloat(const vespalib::string &name, double value) +{ + addLine(make_string("%s: %g", name.c_str(), value)); +} + +void +ObjectDumper::visitString(const vespalib::string &name, const vespalib::string &value) +{ + addLine(make_string("%s: '%s'", name.c_str(), value.c_str())); +} + +void +ObjectDumper::visitNull(const vespalib::string &name) +{ + addLine(make_string("%s: <NULL>", name.c_str())); +} + +void +ObjectDumper::visitNotImplemented() +{ + addLine("<member visit not implemented>"); +} + +} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/objects/objectdumper.h b/vespalib/src/vespa/vespalib/objects/objectdumper.h new file mode 100644 index 00000000000..0fb27c03ac2 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/objectdumper.h @@ -0,0 +1,73 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "objectvisitor.h" + +namespace vespalib { + +/** + * This is a concrete object visitor that will build up a structured + * human-readable string representation of an object. + **/ +class ObjectDumper : public ObjectVisitor +{ +private: + vespalib::string _str; + int _indent; + int _currIndent; + + /** + * Add a number of spaces equal to the current indent to the + * string we are building. + **/ + void addIndent(); + + /** + * Add a complete line of output. Appropriate indentation will be + * added before the given string and a newline will be added after + * it. + * + * @param line the line we want to add + **/ + void addLine(const vespalib::string &line); + + /** + * Open a subscope by increasing the current indent level + **/ + void openScope(); + + /** + * Close a subscope by decreasing the current indent level + **/ + void closeScope(); + +public: + /** + * Create an object dumper with the given indent size; default is + * 4 spaces per indent level. + * + * @param indent indent size in number of spaces + **/ + ObjectDumper(int indent = 4); + ~ObjectDumper(); + + /** + * Obtain the created object string representation. This object + * should be invoked after the complete object structure has been + * visited. + * + * @return object string representation + **/ + vespalib::string toString() const { return _str; } + + void openStruct(const vespalib::string &name, const vespalib::string &type) override; + void closeStruct() override; + void visitBool(const vespalib::string &name, bool value) override; + void visitInt(const vespalib::string &name, int64_t value) override; + void visitFloat(const vespalib::string &name, double value) override; + void visitString(const vespalib::string &name, const vespalib::string &value) override; + void visitNull(const vespalib::string &name) override; + void visitNotImplemented() override; +}; + +} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/objects/objectoperation.cpp b/vespalib/src/vespa/vespalib/objects/objectoperation.cpp new file mode 100644 index 00000000000..0a84ad4b70f --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/objectoperation.cpp @@ -0,0 +1,6 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "objectoperation.h" + +namespace vespalib { + +} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/objects/objectoperation.h b/vespalib/src/vespa/vespalib/objects/objectoperation.h new file mode 100644 index 00000000000..ba2820f1a85 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/objectoperation.h @@ -0,0 +1,28 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +namespace vespalib { + +class Identifiable; + +/** + * An operation that is able to operate on a generic object. + **/ +class ObjectOperation +{ +public: + /** + * Apply this operation to the given object. + * + * @param obj the object to operate on + **/ + virtual void execute(Identifiable &obj) = 0; + + /** + * empty + **/ + virtual ~ObjectOperation() { } +}; + +} // namespace vespalib + diff --git a/vespalib/src/vespa/vespalib/objects/objectpredicate.cpp b/vespalib/src/vespa/vespalib/objects/objectpredicate.cpp new file mode 100644 index 00000000000..61986ed4cf8 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/objectpredicate.cpp @@ -0,0 +1,6 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "objectpredicate.h" + +namespace vespalib { + +} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/objects/objectpredicate.h b/vespalib/src/vespa/vespalib/objects/objectpredicate.h new file mode 100644 index 00000000000..12af9bab3e6 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/objectpredicate.h @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +namespace vespalib { + +class Identifiable; + +/** + * A predicate that is able to say either true or false when presented + * with a generic object. + **/ +class ObjectPredicate +{ +public: + /** + * Apply this predicate to the given object. + * + * @return true or false + * @param obj the object to check + **/ + virtual bool check(const Identifiable &obj) const = 0; + + /** + * empty + **/ + virtual ~ObjectPredicate() { } +}; + +} // namespace vespalib + diff --git a/vespalib/src/vespa/vespalib/objects/objectvisitor.cpp b/vespalib/src/vespa/vespalib/objects/objectvisitor.cpp new file mode 100644 index 00000000000..5466b19f6d7 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/objectvisitor.cpp @@ -0,0 +1,11 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "objectvisitor.h" +#include "identifiable.h" + +namespace vespalib { + +ObjectVisitor::~ObjectVisitor() +{ +} + +} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/objects/objectvisitor.h b/vespalib/src/vespa/vespalib/objects/objectvisitor.h new file mode 100644 index 00000000000..dba731a8940 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/objectvisitor.h @@ -0,0 +1,88 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +namespace vespalib { + +/** + * This is an abstract class used to visit structured objects. It + * contains a basic interface that is intended to be overridden by + * subclasses. As an extension to this class, the visit.hpp file + * contains various versions of the visit method that maps visitation + * of various types into invocations of the basic interface defined by + * this class. + **/ +class ObjectVisitor +{ +public: + /** + * Open a (sub-)structure + * + * @param name name of structure + * @param type type of structure + **/ + virtual void openStruct(const vespalib::string &name, const vespalib::string &type) = 0; + + /** + * Close a (sub-)structure + **/ + virtual void closeStruct() = 0; + + /** + * Visit a boolean value + * + * @param name variable name + * @param value variable value + **/ + virtual void visitBool(const vespalib::string &name, bool value) = 0; + + /** + * Visit an integer value + * + * @param name variable name + * @param value variable value + **/ + virtual void visitInt(const vespalib::string &name, int64_t value) = 0; + + /** + * Visit a floating-point value + * + * @param name variable name + * @param value variable value + **/ + virtual void visitFloat(const vespalib::string &name, double value) = 0; + + /** + * Visit a string value + * + * @param name variable name + * @param value variable value + **/ + virtual void visitString(const vespalib::string &name, const vespalib::string &value) = 0; + + /** + * Visit method used to indicate that an optional substructure is + * not present. + * + * @param name variable name + **/ + virtual void visitNull(const vespalib::string &name) = 0; + + /** + * Visit method invoked by the default implementation of member + * visitation to signal that member visitation is not yet implemented. + * + * @param name variable name + * @param value variable value + **/ + virtual void visitNotImplemented() = 0; + + /** + * empty + **/ + virtual ~ObjectVisitor(); +}; + +} // namespace vespalib + diff --git a/vespalib/src/vespa/vespalib/objects/serializer.cpp b/vespalib/src/vespa/vespalib/objects/serializer.cpp new file mode 100644 index 00000000000..3c1e22e52ac --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/serializer.cpp @@ -0,0 +1,17 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "serializer.h" +#include "identifiable.h" + +namespace vespalib { + +Serializer & Serializer::put(const Identifiable & value) +{ + return value.serializeDirect(*this); +} + +Serializer & Serializer::put(int8_t value) { return put(static_cast< uint8_t>(value)); } +Serializer & Serializer::put(int16_t value) { return put(static_cast<uint16_t>(value)); } +Serializer & Serializer::put(int32_t value) { return put(static_cast<uint32_t>(value)); } +Serializer & Serializer::put(int64_t value) { return put(static_cast<uint64_t>(value)); } + +} diff --git a/vespalib/src/vespa/vespalib/objects/serializer.h b/vespalib/src/vespa/vespalib/objects/serializer.h new file mode 100644 index 00000000000..f10d03b1c04 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/serializer.h @@ -0,0 +1,51 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/util/array.h> +#include <vespa/vespalib/stllike/string.h> +#include <vector> +#include <cstdint> + +namespace vespalib { + +class Identifiable; + +class Serializer +{ +public: + virtual ~Serializer() = default; + virtual Serializer & put(bool value) = 0; + virtual Serializer & put(uint8_t value) = 0; + virtual Serializer & put(uint16_t value) = 0; + virtual Serializer & put(uint32_t value) = 0; + virtual Serializer & put(uint64_t value) = 0; + virtual Serializer & put(float value) = 0; + virtual Serializer & put(double value) = 0; + virtual Serializer & put(stringref value) = 0; + + virtual Serializer & put(const Identifiable & value); + virtual Serializer & put(int8_t value); + virtual Serializer & put(int16_t value); + virtual Serializer & put(int32_t value); + virtual Serializer & put(int64_t value); + + Serializer & operator << (bool value) { return put(value); } + Serializer & operator << (uint8_t value) { return put(value); } + Serializer & operator << (int8_t value) { return put(value); } + Serializer & operator << (uint16_t value) { return put(value); } + Serializer & operator << (int16_t value) { return put(value); } + Serializer & operator << (uint32_t value) { return put(value); } + Serializer & operator << (int32_t value) { return put(value); } + Serializer & operator << (uint64_t value) { return put(value); } + Serializer & operator << (int64_t value) { return put(value); } + Serializer & operator << (float value) { return put(value); } + Serializer & operator << (double value) { return put(value); } + Serializer & operator << (stringref value) { return put(value); } + template <typename T> + Serializer & operator << (const vespalib::Array<T> & v); + template <typename T> + Serializer & operator << (const std::vector<T> & v); +}; + +} + diff --git a/vespalib/src/vespa/vespalib/objects/serializer.hpp b/vespalib/src/vespa/vespalib/objects/serializer.hpp new file mode 100644 index 00000000000..87c02ddf693 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/serializer.hpp @@ -0,0 +1,32 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/util/array.h> +#include <vector> +#include <cstdint> + +namespace vespalib { + +template <typename T> +Serializer & +Serializer::operator << (const vespalib::Array<T> & v) { + uint32_t sz(v.size()); + put(sz); + for(size_t i(0); i < sz; i++) { + (*this) << v[i]; + } + return *this; +} +template <typename T> +Serializer & +Serializer::operator << (const std::vector<T> & v) { + uint32_t sz(v.size()); + put(sz); + for(size_t i(0); i < sz; i++) { + (*this) << v[i]; + } + return *this; +} + +} + diff --git a/vespalib/src/vespa/vespalib/objects/visit.cpp b/vespalib/src/vespa/vespalib/objects/visit.cpp new file mode 100644 index 00000000000..03f46d4c354 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/visit.cpp @@ -0,0 +1,93 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "visit.hpp" + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::Identifiable *obj) { + if (obj != 0) { + self.openStruct(name, obj->getClass().name()); + obj->visitMembers(self); + self.closeStruct(); + } else { + self.visitNull(name); + } +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::Identifiable &obj) { + visit(self, name, &obj); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, bool value) { + self.visitBool(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, char value) +{ + self.visitInt(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, signed char value) +{ + self.visitInt(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, unsigned char value) +{ + self.visitInt(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, short value) +{ + self.visitInt(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, unsigned short value) +{ + self.visitInt(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, int value) { + self.visitInt(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, unsigned int value) { + self.visitInt(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, long value) { + self.visitInt(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, unsigned long value) { + self.visitInt(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, long long value) { + self.visitInt(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, unsigned long long value) { + self.visitInt(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, float value) { + self.visitFloat(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, double value) { + self.visitFloat(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::string &value) { + self.visitString(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const std::string &value) { + self.visitString(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const char *value) { + if (value != 0) { + visit(self, name, vespalib::string(value)); + } else { + self.visitNull(name); + } +} diff --git a/vespalib/src/vespa/vespalib/objects/visit.h b/vespalib/src/vespa/vespalib/objects/visit.h new file mode 100644 index 00000000000..87d42923d61 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/visit.h @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +namespace vespalib { + class Identifiable; + class ObjectVisitor; +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::Identifiable *obj); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::Identifiable &obj); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, bool value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, char value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, signed char value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, unsigned char value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, short value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, unsigned short value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, int value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, unsigned int value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, long value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, unsigned long value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, long long value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, unsigned long long value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, float value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, double value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::string &value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, vespalib::stringref value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const char *value); diff --git a/vespalib/src/vespa/vespalib/objects/visit.hpp b/vespalib/src/vespa/vespalib/objects/visit.hpp new file mode 100644 index 00000000000..e3a82b212c0 --- /dev/null +++ b/vespalib/src/vespa/vespalib/objects/visit.hpp @@ -0,0 +1,64 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "visit.h" +#include <vector> +#include <vespa/vespalib/util/stringfmt.h> +#include "objectvisitor.h" +#include "identifiable.hpp" + +template<typename T> +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::CloneablePtr<T> &ptr) { + if (ptr.get()) { + visit(self, name, *ptr); + } else { + self.visitNull(name); + } +} + +template<typename T> +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const std::shared_ptr<T> &ptr) { + if (ptr.get()) { + visit(self, name, *ptr); + } else { + self.visitNull(name); + } +} + +template<typename T> +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const std::unique_ptr<T> &ptr) { + if (ptr.get()) { + visit(self, name, *ptr); + } else { + self.visitNull(name); + } +} + +template<typename T> +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::IdentifiablePtr<T> &ptr) { + visit(self, name, ptr.get()); +} + +template<typename T> +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::IdentifiableSharedPtr<T> &ptr) { + visit(self, name, ptr.get()); +} + +template<typename T> +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const std::vector<T> &list) { + self.openStruct(name, "std::vector"); + for (uint32_t i = 0; i < list.size(); ++i) { + ::visit(self, vespalib::make_string("[%u]", i), list[i]); + } + self.closeStruct(); +} + +template<typename T> +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::Array<T> &list) { + self.openStruct(name, "vespalib::Array"); + for (uint32_t i = 0; i < list.size(); ++i) { + ::visit(self, vespalib::make_string("[%u]", i), list[i]); + } + self.closeStruct(); +} + diff --git a/vespalib/src/vespa/vespalib/stllike/cache.h b/vespalib/src/vespa/vespalib/stllike/cache.h new file mode 100644 index 00000000000..bf58f1aef98 --- /dev/null +++ b/vespalib/src/vespa/vespalib/stllike/cache.h @@ -0,0 +1,174 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "lrucache_map.h" +#include <atomic> +#include <mutex> + +namespace vespalib { + +struct CacheStats; + +template<typename K, typename V> +class NullStore { +public: + bool read(const K &, V &) const { return false; } + void write(const K &, const V &) { } + void erase(const K &) { } +}; + +/** + * These are the parameters needed for setting up the cache. + * @param P is the set of parameters needed for setting up the underlying lrucache. See @ref LruParam + * @param B is the backing store. That is where the real data is backed up if there are any. + * If there are no backing store or you mix and match yourself, you can give it the @ref NullStore. + * @param SizeK is the method to get the space needed by the key in addition to what you get with sizeof. + * @param SizeV is the method to get the space needed by the value in addition to what you get with sizeof. + */ +template<typename P, typename B, typename sizeK = vespalib::zero<typename P::Key>, typename sizeV = vespalib::zero<typename P::Value> > +struct CacheParam : public P +{ + typedef B BackingStore; + typedef sizeK SizeK; + typedef sizeV SizeV; +}; + +/** + * This is a cache using the underlying lru implementation as the store. It is modelled as a pure cache + * with an backing store underneath it. That backing store is given to the constructor and must of course have + * proper lifetime. The store must implement the same 3 methods as the @ref NullStore above. + * Stuff is evicted from the cache if either number of elements or the accounted size passes the limits given. + * The cache is thread safe by a single lock for accessing the underlying Lru. In addition a striped locking with + * 64 locks chosen by the hash of the key to enable a single fetch for any element required by multiple readers. + */ +template< typename P > +class cache : private lrucache_map<P> +{ + using Lru = lrucache_map<P>; +protected: + typedef typename P::BackingStore BackingStore; + typedef typename P::Hash Hash; + typedef typename P::Key K; + typedef typename P::Value V; + typedef typename P::SizeK SizeK; + typedef typename P::SizeV SizeV; + typedef typename P::value_type value_type; +public: + /** + * Will create a cache that populates on demand from the backing store. + * The cache uses LRU and evicts whne its size in bytes or elements is reached. + * By max elements is initialized to max bytes. + * + * @param backingStore is the store for populating the cache on a cache miss. + * @maxBytes is the maximum limit of bytes the store can hold, before eviction starts. + */ + cache(BackingStore & b, size_t maxBytes); + ~cache(); + /** + * Can be used for controlling max number of elements. + */ + cache & maxElements(size_t elems); + /** + * Can be used for reserving space for elements. + */ + cache & reserveElements(size_t elems); + + cache & setCapacityBytes(size_t sz); + + size_t capacity() const { return Lru::capacity(); } + size_t capacityBytes() const { return _maxBytes.load(std::memory_order_relaxed); } + size_t size() const { return Lru::size(); } + size_t sizeBytes() const { return _sizeBytes.load(std::memory_order_relaxed); } + bool empty() const { return Lru::empty(); } + + /** + * This simply erases the object. + * This will also erase from backing store. + */ + void erase(const K & key); + /** + * This simply erases the object from the cache. + */ + void invalidate(const K & key); + + /** + * Return the object with the given key. If it does not exist, the backing store will be consulted. + * and the cache will be updated. + * If none exist an empty one will be created. + * Object is then put at head of LRU list. + */ + V read(const K & key); + + /** + * Update the cache and write through to backing store. + * Object is then put at head of LRU list. + */ + void write(const K & key, V value); + + /** + * Tell if an object with given key exists in the cache. + * Does not alter the LRU list. + */ + bool hasKey(const K & key) const; + + CacheStats get_stats() const; + + size_t getHit() const { return _hit.load(std::memory_order_relaxed); } + size_t getMiss() const { return _miss.load(std::memory_order_relaxed); } + size_t getNoneExisting() const { return _noneExisting.load(std::memory_order_relaxed); } + size_t getRace() const { return _race.load(std::memory_order_relaxed); } + size_t getInsert() const { return _insert.load(std::memory_order_relaxed); } + size_t getWrite() const { return _write.load(std::memory_order_relaxed); } + size_t getInvalidate() const { return _invalidate.load(std::memory_order_relaxed); } + size_t getlookup() const { return _lookup.load(std::memory_order_relaxed); } + +protected: + using UniqueLock = std::unique_lock<std::mutex>; + UniqueLock getGuard(); + void invalidate(const UniqueLock & guard, const K & key); + bool hasKey(const UniqueLock & guard, const K & key) const; +private: + void verifyHashLock(const UniqueLock & guard) const; + /** + * Called when an object is inserted, to see if the LRU should be removed. + * Default is to obey the maxsize given in constructor. + * The obvious extension is when you are storing pointers and want to cap + * on the real size of the object pointed to. + */ + bool removeOldest(const value_type & v) override; + size_t calcSize(const K & k, const V & v) const { return sizeof(value_type) + _sizeK(k) + _sizeV(v); } + std::mutex & getLock(const K & k) { + size_t h(_hasher(k)); + return _addLocks[h%(sizeof(_addLocks)/sizeof(_addLocks[0]))]; + } + + template <typename V> + static void increment_stat(std::atomic<V>& v, const std::lock_guard<std::mutex>&) { + v.store(v.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed); + } + template <typename V> + static void increment_stat(std::atomic<V>& v, const std::unique_lock<std::mutex>&) { + v.store(v.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed); + } + + Hash _hasher; + SizeK _sizeK; + SizeV _sizeV; + std::atomic<size_t> _maxBytes; + std::atomic<size_t> _sizeBytes; + mutable std::atomic<size_t> _hit; + mutable std::atomic<size_t> _miss; + std::atomic<size_t> _noneExisting; + mutable std::atomic<size_t> _race; + mutable std::atomic<size_t> _insert; + mutable std::atomic<size_t> _write; + mutable std::atomic<size_t> _update; + mutable std::atomic<size_t> _invalidate; + mutable std::atomic<size_t> _lookup; + BackingStore & _store; + mutable std::mutex _hashLock; + /// Striped locks that can be used for having a locked access to the backing store. + std::mutex _addLocks[113]; +}; + +} diff --git a/vespalib/src/vespa/vespalib/stllike/cache.hpp b/vespalib/src/vespa/vespalib/stllike/cache.hpp new file mode 100644 index 00000000000..bb05d564f1f --- /dev/null +++ b/vespalib/src/vespa/vespalib/stllike/cache.hpp @@ -0,0 +1,185 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "cache.h" +#include "cache_stats.h" +#include "lrucache_map.hpp" + +namespace vespalib { + +template< typename P > +cache<P> & +cache<P>::maxElements(size_t elems) { + Lru::maxElements(elems); + return *this; +} + +template< typename P > +cache<P> & +cache<P>::reserveElements(size_t elems) { + Lru::reserve(elems); + return *this; +} + +template< typename P > +cache<P> & +cache<P>::setCapacityBytes(size_t sz) { + _maxBytes.store(sz, std::memory_order_relaxed); + return *this; +} + +template< typename P > +void +cache<P>::invalidate(const K & key) { + UniqueLock guard(_hashLock); + invalidate(guard, key); +} + +template< typename P > +bool +cache<P>::hasKey(const K & key) const { + UniqueLock guard(_hashLock); + return hasKey(guard, key); +} + +template< typename P > +cache<P>::~cache() = default; + +template< typename P > +cache<P>::cache(BackingStore & b, size_t maxBytes) : + Lru(Lru::UNLIMITED), + _maxBytes(maxBytes), + _sizeBytes(0), + _hit(0), + _miss(0), + _noneExisting(0), + _race(0), + _insert(0), + _write(0), + _update(0), + _invalidate(0), + _lookup(0), + _store(b) +{ } + +template< typename P > +bool +cache<P>::removeOldest(const value_type & v) { + bool remove(Lru::removeOldest(v) || (sizeBytes() >= capacityBytes())); + if (remove) { + _sizeBytes.store(sizeBytes() - calcSize(v.first, v.second._value), std::memory_order_relaxed); + } + return remove; +} + +template< typename P > +std::unique_lock<std::mutex> +cache<P>::getGuard() { + return UniqueLock(_hashLock); +} + +template< typename P > +typename P::Value +cache<P>::read(const K & key) +{ + { + std::lock_guard guard(_hashLock); + if (Lru::hasKey(key)) { + increment_stat(_hit, guard); + return (*this)[key]; + } else { + increment_stat(_miss, guard); + } + } + + std::lock_guard storeGuard(getLock(key)); + { + std::lock_guard guard(_hashLock); + if (Lru::hasKey(key)) { + // Somebody else just fetched it ahead of me. + increment_stat(_race, guard); + return (*this)[key]; + } + } + V value; + if (_store.read(key, value)) { + std::lock_guard guard(_hashLock); + Lru::insert(key, value); + _sizeBytes.store(sizeBytes() + calcSize(key, value), std::memory_order_relaxed); + increment_stat(_insert, guard); + } else { + _noneExisting.fetch_add(1); + } + return value; +} + +template< typename P > +void +cache<P>::write(const K & key, V value) +{ + size_t newSize = calcSize(key, value); + std::lock_guard storeGuard(getLock(key)); + { + std::lock_guard guard(_hashLock); + if (Lru::hasKey(key)) { + _sizeBytes.store(sizeBytes() - calcSize(key, (*this)[key]), std::memory_order_relaxed); + increment_stat(_update, guard); + } + } + + _store.write(key, value); + { + std::lock_guard guard(_hashLock); + (*this)[key] = std::move(value); + _sizeBytes.store(sizeBytes() + newSize, std::memory_order_relaxed); + increment_stat(_write, guard); + } +} + +template< typename P > +void +cache<P>::erase(const K & key) +{ + std::lock_guard storeGuard(getLock(key)); + invalidate(key); + _store.erase(key); +} + +template< typename P > +void +cache<P>::invalidate(const UniqueLock & guard, const K & key) +{ + verifyHashLock(guard); + if (Lru::hasKey(key)) { + _sizeBytes.store(sizeBytes() - calcSize(key, (*this)[key]), std::memory_order_relaxed); + increment_stat(_invalidate, guard); + Lru::erase(key); + } +} + +template< typename P > +bool +cache<P>::hasKey(const UniqueLock & guard, const K & key) const +{ + verifyHashLock(guard); + increment_stat(_lookup, guard); + return Lru::hasKey(key); +} + +template< typename P > +void +cache<P>::verifyHashLock(const UniqueLock & guard) const { + assert(guard.mutex() == & _hashLock); + assert(guard.owns_lock()); +} + +template <typename P> +CacheStats +cache<P>::get_stats() const +{ + std::lock_guard guard(_hashLock); + return CacheStats(getHit(), getMiss(), Lru::size(), sizeBytes(), getInvalidate()); +} + +} + diff --git a/vespalib/src/vespa/vespalib/stllike/cache_stats.h b/vespalib/src/vespa/vespalib/stllike/cache_stats.h new file mode 100644 index 00000000000..7c83da2b004 --- /dev/null +++ b/vespalib/src/vespa/vespalib/stllike/cache_stats.h @@ -0,0 +1,50 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstdint> +#include <sys/types.h> + +namespace vespalib { + +struct CacheStats { + size_t hits; + size_t misses; + size_t elements; + size_t memory_used; + size_t invalidations; + + CacheStats() + : hits(0), + misses(0), + elements(0), + memory_used(0), + invalidations(0) + { } + + CacheStats(size_t hits_, size_t misses_, size_t elements_, size_t memory_used_, size_t invalidations_) + : hits(hits_), + misses(misses_), + elements(elements_), + memory_used(memory_used_), + invalidations(invalidations_) + { } + + CacheStats & + operator+=(const CacheStats &rhs) + { + hits += rhs.hits; + misses += rhs.misses; + elements += rhs.elements; + memory_used += rhs.memory_used; + invalidations += rhs.invalidations; + return *this; + } + + void add_extra_misses(size_t extra_misses) { misses += extra_misses; } + + size_t lookups() const { return hits + misses; } +}; + +} + diff --git a/vespalib/src/vespa/vespalib/stllike/lrucache_map.h b/vespalib/src/vespa/vespalib/stllike/lrucache_map.h new file mode 100644 index 00000000000..ffb7a427d11 --- /dev/null +++ b/vespalib/src/vespa/vespalib/stllike/lrucache_map.h @@ -0,0 +1,210 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/hashtable.h> +#include <vespa/vespalib/stllike/hash_fun.h> +#include <vespa/vespalib/stllike/select.h> +#include <atomic> +#include <vector> + +namespace vespalib { + +struct LinkedValueBase { + static const uint32_t npos = static_cast<uint32_t>(-1); + LinkedValueBase() : _prev(npos), _next(npos) { } + LinkedValueBase(uint32_t prev, uint32_t next) : _prev(prev), _next(next) { } + uint32_t _prev; + uint32_t _next; +}; + +template<typename V> +struct LinkedValue : public LinkedValueBase +{ + LinkedValue() {} + LinkedValue(const V & v) : LinkedValueBase(), _value(v) { } + LinkedValue(V && v) : LinkedValueBase(), _value(std::move(v)) { } + V _value; +}; + +template<typename K, typename V, typename H = vespalib::hash<K>, typename EQ = std::equal_to<K> > +struct LruParam +{ + using LV = LinkedValue<V>; + using value_type = std::pair< K, LV >; + using select_key = vespalib::Select1st< value_type >; + using Key = K; + using Value = V; + using Hash = H; + using Equal = EQ; + using HashTable = hashtable< Key, value_type, Hash, Equal, select_key >; +}; + +template< typename P > +class lrucache_map : private P::HashTable +{ +private: + using HashTable = typename P::HashTable; + using V = typename P::Value; + using K = typename P::Key; + using value_type = typename P::value_type; + using LV = typename P::LV; + using internal_iterator = typename HashTable::iterator; + using next_t = typename HashTable::next_t; + using NodeStore = typename HashTable::NodeStore; +protected: + static constexpr size_t UNLIMITED = std::numeric_limits<size_t>::max(); +public: + using insert_result = typename HashTable::insert_result; + + class iterator { + public: + iterator(lrucache_map * cache, uint32_t current) : _cache(cache), _current(current) { } + V & operator * () const { return _cache->getByInternalIndex(_current).second._value; } + V * operator -> () const { return & _cache->getByInternalIndex(_current).second._value; } + iterator & operator ++ () { + if (_current != LinkedValueBase::npos) { + _current = _cache->getByInternalIndex(_current).second._next; + } + return *this; + } + iterator operator ++ (int) { + iterator prev = *this; + ++(*this); + return prev; + } + bool operator==(const iterator& rhs) const { return (_current == rhs._current); } + bool operator!=(const iterator& rhs) const { return (_current != rhs._current); } + private: + lrucache_map * _cache; + uint32_t _current; + + friend class lrucache_map; + }; + + /** + * Will create a lrucache with max elements. Use the chained setter + * @ref reserve to control initial size. + * + * @param maxElements in cache unless you override @ref removeOldest. + */ + lrucache_map(size_t maxElems); + virtual ~lrucache_map(); + + lrucache_map & maxElements(size_t elems) { + _maxElements.store(elems, std::memory_order_relaxed); + return *this; + } + lrucache_map & reserve(size_t elems) { + HashTable::reserve(elems); + return *this; + } + + + size_t capacity() const { return _maxElements.load(std::memory_order_relaxed); } + size_t size() const { return HashTable::size(); } + bool empty() const { return HashTable::empty(); } + iterator begin() { return iterator(this, _head); } + iterator end() { return iterator(this, LinkedValueBase::npos); } + + /** + * This fetches the object without modifying the lru list. + */ + const V & get(const K & key) const { return HashTable::find(key)->second._value; } + + /** + * This simply erases the object. + */ + void erase(const K & key); + + /** + * Erase object pointed to by iterator. + */ + iterator erase(const iterator & it); + + /** + * Object is inserted in cache with given key. + * Object is then put at head of LRU list. + */ + insert_result insert(const K & key, const V & value); + + /** + * Object is inserted in cache with given key. + * Object is then put at head of LRU list. + */ + insert_result insert(const K & key, V && value); + + /** + * Return pointer to the object with the given key. + * Object is then put at head of LRU list if found. + * If not found nullptr is returned. + */ + V * findAndRef(const K & key); + + /** + * Return the object with the given key. If it does not exist an empty one will be created. + * This can be used as an insert. + * Object is then put at head of LRU list. + */ + V & operator [] (const K & key); + + /** + * Tell if an object with given key exists in the cache. + * Does not alter the LRU list. + */ + bool hasKey(const K & key) const { return HashTable::find(key) != HashTable::end(); } + + /** + * Called when an object is inserted, to see if the LRU should be removed. + * Default is to obey the maxsize given in constructor. + * The obvious extension is when you are storing pointers and want to cap + * on the real size of the object pointed to. + */ + virtual bool removeOldest(const value_type & v); + virtual void onRemove(const K & key); + virtual void onInsert(const K & key); + + /** + * Method for testing that internal consitency is good. + */ + bool verifyInternals(); + + /** + * Implements the move callback from the hashtable + */ + void move(next_t from, next_t to); + + void swap(lrucache_map & rhs); + +private: + using MoveRecord = std::pair<uint32_t, uint32_t>; + using MoveRecords = std::vector<MoveRecord>; + /** + * Implements the resize of the hashtable + */ + void move(NodeStore && oldStore) override; + void ref(const internal_iterator & it); + insert_result insert(value_type && value); + void removeOld(); + class RecordMoves { + public: + RecordMoves(const RecordMoves &) = delete; + RecordMoves & operator = (const RecordMoves &) = delete; + RecordMoves(lrucache_map & lru) : + _lru(lru) + { + _lru._moveRecordingEnabled = true; + } + ~RecordMoves(); + uint32_t movedTo(uint32_t from); + private: + lrucache_map & _lru; + }; + + std::atomic<size_t> _maxElements; + mutable uint32_t _head; + mutable uint32_t _tail; + bool _moveRecordingEnabled; + MoveRecords _moved; +}; + +} diff --git a/vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp b/vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp new file mode 100644 index 00000000000..73e9956ffdf --- /dev/null +++ b/vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp @@ -0,0 +1,283 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "lrucache_map.h" +#include <vespa/vespalib/stllike/hashtable.hpp> +#include <cassert> + +namespace vespalib { + +template< typename P > +typename lrucache_map<P>::insert_result +lrucache_map<P>::insert(const K & key, const V & value) { + insert_result res = insert(value_type(key, LV(value))); + if (res.second) { + onInsert(key); + } + return res; +} + +template< typename P > +typename lrucache_map<P>::insert_result +lrucache_map<P>::insert(const K & key, V && value) { + insert_result res = insert(value_type(key, LV(std::move(value)))); + if (res.second) { + onInsert(key); + } + return res; +} + +template< typename P > +bool +lrucache_map<P>::removeOldest(const value_type & v) { + (void) v; + return (size() > capacity()); +} + +template< typename P > +void +lrucache_map<P>::onRemove(const K & key) { + (void) key; +} + +template< typename P > +void +lrucache_map<P>::onInsert(const K & key) { + (void) key; +} + +template< typename P > +uint32_t +lrucache_map<P>::RecordMoves::movedTo(uint32_t from) { + for (size_t i(0); i < _lru._moved.size(); i++) { + const MoveRecord & mr(_lru._moved[i]); + if (mr.first == from) { + from = mr.second; + } + } + return from; +} + +template< typename P > +lrucache_map<P>::RecordMoves::~RecordMoves() { + _lru._moveRecordingEnabled = false; + _lru._moved.clear(); +} + +template< typename P > +lrucache_map<P>::lrucache_map(size_t maxElems) : + HashTable(0), + _maxElements(maxElems), + _head(LinkedValueBase::npos), + _tail(LinkedValueBase::npos), + _moveRecordingEnabled(false), + _moved() +{ } + +template< typename P > +lrucache_map<P>::~lrucache_map() = default; + +template< typename P > +void +lrucache_map<P>::swap(lrucache_map & rhs) { + auto maxElements = rhs._maxElements.load(std::memory_order_relaxed); + rhs._maxElements.store(_maxElements.load(std::memory_order_relaxed), std::memory_order_relaxed); + _maxElements.store(maxElements, std::memory_order_relaxed); + std::swap(_head, rhs._head); + std::swap(_tail, rhs._tail); + HashTable::swap(rhs); +} + +template< typename P > +void +lrucache_map<P>::move(next_t from, next_t to) { + (void) from; + if (_moveRecordingEnabled) { + _moved.push_back(std::make_pair(from, to)); + } + value_type & moved = HashTable::getByInternalIndex(to); + if (moved.second._prev != LinkedValueBase::npos) { + HashTable::getByInternalIndex(moved.second._prev).second._next = to; + } else { + _head = to; + } + if (moved.second._next != LinkedValueBase::npos) { + HashTable::getByInternalIndex(moved.second._next).second._prev = to; + } else { + _tail = to; + } +} + +template< typename P > +void +lrucache_map<P>::erase(const K & key) { + internal_iterator it = HashTable::find(key); + if (it != HashTable::end()) { + next_t h = HashTable::hash(key); + onRemove(key); + LV & v = it->second; + if (v._prev != LinkedValueBase::npos) { + HashTable::getByInternalIndex(v._prev).second._next = v._next; + } else { + _head = v._next; + } + if (v._next != LinkedValueBase::npos) { + HashTable::getByInternalIndex(v._next).second._prev = v._prev; + } else { + _tail = v._prev; + } + HashTable::erase(*this, h, it); + } +} + +template< typename P > +typename lrucache_map<P>::iterator +lrucache_map<P>::erase(const iterator & it) +{ + iterator next(it); + if (it != end()) { + RecordMoves moves(*this); + next++; + const K & key(HashTable::getByInternalIndex(it._current).first); + erase(key); + next = iterator(this, moves.movedTo(next._current)); + } + return next; +} + +template< typename P > +bool +lrucache_map<P>::verifyInternals() +{ + bool retval(true); + assert(_head != LinkedValueBase::npos); + assert(_tail != LinkedValueBase::npos); + assert(HashTable::getByInternalIndex(_head).second._prev == LinkedValueBase::npos); + assert(HashTable::getByInternalIndex(_tail).second._next == LinkedValueBase::npos); + { + size_t i(0); + size_t prev(LinkedValueBase::npos); + size_t c(_head); + for(size_t m(size()); (c != LinkedValueBase::npos) && (i < m); c = HashTable::getByInternalIndex(c).second._next, i++) { + assert((HashTable::getByInternalIndex(c).second._prev == prev)); + prev = c; + } + assert(i == size()); + assert(c == LinkedValueBase::npos); + } + { + size_t i(0); + size_t next(LinkedValueBase::npos); + size_t c(_tail); + for(size_t m(size()); (c != LinkedValueBase::npos) && (i < m); c = HashTable::getByInternalIndex(c).second._prev, i++) { + assert((HashTable::getByInternalIndex(c).second._next == next)); + next = c; + } + assert(i == size()); + assert(c == LinkedValueBase::npos); + } + return retval; +} + +template< typename P > +void +lrucache_map<P>::move(NodeStore && oldStore) +{ + next_t curr(_tail); + _tail = LinkedValueBase::npos; + _head = LinkedValueBase::npos; + + while (curr != LinkedValueBase::npos) { + value_type & v = oldStore[curr].getValue(); + curr = v.second._prev; + v.second._prev = LinkedValueBase::npos; + v.second._next = LinkedValueBase::npos; + insert(std::move(v)); + } +} + +template< typename P > +void +lrucache_map<P>::removeOld() { + if (_tail != LinkedValueBase::npos) { + for (value_type * last(& HashTable::getByInternalIndex(_tail)); + (_tail != _head) && removeOldest(*last); + last = & HashTable::getByInternalIndex(_tail)) + { + _tail = last->second._prev; + HashTable::getByInternalIndex(_tail).second._next = LinkedValueBase::npos; + HashTable::erase(*this, HashTable::hash(last->first), HashTable::find(last->first)); + } + } +} + +template< typename P > +void +lrucache_map<P>::ref(const internal_iterator & it) { + uint32_t me(it.getInternalIndex()); + if (me != _head) { + LV & v = it->second; + LV & oldPrev = HashTable::getByInternalIndex(v._prev).second; + oldPrev._next = v._next; + if (me != _tail) { + LV & oldNext = HashTable::getByInternalIndex(v._next).second; + oldNext._prev = v._prev; + } else { + // I am tail and I am not the only one. + _tail = v._prev; + } + LV & oldHead = HashTable::getByInternalIndex(_head).second; + oldHead._prev = me; + v._next = _head; + v._prev = LinkedValueBase::npos; + _head = me; + } +} + +template< typename P > +typename lrucache_map<P>::insert_result +lrucache_map<P>::insert(value_type && value) { + insert_result res = HashTable::insertInternal(std::forward<value_type>(value)); + uint32_t next(_head); + if ( ! res.second) { + ref(res.first); + } else { + _head = res.first.getInternalIndex(); + HashTable::getByInternalIndex(_head).second._next = next; + if (next != LinkedValueBase::npos) { + HashTable::getByInternalIndex(next).second._prev = _head; + } + if (_tail == LinkedValueBase::npos) { + _tail = _head; + } + removeOld(); + if (_head != res.first.getInternalIndex()) { + res.first.setInternalIndex(_head); + } + } + return res; +} + +template< typename P > +typename P::Value & +lrucache_map<P>::operator [] (const K & key) +{ + return insert(key, V()).first->second._value; +} + +template< typename P > +typename P::Value * +lrucache_map<P>::findAndRef(const K & key) +{ + internal_iterator found = HashTable::find(key); + if (found != HashTable::end()) { + if (size()*2 > capacity()) { + ref(found); + } + return &found->second._value; + } + return nullptr; +} + +} + diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index c61de250f53..59c05dffbc6 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -13,15 +13,20 @@ vespa_add_library(vespalib_vespalib_util OBJECT benchmark_timer.cpp bfloat16.cpp binary_hamming_distance.cpp + bits.cpp blockingthreadstackexecutor.cpp box.cpp cgroup_resource_limits.cpp classname.cpp + clock.cpp compress.cpp compressor.cpp count_down_latch.cpp cpu_usage.cpp + crc.cpp destructor_callbacks.cpp + doom.cpp + document_runnable.cpp dual_merge_director.cpp error.cpp exception.cpp @@ -33,8 +38,11 @@ vespa_add_library(vespalib_vespalib_util OBJECT gencnt.cpp generationhandler.cpp generationholder.cpp + growablebytebuffer.cpp hdr_abort.cpp host_name.cpp + jsonexception.cpp + jsonstream.cpp jsonwriter.cpp invokeserviceimpl.cpp isequencedtaskexecutor.cpp @@ -43,6 +51,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT latch.cpp left_right_heap.cpp lz4compressor.cpp + malloc_mmap_guard.cpp md5.c memoryusage.cpp mmap_file_allocator.cpp @@ -51,6 +60,8 @@ vespa_add_library(vespalib_vespalib_util OBJECT nice.cpp printable.cpp priority_queue.cpp + process_memory_stats.cpp + programoptions.cpp random.cpp rcuvector.cpp regexp.cpp @@ -62,12 +73,14 @@ vespa_add_library(vespalib_vespalib_util OBJECT round_up_to_page_size.cpp runnable.cpp runnable_pair.cpp + rusage.cpp sequence.cpp sequencedtaskexecutor.cpp sequencedtaskexecutorobserver.cpp sha1.cpp shared_operation_throttler.cpp shared_string_repo.cpp + shutdownguard.cpp sig_catch.cpp signalhandler.cpp simple_thread_bundle.cpp @@ -76,6 +89,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT stash.cpp string_hash.cpp stringfmt.cpp + testclock.cpp thread.cpp thread_bundle.cpp threadstackexecutor.cpp @@ -83,6 +97,8 @@ vespa_add_library(vespalib_vespalib_util OBJECT time.cpp unwind_message.cpp valgrind.cpp + xmlserializable.cpp + xmlstream.cpp zstdcompressor.cpp DEPENDS ) diff --git a/vespalib/src/vespa/vespalib/util/bits.cpp b/vespalib/src/vespa/vespalib/util/bits.cpp new file mode 100644 index 00000000000..2d9848f3cdf --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/bits.cpp @@ -0,0 +1,45 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/util/bits.h> + +namespace vespalib { + +uint8_t Bits::_reverse[256]; +Bits::ReverseTableInit Bits::_reverseTableInit; + +void * Bits::reverse(void * srcDst, size_t sz) +{ + uint8_t *v(static_cast<uint8_t *>(srcDst)); + size_t i(0); + for(; i < sz/2; i++) { + v[i] = reverse(v[sz-1-i]); + } + return v; +} + +void Bits::forceInitNow() +{ + ReverseTableInit now; +} + +Bits::ReverseTableInit::ReverseTableInit() +{ + if (_reverse[128] == 0) { + for (size_t i(0); i < 256; i++) { + _reverse[i] = reverse(i); + } + } +} + +uint8_t Bits::ReverseTableInit::reverse(uint8_t v) +{ + return ((v >> 7) & 0x01) | + ((v >> 5) & 0x02) | + ((v >> 3) & 0x04) | + ((v >> 1) & 0x08) | + ((v << 1) & 0x10) | + ((v << 3) & 0x20) | + ((v << 5) & 0x40) | + ((v << 7) & 0x80); +} + +} diff --git a/vespalib/src/vespa/vespalib/util/bits.h b/vespalib/src/vespa/vespalib/util/bits.h new file mode 100644 index 00000000000..fbe1a59f62a --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/bits.h @@ -0,0 +1,66 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <cstdint> +#include <sys/types.h> + +namespace vespalib { + +/** + * @brief Bit fiddling class + * + * This class handles low level bit manipulation operations. + * - Fast bit reversal by table lookup. + **/ +class Bits +{ +public: + static uint8_t reverse(uint8_t v) { return _reverse[v]; } + static uint16_t reverse(uint16_t v) { + union { uint16_t v; uint8_t a[2]; } s, d; + s.v = v; + d.a[0] = _reverse[s.a[1]]; + d.a[1] = _reverse[s.a[0]]; + return d.v; + } + static uint32_t reverse(uint32_t v) { + union { uint32_t v; uint8_t a[4]; } s, d; + s.v = v; + d.a[0] = _reverse[s.a[3]]; + d.a[1] = _reverse[s.a[2]]; + d.a[2] = _reverse[s.a[1]]; + d.a[3] = _reverse[s.a[0]]; + return d.v; + } + static uint64_t reverse(uint64_t v) { + union { uint64_t v; uint8_t a[8]; } s, d; + s.v = v; + d.a[0] = _reverse[s.a[7]]; + d.a[1] = _reverse[s.a[6]]; + d.a[2] = _reverse[s.a[5]]; + d.a[3] = _reverse[s.a[4]]; + d.a[4] = _reverse[s.a[3]]; + d.a[5] = _reverse[s.a[2]]; + d.a[6] = _reverse[s.a[1]]; + d.a[7] = _reverse[s.a[0]]; + return d.v; + } + static void * reverse(void * v, size_t sz); + /** + Utility for other statically contructed objects to force correct init order." + **/ + static void forceInitNow(); +private: + class ReverseTableInit + { + public: + ReverseTableInit(); + static uint8_t reverse(uint8_t v); + }; + static uint8_t _reverse[256]; + static ReverseTableInit _reverseTableInit; +}; + +} + + diff --git a/vespalib/src/vespa/vespalib/util/clock.cpp b/vespalib/src/vespa/vespalib/util/clock.cpp new file mode 100644 index 00000000000..4b6e1e8ab85 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/clock.cpp @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "clock.h" +#include <cassert> +namespace vespalib { + +Clock::Clock(const std::atomic<steady_time> & source) noexcept + : _timeNS(source) +{ + static_assert(std::atomic<steady_time>::is_always_lock_free); +} + +Clock::~Clock() = default; + +} diff --git a/vespalib/src/vespa/vespalib/util/clock.h b/vespalib/src/vespa/vespalib/util/clock.h new file mode 100644 index 00000000000..4cd579b8f69 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/clock.h @@ -0,0 +1,33 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/util/time.h> +#include <atomic> +#include <memory> + +namespace vespalib { + +/** + * Clock is a clock that updates the time at defined intervals. + * It is intended used where you want to check the time with low cost, but where + * resolution is not that important. + */ + +class Clock +{ +private: + const std::atomic<steady_time> &_timeNS; +public: + Clock(const std::atomic<steady_time> & source) noexcept; + Clock(const Clock &) = delete; + Clock & operator =(const Clock &) = delete; + Clock(Clock &&) = delete; + Clock & operator =(Clock &&) = delete; + ~Clock(); + + vespalib::steady_time getTimeNS() const noexcept { + return vespalib::steady_time(_timeNS.load(std::memory_order_relaxed)); + } +}; + +} diff --git a/vespalib/src/vespa/vespalib/util/crc.cpp b/vespalib/src/vespa/vespalib/util/crc.cpp new file mode 100644 index 00000000000..1a9baf8e600 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/crc.cpp @@ -0,0 +1,65 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/util/crc.h> + +namespace vespalib { + +uint32_t crc_32_type::_crc[256]; +crc_32_type::CrcTableInit crc_32_type::_crcTableInit; + +uint32_t crc_32_type::crc(const void * src, size_t sz) +{ + const uint8_t *v = static_cast<const uint8_t *>(src); + uint32_t c(uint32_t(-1)); + + for (size_t i(0); i < sz; i++ ) { + c = (c >> 8) ^ _crc[ uint8_t(c ^ v[i]) ]; + } + + return c ^ (uint32_t(-1)); +} + +void crc_32_type::process_bytes(const void *start, size_t sz) +{ + const uint8_t *v = static_cast<const uint8_t *>(start); + uint32_t c(_c); + + for (size_t i(0); i < sz; i++ ) { + c = (c >> 8) ^ _crc[ uint8_t(c ^ v[i]) ]; + } + _c = c; +} + +crc_32_type::CrcTableInit::CrcTableInit() +{ + Bits::forceInitNow(); + uint8_t dividend = 0; + do { + _crc[Bits::reverse(dividend)] = crc(dividend); + } while ( ++dividend ); +} + +uint32_t crc_32_type::CrcTableInit::crc(uint8_t dividend) +{ + const uint32_t fast_hi_bit = 1ul << 31; + const uint8_t byte_hi_bit = 1u << 7; + uint32_t remainder = 0; + + // go through all the dividend's bits + for ( uint8_t mask = byte_hi_bit ; mask ; mask >>= 1 ) { + // check if divisor fits + if ( dividend & mask ) { + remainder ^= fast_hi_bit; + } + + // do polynominal division + if ( remainder & fast_hi_bit ) { + remainder <<= 1; + remainder ^= 0x04C11DB7; + } else { + remainder <<= 1; + } + } + return Bits::reverse( remainder ); +} + +} diff --git a/vespalib/src/vespa/vespalib/util/crc.h b/vespalib/src/vespa/vespalib/util/crc.h new file mode 100644 index 00000000000..f5acafeff36 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/crc.h @@ -0,0 +1,34 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/util/bits.h> + +namespace vespalib { + +/** + * @brief Crc32 class + * + * This has fast Crc32 calculation base on table lookup. + **/ +class crc_32_type +{ +public: + crc_32_type() : _c(uint32_t(-1)) { } + void process_block(const void *start, const void *end) { process_bytes(start, (const uint8_t *)end-(const uint8_t *)start); } + void process_bytes(const void *start, size_t sz); + uint32_t checksum() const { return _c ^ uint32_t(-1); } + static uint32_t crc(const void * v, size_t sz); +private: + uint32_t _c; + class CrcTableInit + { + public: + CrcTableInit(); + static uint32_t crc(uint8_t v); + }; + static uint32_t _crc[256]; + static CrcTableInit _crcTableInit; +}; + +} + diff --git a/vespalib/src/vespa/vespalib/util/document_runnable.cpp b/vespalib/src/vespa/vespalib/util/document_runnable.cpp new file mode 100644 index 00000000000..d7534514f41 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/document_runnable.cpp @@ -0,0 +1,98 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "document_runnable.h" +#include <vespa/vespalib/util/exceptions.h> +#include <cassert> + +namespace document { + +Runnable::Runnable() + : _stateLock(), + _stateCond(), + _state(NOT_RUNNING) +{ +} + +Runnable::~Runnable() { + std::lock_guard monitorGuard(_stateLock); + assert(getState() == NOT_RUNNING); +} + +bool Runnable::start(FastOS_ThreadPool& pool) +{ + std::unique_lock guard(_stateLock); + _stateCond.wait(guard, [&](){ return (getState() != STOPPING);}); + + if (getState() != NOT_RUNNING) return false; + set_state(STARTING); + if (pool.NewThread(this) == nullptr) { + throw vespalib::IllegalStateException("Failed starting a new thread", VESPA_STRLOC); + } + return true; +} + +void Runnable::set_state(State new_state) noexcept +{ + _state.store(new_state, std::memory_order_relaxed); +} + +bool Runnable::stopping() const noexcept +{ + State s(getState()); + return (s == STOPPING) || (s == RUNNING && GetThread()->GetBreakFlag()); +} + +bool Runnable::running() const noexcept +{ + State s(getState()); + // Must check break-flag too, as threadpool will use that to close + // down. + return (s == STARTING || (s == RUNNING && !GetThread()->GetBreakFlag())); +} + +bool Runnable::stop() +{ + std::lock_guard monitor(_stateLock); + if (getState() == STOPPING || getState() == NOT_RUNNING) return false; + GetThread()->SetBreakFlag(); + set_state(STOPPING); + return onStop(); +} + +bool Runnable::onStop() +{ + return true; +} + +bool Runnable::join() const +{ + std::unique_lock guard(_stateLock); + assert ((getState() != STARTING) && (getState() != RUNNING)); + _stateCond.wait(guard, [&](){ return (getState() == NOT_RUNNING);}); + return true; +} + +void Runnable::Run(FastOS_ThreadInterface*, void*) +{ + { + std::lock_guard guard(_stateLock); + // Don't set state if its already at stopping. (And let run() be + // called even though about to stop for consistency) + if (getState() == STARTING) { + set_state(RUNNING); + } + } + + // By not catching exceptions, they should abort whole application. + // We should thus not need to have a catch-all to set state to not + // running. + run(); + + { + std::lock_guard guard(_stateLock); + set_state(NOT_RUNNING); + _stateCond.notify_all(); + } +} + +} diff --git a/vespalib/src/vespa/vespalib/util/document_runnable.h b/vespalib/src/vespa/vespalib/util/document_runnable.h new file mode 100644 index 00000000000..5ca344ea7ef --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/document_runnable.h @@ -0,0 +1,94 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @class document::Runnable + * @ingroup util + * + * @brief Implementation of FastOS_Runnable that implements threadsafe stop. + * + * FastOS_Runnable can easily be used unsafe. If you use the thread pointer for + * anything after your runnable had returned from Run(), it could affect another + * runnable now using that thread. + * + * Using this class should be foolproof to avoid synchronization issues during + * thread starting and stopping :) + * + * @author H�kon Humberset + * @date 2005-09-19 + */ + +#pragma once + +#include <vespa/fastos/thread.h> +#include <atomic> + +namespace document { + +class Runnable : private FastOS_Runnable { +public: + enum State { NOT_RUNNING, STARTING, RUNNING, STOPPING }; + +private: + mutable std::mutex _stateLock; + mutable std::condition_variable _stateCond; + std::atomic<State> _state; + + void Run(FastOS_ThreadInterface*, void*) override; + void set_state(State new_state) noexcept; // _stateLock must be held +public: + /** + * Create a runnable. + * @param pool If set, runnable will be started in constructor. + */ + Runnable(); + ~Runnable() override; + + /** + * Start this runnable. + * @param pool The threadpool from which a thread is acquired. + * @return True if thread was started, false if thread was already running. + */ + bool start(FastOS_ThreadPool& pool); + + /** + * Stop this runnable. + * @return True if thread was stopped, false if thread was not running. + */ + bool stop(); + + /** + * Called in stop(). Implement, to for instance notify any monitors that + * can be waiting. + */ + virtual bool onStop(); + + /** + * Wait for this thread to finish, if it is in the process of stopping. + * @return True if thread finished (or not running), false if thread is + * running normally and no stop is scheduled. + */ + bool join() const; + + /** + * Implement this to make the runnable actually do something. + */ + virtual void run() = 0; + + /** + * Get the current state of this runnable. + * Thread safe (but relaxed) read; may be stale if done outside _stateLock. + */ + [[nodiscard]] State getState() const noexcept { + return _state.load(std::memory_order_relaxed); + } + + /** Check if system is in the process of stopping. */ + [[nodiscard]] bool stopping() const noexcept; + + /** + * Checks if runnable is running or not. (Started is considered running) + */ + [[nodiscard]] bool running() const noexcept; +}; + +} + diff --git a/vespalib/src/vespa/vespalib/util/doom.cpp b/vespalib/src/vespa/vespalib/util/doom.cpp new file mode 100644 index 00000000000..93ebe317ae0 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/doom.cpp @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "doom.h" + +namespace vespalib { + +Doom::Doom(const Clock &clock, steady_time softDoom, + steady_time hardDoom, bool explicitSoftDoom) + : _clock(clock), + _softDoom(softDoom), + _hardDoom(hardDoom), + _isExplicitSoftDoom(explicitSoftDoom) +{ } + +}
\ No newline at end of file diff --git a/vespalib/src/vespa/vespalib/util/doom.h b/vespalib/src/vespa/vespalib/util/doom.h new file mode 100644 index 00000000000..e7db0795fb7 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/doom.h @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "clock.h" + +namespace vespalib { + +class Doom { +public: + Doom(const Clock &clock, steady_time doom) + : Doom(clock, doom, doom, false) + {} + Doom(const Clock &clock, steady_time softDoom, + steady_time hardDoom, bool explicitSoftDoom); + + bool soft_doom() const { return (_clock.getTimeNS() > _softDoom); } + bool hard_doom() const { return (_clock.getTimeNS() > _hardDoom); } + duration soft_left() const { return _softDoom - _clock.getTimeNS(); } + duration hard_left() const { return _hardDoom - _clock.getTimeNS(); } + bool isExplicitSoftDoom() const { return _isExplicitSoftDoom; } +private: + const Clock &_clock; + steady_time _softDoom; + steady_time _hardDoom; + bool _isExplicitSoftDoom; +}; + +} diff --git a/vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp b/vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp new file mode 100644 index 00000000000..424ad7ba470 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp @@ -0,0 +1,88 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "growablebytebuffer.h" +#include <arpa/inet.h> + +using namespace vespalib; + +GrowableByteBuffer::GrowableByteBuffer(uint32_t initialLen) : + _buffer(Alloc::alloc(initialLen)), + _position(0) +{ +} + +char* +GrowableByteBuffer::allocate(uint32_t len) +{ + size_t need(_position + len); + if (need > _buffer.size()) { + uint32_t newSize = vespalib::roundUp2inN(need); + Alloc newBuf(Alloc::alloc(newSize)); + memcpy(newBuf.get(), _buffer.get(), _position); + _buffer.swap(newBuf); + } + + char* pos = static_cast<char *>(_buffer.get()) + _position; + _position += len; + return pos; +} + +void +GrowableByteBuffer::putBytes(const void * buffer, uint32_t length) +{ + char* buf = allocate(length); + memcpy(buf, buffer, length); +} + +void +GrowableByteBuffer::putShort(uint16_t v) +{ + uint16_t val = htons(v); + putBytes(reinterpret_cast<const char*>(&val), sizeof(v)); +} + +void +GrowableByteBuffer::putInt(uint32_t v) +{ + uint32_t val = htonl(v); + putBytes(reinterpret_cast<const char*>(&val), sizeof(v)); +} + +void +GrowableByteBuffer::putReverse(const char* buffer, uint32_t length) +{ + char* buf = allocate(length); + for (uint32_t i = 0; i < length; i++) { + buf[(length - i - 1)] = buffer[i]; + } +} + +void +GrowableByteBuffer::putLong(uint64_t v) +{ + putReverse(reinterpret_cast<const char*>(&v), sizeof(v)); +} + +void +GrowableByteBuffer::putDouble(double v) +{ + putReverse(reinterpret_cast<const char*>(&v), sizeof(v)); +} + +void +GrowableByteBuffer::putString(vespalib::stringref v) +{ + putInt(v.size()); + putBytes(v.data(), v.size()); +} + +void +GrowableByteBuffer::putByte(uint8_t v) +{ + putBytes(reinterpret_cast<const char*>(&v), sizeof(v)); +} + +void +GrowableByteBuffer::putBoolean(bool v) +{ + putByte(v); +} diff --git a/vespalib/src/vespa/vespalib/util/growablebytebuffer.h b/vespalib/src/vespa/vespalib/util/growablebytebuffer.h new file mode 100644 index 00000000000..b0fb30606d4 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/growablebytebuffer.h @@ -0,0 +1,94 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/util/alloc.h> +#include <vespa/vespalib/stllike/string.h> + +namespace vespalib { + +/** +Class that wraps a vector<char> and resizes it as necessary to hold data +put into it. Also has several utility functions commonly used in +data serialization. + +All of the numeric helper methods (putInt/Long/Double etc.) in this class +store the numbers in network byte order. +*/ +class GrowableByteBuffer +{ +public: + /** + Creates a new GrowableByteBuffer with the given initial length and grow factor. + */ + GrowableByteBuffer(uint32_t initialLen = 256); + + /** + If necessary, grows the buffer so it can hold the specified number of bytes, + then moves the position to after that length, and returns a pointer to the previous + position. + + @param len The length to allocate + @return Returns a pointer to a buffer that the user can write data to. + */ + char* allocate(uint32_t len); + + /** + Returns a pointer to the start of the allocated buffer. + */ + const char* getBuffer() const { return static_cast<const char *>(_buffer.get()); } + + /** + Returns the current position. + */ + uint32_t position() const { return _position; } + + /** + Adds the given buffer to this buffer. + */ + void putBytes(const void * buffer, uint32_t length); + + /** + Adds a short to the buffer. + */ + void putShort(uint16_t v); + + /** + Adds an int to the buffer. + */ + void putInt(uint32_t v); + + /** + Adds a long to the buffer. + */ + void putLong(uint64_t v); + + /** + Adds a double to the buffer. + */ + void putDouble(double v); + + /** + Adds a string to the buffer. + */ + void putString(vespalib::stringref v); + + /** + Adds a single byte to the buffer. + */ + void putByte(uint8_t v); + + /** + Adds a boolean to the buffer. + */ + void putBoolean(bool v); + +private: + void putReverse(const char* buffer, uint32_t length); + using Alloc = vespalib::alloc::Alloc; + Alloc _buffer; + + uint32_t _position; +}; + +} + diff --git a/vespalib/src/vespa/vespalib/util/jsonexception.cpp b/vespalib/src/vespa/vespalib/util/jsonexception.cpp new file mode 100644 index 00000000000..5c04aefe2d1 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/jsonexception.cpp @@ -0,0 +1,18 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "jsonexception.h" + +namespace vespalib { + +VESPA_IMPLEMENT_EXCEPTION_SPINE(JsonStreamException); + +JsonStreamException::JsonStreamException(stringref reason, stringref history, + stringref location, int skipStack) + : Exception(reason + (history.empty() ? "" : "\nHistory:\n" + history), + location, skipStack + 1), + _reason(reason) +{ } + +JsonStreamException::~JsonStreamException() { } + +} diff --git a/vespalib/src/vespa/vespalib/util/jsonexception.h b/vespalib/src/vespa/vespalib/util/jsonexception.h new file mode 100644 index 00000000000..3bf954420a6 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/jsonexception.h @@ -0,0 +1,19 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/util/exception.h> + +namespace vespalib { + +class JsonStreamException : public Exception { + string _reason; +public: + JsonStreamException(stringref reason, + stringref history, + stringref location, int skipStack = 0); + stringref getReason() const { return _reason; } + VESPA_DEFINE_EXCEPTION_SPINE(JsonStreamException); + ~JsonStreamException(); +}; + +} diff --git a/vespalib/src/vespa/vespalib/util/jsonstream.cpp b/vespalib/src/vespa/vespalib/util/jsonstream.cpp new file mode 100644 index 00000000000..135611e975f --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/jsonstream.cpp @@ -0,0 +1,364 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "jsonstream.h" +#include "jsonexception.h" +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/stringfmt.h> + +namespace vespalib { + +const char* +JsonStream::getStateName(const State& s) { + switch (s) { + case State::OBJECT_EXPECTING_KEY: return "ObjectExpectingKey"; + case State::OBJECT_EXPECTING_VALUE: return "ObjectExpectingValue"; + case State::ARRAY: return "ArrayExpectingValue"; + case State::ROOT: return "RootExpectingArrayOrObjectStart"; + } + throw IllegalStateException("Control should not reach this point", VESPA_STRLOC); +} + +JsonStream::JsonStream(asciistream& as, bool createIndents) + : _writer(as) +{ + if (createIndents) _writer.setPretty(); + push({State::ROOT}); +} + +JsonStream::~JsonStream() {} + +JsonStream& +JsonStream::operator<<(stringref value) +{ + if (_state.empty()) { + fail("Stream already finalized. Can't add a string value."); + } + switch (top().state) { + case State::OBJECT_EXPECTING_KEY: { + _writer.appendKey(value); + top() = {State::OBJECT_EXPECTING_VALUE, value}; + break; + } + case State::OBJECT_EXPECTING_VALUE: { + _writer.appendString(value); + top().state = State::OBJECT_EXPECTING_KEY; + break; + } + case State::ARRAY: { + _writer.appendString(value); + ++top().array_index; + break; + } + case State::ROOT: { + _writer.appendString(value); + pop(); + break; + } + } + return *this; +} + +JsonStream& +JsonStream::operator<<(bool value) +{ + if (_state.empty()) { + fail("Stream already finalized. Can't add a bool value."); + } + switch (top().state) { + case State::OBJECT_EXPECTING_KEY: { + fail("A bool value cannot be an object key"); + break; + } + case State::OBJECT_EXPECTING_VALUE: { + _writer.appendBool(value); + top().state = State::OBJECT_EXPECTING_KEY; + break; + } + case State::ARRAY: { + _writer.appendBool(value); + ++top().array_index; + break; + } + case State::ROOT: { + _writer.appendBool(value); + pop(); + break; + } + } + return *this; +} + +JsonStream& +JsonStream::operator<<(double value) +{ + if (_state.empty()) { + fail("Stream already finalized. Can't add a double value."); + } + switch (top().state) { + case State::OBJECT_EXPECTING_KEY: { + fail("A double value cannot be an object key"); + break; + } + case State::OBJECT_EXPECTING_VALUE: { + _writer.appendDouble(value); + top().state = State::OBJECT_EXPECTING_KEY; + break; + } + case State::ARRAY: { + _writer.appendDouble(value); + ++top().array_index; + break; + } + case State::ROOT: { + _writer.appendDouble(value); + pop(); + break; + } + } + return *this; +} + +JsonStream& +JsonStream::operator<<(float value) +{ + if (_state.empty()) { + fail("Stream already finalized. Can't add a float value."); + } + switch (top().state) { + case State::OBJECT_EXPECTING_KEY: { + fail("A float value cannot be an object key"); + break; + } + case State::OBJECT_EXPECTING_VALUE: { + _writer.appendFloat(value); + top().state = State::OBJECT_EXPECTING_KEY; + break; + } + case State::ARRAY: { + _writer.appendFloat(value); + ++top().array_index; + break; + } + case State::ROOT: { + _writer.appendDouble(value); + pop(); + break; + } + } + return *this; +} + +JsonStream& +JsonStream::operator<<(long long value) +{ + if (_state.empty()) { + fail("Stream already finalized. Can't add a long long value."); + } + switch (top().state) { + case State::OBJECT_EXPECTING_KEY: { + fail("An int64_t value cannot be an object key"); + break; + } + case State::OBJECT_EXPECTING_VALUE: { + _writer.appendInt64(value); + top().state = State::OBJECT_EXPECTING_KEY; + break; + } + case State::ARRAY: { + _writer.appendInt64(value); + ++top().array_index; + break; + } + case State::ROOT: { + _writer.appendInt64(value); + pop(); + break; + } + } + return *this; +} + +JsonStream& +JsonStream::operator<<(unsigned long long value) +{ + if (_state.empty()) { + fail("Stream already finalized. Can't add an unsigned long long value."); + } + switch (top().state) { + case State::OBJECT_EXPECTING_KEY: { + fail("A uint64_t value cannot be an object key"); + break; + } + case State::OBJECT_EXPECTING_VALUE: { + _writer.appendUInt64(value); + top().state = State::OBJECT_EXPECTING_KEY; + break; + } + case State::ARRAY: { + _writer.appendUInt64(value); + ++top().array_index; + break; + } + case State::ROOT: { + _writer.appendUInt64(value); + pop(); + break; + } + } + return *this; +} + +JsonStream& +JsonStream::operator<<(const Object&) +{ + if (_state.empty()) { + fail("Stream already finalized. Can't start a new object."); + } + switch (top().state) { + case State::OBJECT_EXPECTING_KEY: { + fail("An object value cannot be an object key"); + break; + } + case State::OBJECT_EXPECTING_VALUE: { + _writer.beginObject(); + top().state = State::OBJECT_EXPECTING_KEY; + push({State::OBJECT_EXPECTING_KEY, ""}); + break; + } + case State::ARRAY: { + _writer.beginObject(); + push({State::OBJECT_EXPECTING_KEY, ""}); + break; + } + case State::ROOT: { + _writer.beginObject(); + top() = {State::OBJECT_EXPECTING_KEY, ""}; + break; + } + } + return *this; +} + +JsonStream& +JsonStream::operator<<(const Array&) +{ + if (_state.empty()) { + fail("Stream already finalized. Can't start a new array."); + } + switch (top().state) { + case State::OBJECT_EXPECTING_KEY: { + fail("An array value cannot be an object key"); + break; + } + case State::OBJECT_EXPECTING_VALUE: { + _writer.beginArray(); + top().state = State::OBJECT_EXPECTING_KEY; + push({State::ARRAY}); + break; + } + case State::ARRAY: { + _writer.beginArray(); + push({State::ARRAY}); + break; + } + case State::ROOT: { + _writer.beginArray(); + top() = {State::ARRAY}; + break; + } + } + return *this; +} + +JsonStream& +JsonStream::operator<<(const End&) +{ + if (_state.empty()) { + fail("Stream already finalized. Can't end it."); + } + switch (top().state) { + case State::OBJECT_EXPECTING_KEY: { + _writer.endObject(); + pop(); + break; + } + case State::OBJECT_EXPECTING_VALUE: { + fail("Object got key but not value. Cannot end it now"); + break; + } + case State::ARRAY: { + _writer.endArray(); + pop(); + break; + } + case State::ROOT: { + fail("No tag to end. At root"); + break; + } + } + if (!_state.empty() && top().state == State::ARRAY) { + ++top().array_index; + } + return *this; +} + +JsonStream& +JsonStream::finalize() +{ + while (!_state.empty()) { + operator<<(End()); + } + return *this; +} + +string +JsonStream::getStateString() const +{ + asciistream as; + for (auto it(_state.begin()), mt(_state.end()); it != mt; it++) { + switch (it->state) { + case State::OBJECT_EXPECTING_KEY: + case State::OBJECT_EXPECTING_VALUE: { + as << "{" << it->object_key << "}"; + break; + } + case State::ARRAY: { + as << "["; + if (it->array_index != 0) { + as << (it->array_index - 1); + } + as << "]"; + break; + } + case State::ROOT: { + break; + } + } + } + if (_state.empty()) { + as << "Finalized"; + } else { + as << "(" << getStateName(_state.back().state) << ")"; + } + return as.str(); +} + +string +JsonStream::getJsonStreamState() const +{ + asciistream report; + report << "Current: " << getStateString(); + return report.str(); +} + +void +JsonStream::fail(stringref error) const +{ + asciistream report; + report << "Invalid state on call: " << error + << " (" << getStateString() << ")"; + throw JsonStreamException(report.str(), "", VESPA_STRLOC); +} + +} diff --git a/vespalib/src/vespa/vespalib/util/jsonstream.h b/vespalib/src/vespa/vespalib/util/jsonstream.h new file mode 100644 index 00000000000..d60151f0478 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/jsonstream.h @@ -0,0 +1,101 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +/** + * An overbuild of the json writer, making the code writing the json looking + * neater. Also allows templates to use it with unknown (but supported) types, + * letting the compiler take care of calling the correct function rather than + * having to resort to template specialization. + */ + +#include <vespa/vespalib/util/jsonwriter.h> + +namespace vespalib { + +// Inherit to refer to types without namespace prefix in header file. +struct JsonStreamTypes { + class Object {}; + class Array {}; + class End {}; +}; +// Use namespace in function to avoid prefixing namespace everywhere. +namespace jsonstream { + typedef JsonStreamTypes::Object Object; + typedef JsonStreamTypes::Array Array; + typedef JsonStreamTypes::End End; +} + +// We can disable this if it ends up being a performance issue. +// Really useful to explain what code bits have tried to write invalid json +// though. +#define TRACK_JSON_CREATION_TO_CREATE_EASY_TO_DEBUG_ERROR_MESSAGES 1 + +class JsonStream : public JsonStreamTypes { + JSONWriter _writer; + enum class State { + ROOT, + OBJECT_EXPECTING_KEY, + OBJECT_EXPECTING_VALUE, + ARRAY + }; + static const char* getStateName(const State&); + struct StateEntry { + State state; + string object_key; + size_t array_index; + + StateEntry() noexcept + : state(State::ROOT), object_key(""), array_index(size_t(0)) {} + StateEntry(State s) + : state(s), object_key(""), array_index(size_t(0)) {} + StateEntry(State s, stringref key) + : state(s), object_key(key), array_index(size_t(0)) {} + }; + std::vector<StateEntry> _state; + + StateEntry & top() { return _state.back(); } + const StateEntry & top() const { return _state.back(); } + void pop() { _state.resize(_state.size() - 1); } + void push(const StateEntry & e) { _state.push_back(e); } +public: + JsonStream(asciistream&, bool createIndents = false); + JsonStream(const JsonStream&) = delete; + JsonStream& operator=(const JsonStream&) = delete; + JsonStream(JsonStream &&) = default; + JsonStream& operator=(JsonStream &&) = default; + ~JsonStream(); + + JsonStream& operator<<(stringref); + JsonStream& operator<<(bool); + JsonStream& operator<<(double); + JsonStream& operator<<(float); // Less precision that double + JsonStream& operator<<(long long); + JsonStream& operator<<(unsigned long long); + JsonStream& operator<<(const Object&); + JsonStream& operator<<(const Array&); + JsonStream& operator<<(const End&); + + // Additional functions provided to let compiler work out correct + // function without requiring user to cast their value + JsonStream& operator<<(unsigned long v) + { return operator<<(static_cast<unsigned long long>(v)); } + JsonStream& operator<<(unsigned int v) + { return operator<<(static_cast<unsigned long long>(v)); } + JsonStream& operator<<(long v) + { return operator<<(static_cast<long long>(v)); } + JsonStream& operator<<(int v) + { return operator<<(static_cast<long long>(v)); } + JsonStream& operator<<(const char* c) + { return operator<<(stringref(c)); } + + JsonStream& finalize(); + + vespalib::string getJsonStreamState() const; + +private: + string getStateString() const; + void fail(stringref error) const; +}; + +} // vespalib + diff --git a/vespalib/src/vespa/vespalib/util/malloc_mmap_guard.cpp b/vespalib/src/vespa/vespalib/util/malloc_mmap_guard.cpp new file mode 100644 index 00000000000..67181dfd16f --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/malloc_mmap_guard.cpp @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "malloc_mmap_guard.h" +#include <vespa/vespalib/util/size_literals.h> +#ifdef __linux__ +#include <malloc.h> +#endif +#include <limits> +#include <cassert> + +namespace vespalib { + +MallocMmapGuard::MallocMmapGuard(size_t mmapLimit) : + _threadId(std::this_thread::get_id()) +{ +#ifdef __linux__ + int limit = mmapLimit <= std::numeric_limits<int>::max() ? mmapLimit : std::numeric_limits<int>::max(); + mallopt(M_MMAP_THRESHOLD, limit); +#else + (void) mmapLimit; +#endif +} + +MallocMmapGuard::~MallocMmapGuard() +{ + assert(_threadId == std::this_thread::get_id()); +#ifdef __linux__ + mallopt(M_MMAP_THRESHOLD, 1_Gi); +#endif +} + +} diff --git a/vespalib/src/vespa/vespalib/util/malloc_mmap_guard.h b/vespalib/src/vespa/vespalib/util/malloc_mmap_guard.h new file mode 100644 index 00000000000..03e6d38c03c --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/malloc_mmap_guard.h @@ -0,0 +1,28 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <thread> + +namespace vespalib { + +/** + * Provides a hint to malloc implementation that all allocations in the scope of this guard + * will use mmap directly for allocation larger than the given limit. + * NB !! Note that guards can not be nested. Intention is to use around third party libraries where + * you do not control allocation yourself. + * The effect is implementation dependent. vespamalloc applies this only for the calling thread. + **/ +class MallocMmapGuard +{ +public: + MallocMmapGuard(size_t mmapLimit); + MallocMmapGuard(const MallocMmapGuard &) = delete; + MallocMmapGuard & operator=(const MallocMmapGuard &) = delete; + MallocMmapGuard(MallocMmapGuard &&) = delete; + MallocMmapGuard & operator=(MallocMmapGuard &&) = delete; + ~MallocMmapGuard(); +private: + std::thread::id _threadId; +}; + +} // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/util/polymorphicarray.h b/vespalib/src/vespa/vespalib/util/polymorphicarray.h new file mode 100644 index 00000000000..de93ae03657 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/polymorphicarray.h @@ -0,0 +1,82 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// +#pragma once + +#include "polymorphicarraybase.h" + +namespace vespalib { + +/** + * Describes an interface an array of polymorphic types. + * The intention is to allow efficient implementations when that is possible + * while still enjoying the flexibility of the polymorph interface. + * It is not a full feldged Array implementation as std::vector. It contains just + * the minimum required to allow for efficient implementations for document::ArrayFieldValue. + * + * You specify the base type the interface shall provide. This base type must define + * virtual void assign(const B & rhs); + * For use with ComplexArrayT your type also need + * virtual T * clone() const; + */ +template<typename B> +class IArrayT : public IArrayBase { +public: + class iterator { + public: + iterator(IArrayT &a, size_t i) : _a(&a), _i(i) {} + iterator operator+(size_t diff) const { return iterator(*_a, _i + diff); } + iterator &operator++() { + ++_i; + return *this; + } + iterator operator++(int) { + iterator other(*this); + ++_i; + return other; + } + bool operator==(const iterator &other) const { return (_a == other._a) && (_i == other._i); } + bool operator!=(const iterator &other) const { return (_i != other._i) || (_a != other._a); } + B &operator*() { return (*_a)[_i]; } + B *operator->() { return &(*_a)[_i]; } + friend ssize_t operator-(const iterator &a, const iterator &b) { return a._i - b._i; } + private: + IArrayT *_a; + size_t _i; + }; + + class const_iterator { + public: + const_iterator(const IArrayT &a, size_t i) : _a(&a), _i(i) {} + const_iterator operator+(size_t diff) const { return const_iterator(*_a, _i + diff); } + const_iterator &operator++() { + ++_i; + return *this; + } + const_iterator operator++(int) { + const_iterator other(*this); + ++_i; + return other; + } + bool operator==(const const_iterator &other) const { return (_a == other._a) && (_i == other._i); } + bool operator!=(const const_iterator &other) const { return (_i != other._i) || (_a != other._a); } + const B &operator*() const { return (*_a)[_i]; } + const B *operator->() const { return &(*_a)[_i]; } + size_t operator-(const const_iterator &b) const { return _i - b._i; } + private: + const IArrayT *_a; + size_t _i; + }; + + typedef std::unique_ptr<IArrayT> UP; + virtual const B &operator[](size_t i) const = 0; + virtual B &operator[](size_t i) = 0; + virtual IArrayT *clone() const override = 0; + virtual iterator erase(iterator it) = 0; + virtual const_iterator begin() const { return const_iterator(*this, 0); } + virtual const_iterator end() const { return const_iterator(*this, size()); } + virtual iterator begin() { return iterator(*this, 0); } + virtual iterator end() { return iterator(*this, size()); } + virtual void push_back(const B &v) = 0; +}; + +} diff --git a/vespalib/src/vespa/vespalib/util/polymorphicarraybase.h b/vespalib/src/vespa/vespalib/util/polymorphicarraybase.h new file mode 100644 index 00000000000..f3f63f6a43c --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/polymorphicarraybase.h @@ -0,0 +1,18 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// +#pragma once + +namespace vespalib { + +class IArrayBase { +public: + virtual ~IArrayBase() {} + virtual void resize(size_t sz) = 0; + virtual void reserve(size_t sz) = 0; + virtual void clear() = 0; + virtual IArrayBase *clone() const = 0; + virtual size_t size() const = 0; + bool empty() const { return size() == 0; } +}; + +} diff --git a/vespalib/src/vespa/vespalib/util/polymorphicarrays.h b/vespalib/src/vespa/vespalib/util/polymorphicarrays.h new file mode 100644 index 00000000000..55f64ec51cd --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/polymorphicarrays.h @@ -0,0 +1,71 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// +#pragma once + +#include "polymorphicarray.h" +#include <vespa/vespalib/util/memory.h> +#include <vector> + +namespace vespalib { + +template <typename T, typename B> +class PrimitiveArrayT final : public IArrayT<B> +{ + using typename IArrayT<B>::iterator; +public: + PrimitiveArrayT() : _array() { } + ~PrimitiveArrayT() { } + const T & operator [] (size_t i) const override { return _array[i]; } + T & operator [] (size_t i) override { return _array[i]; } + void resize(size_t sz) override { _array.resize(sz); } + void reserve(size_t sz) override { _array.reserve(sz); } + void clear() override { _array.clear(); } + IArrayT<B> * clone() const override { return new PrimitiveArrayT<T, B>(*this); } + size_t size() const override { return _array.size(); } + iterator erase(iterator it) override { _array.erase(_array.begin() + (it - this->begin())); return it; } + void push_back(const B & v) override { + _array.emplace_back(); + _array.back().assign(v); + } +private: + std::vector<T> _array; +}; + +template <typename B> +class ComplexArrayT final : public IArrayT<B> +{ + using typename IArrayT<B>::iterator; +public: + class Factory { + public: + typedef std::unique_ptr<Factory> UP; + typedef vespalib::CloneablePtr<Factory> CP; + virtual B * create() = 0; + virtual Factory * clone() const = 0; + virtual ~Factory() { } + }; + explicit ComplexArrayT(typename Factory::UP factory) : _array(), _factory(factory.release()) { } + ~ComplexArrayT() { } + const B & operator [] (size_t i) const override { return *_array[i]; } + B & operator [] (size_t i) override { return *_array[i]; } + void resize(size_t sz) override { + _array.resize(sz); + for (auto & cp : _array) { + if ( cp.get() == nullptr) { + cp.reset(_factory->create()); + } + } + } + void reserve(size_t sz) override { _array.reserve(sz); } + void clear() override { _array.clear(); } + IArrayT<B> * clone() const override { return new ComplexArrayT<B>(*this); } + size_t size() const override { return _array.size(); } + iterator erase(iterator it) override { _array.erase(_array.begin() + (it - this->begin())); return it; } + void push_back(const B & v) override { _array.push_back(v.clone()); } +private: + typedef vespalib::CloneablePtr<B> CP; + std::vector<CP> _array; + typename Factory::CP _factory; +}; + +} diff --git a/vespalib/src/vespa/vespalib/util/process_memory_stats.cpp b/vespalib/src/vespa/vespalib/util/process_memory_stats.cpp new file mode 100644 index 00000000000..f7e8e087727 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/process_memory_stats.cpp @@ -0,0 +1,204 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "process_memory_stats.h" +#include <vespa/vespalib/stllike/asciistream.h> +#include <algorithm> +#include <vector> + +#include <vespa/log/log.h> + +LOG_SETUP(".vespalib.util.process_memory_stats"); + +namespace vespalib { + +namespace { + +#ifdef __linux__ +/* + * Check if line specifies an address range. + * + * address perms offset dev inode pathname + * + * 00400000-00420000 r-xp 00000000 fd:04 16545041 /usr/bin/less + */ + +bool +isRange(vespalib::stringref line) { + for (char c : line) { + if (c == ' ') { + return true; + } + if (c == ':') { + return false; + } + } + return false; +} + + +/* + * Check if address range is anonymous, e.g. not mapped from file. + * inode number is 0 in that case. + * + * address perms offset dev inode pathname + * + * 00400000-00420000 r-xp 00000000 fd:04 16545041 /usr/bin/less + * 00625000-00628000 rw-p 00000000 00:00 0 + * + * The range starting at 00400000 is not anonymous. + * The range starting at 00625000 is anonymous. + */ + +bool +isAnonymous(vespalib::stringref line) { + int delims = 0; + for (char c : line) { + if (delims >= 4) { + return (c == '0'); + } + if (c == ' ') { + ++delims; + } + } + return true; +} + + +/* + * Lines not containing an address range contains a header and a + * value, e.g. + * + * Size: 128 kB + * Rss: 96 kB + * Anonymous: 0 kB + * + * The lines with header Anonymous are ignored, thus anonymous pages + * caused by mmap() of a file with MAP_PRIVATE flags are counted as + * mapped pages. + */ + +vespalib::stringref +getLineHeader(vespalib::stringref line) +{ + return line.substr(0, line.find(':')); +} +#endif + +} + +ProcessMemoryStats +ProcessMemoryStats::createStatsFromSmaps() +{ + ProcessMemoryStats ret; +#ifdef __linux__ + asciistream smaps = asciistream::createFromDevice("/proc/self/smaps"); + bool anonymous = true; + uint64_t lineVal = 0; + while (!smaps.eof()) { + string backedLine = smaps.getline(); + stringref line(backedLine); + if (isRange(line)) { + ret._mappings_count += 1; + anonymous = isAnonymous(line); + } else if (!line.empty()) { + stringref lineHeader = getLineHeader(line); + if (lineHeader == "Size") { + asciistream is(line.substr(lineHeader.size() + 1)); + is >> lineVal; + if (anonymous) { + ret._anonymous_virt += lineVal * 1024; + } else { + ret._mapped_virt += lineVal * 1024; + } + } else if (lineHeader == "Rss") { + asciistream is(line.substr(lineHeader.size() + 1)); + is >> lineVal; + if (anonymous) { + ret._anonymous_rss += lineVal * 1024; + } else { + ret._mapped_rss += lineVal * 1024; + } + } + } + } +#endif + return ret; +} + + +ProcessMemoryStats::ProcessMemoryStats() + : _mapped_virt(0), + _mapped_rss(0), + _anonymous_virt(0), + _anonymous_rss(0), + _mappings_count(0) +{ +} + +ProcessMemoryStats::ProcessMemoryStats(uint64_t mapped_virt, + uint64_t mapped_rss, + uint64_t anonymous_virt, + uint64_t anonymous_rss, + uint64_t mappings_cnt) + : _mapped_virt(mapped_virt), + _mapped_rss(mapped_rss), + _anonymous_virt(anonymous_virt), + _anonymous_rss(anonymous_rss), + _mappings_count(mappings_cnt) +{ +} + +namespace { + +bool +similar(uint64_t lhs, uint64_t rhs, uint64_t epsilon) +{ + return (lhs < rhs) ? ((rhs - lhs) <= epsilon) : ((lhs - rhs) <= epsilon); +} + +} + +bool +ProcessMemoryStats::similarTo(const ProcessMemoryStats &rhs, uint64_t sizeEpsilon) const +{ + return similar(_mapped_virt, rhs._mapped_virt, sizeEpsilon) && + similar(_mapped_rss, rhs._mapped_rss, sizeEpsilon) && + similar(_anonymous_virt, rhs._anonymous_virt, sizeEpsilon) && + similar(_anonymous_rss, rhs._anonymous_rss, sizeEpsilon) && + (_mappings_count == rhs._mappings_count); +} + +vespalib::string +ProcessMemoryStats::toString() const +{ + vespalib::asciistream stream; + stream << "_mapped_virt=" << _mapped_virt << ", " + << "_mapped_rss=" << _mapped_rss << ", " + << "_anonymous_virt=" << _anonymous_virt << ", " + << "_anonymous_rss=" << _anonymous_rss << ", " + << "_mappings_count=" << _mappings_count; + return stream.str(); +} + +ProcessMemoryStats +ProcessMemoryStats::create(uint64_t sizeEpsilon) +{ + constexpr size_t NUM_TRIES = 3; + std::vector<ProcessMemoryStats> samples; + samples.reserve(NUM_TRIES); + samples.push_back(createStatsFromSmaps()); + for (size_t i = 0; i < NUM_TRIES; ++i) { + samples.push_back(createStatsFromSmaps()); + if (samples.back().similarTo(*(samples.rbegin()+1), sizeEpsilon)) { + return samples.back(); + } + LOG(debug, "create(): Memory stats have changed, trying to read smaps file again: i=%zu, prevStats={%s}, currStats={%s}", + i, (samples.rbegin()+1)->toString().c_str(), samples.back().toString().c_str()); + } + std::sort(samples.begin(), samples.end()); + LOG(debug, "We failed to find 2 consecutive samples that where similar with epsilon of %" PRIu64 ".\nSmallest is '%s',\n median is '%s',\n largest is '%s'", + sizeEpsilon, samples.front().toString().c_str(), samples[samples.size()/2].toString().c_str(), samples.back().toString().c_str()); + return samples[samples.size()/2]; +} + +} diff --git a/vespalib/src/vespa/vespalib/util/process_memory_stats.h b/vespalib/src/vespa/vespalib/util/process_memory_stats.h new file mode 100644 index 00000000000..000c0942905 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/process_memory_stats.h @@ -0,0 +1,45 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +namespace vespalib { + +/* + * Class for linux specific way to get memory stats for current process. + */ +class ProcessMemoryStats +{ + uint64_t _mapped_virt; // virtual size + uint64_t _mapped_rss; // resident size + uint64_t _anonymous_virt; // virtual size + uint64_t _anonymous_rss; // resident size + uint64_t _mappings_count; // number of mappings + // (limited by sysctl vm.max_map_count) + + static ProcessMemoryStats createStatsFromSmaps(); + +public: + ProcessMemoryStats(); + /** + * Sample memory stats for the current process based on reading the file /proc/self/smaps. + * + * Samples are taken until two consecutive memory stats are similar given the size epsilon. + * This ensures a somewhat consistent memory stats snapshot. + */ + static ProcessMemoryStats create(uint64_t sizeEpsilon = 1 * 1024 * 1024); + uint64_t getMappedVirt() const { return _mapped_virt; } + uint64_t getMappedRss() const { return _mapped_rss; } + uint64_t getAnonymousVirt() const { return _anonymous_virt; } + uint64_t getAnonymousRss() const { return _anonymous_rss; } + uint64_t getMappingsCount() const { return _mappings_count; } + bool similarTo(const ProcessMemoryStats &rhs, uint64_t sizeEpsilon) const; + vespalib::string toString() const; + bool operator < (const ProcessMemoryStats & rhs) const { return _anonymous_rss < rhs._anonymous_rss; } + + /** for unit tests only */ + ProcessMemoryStats(uint64_t, uint64_t, uint64_t, uint64_t, uint64_t); +}; + +} diff --git a/vespalib/src/vespa/vespalib/util/programoptions.cpp b/vespalib/src/vespa/vespalib/util/programoptions.cpp new file mode 100644 index 00000000000..9ea7c1648d1 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/programoptions.cpp @@ -0,0 +1,799 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "programoptions.h" +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/exceptions.h> +#include <boost/lexical_cast.hpp> +#include <cassert> + +#include <vespa/log/log.h> +LOG_SETUP(".programoptions"); + +namespace vespalib { + +VESPA_IMPLEMENT_EXCEPTION(InvalidCommandLineArgumentsException, Exception); + +namespace { + + std::string UNSET_TOKEN = "-_-/#UNSET#\\-_-"; + + // Use tokenizer instead when moved from document + std::vector<std::string> splitString(const std::string& source, char split) + { + std::vector<std::string> target; + std::string::size_type start = 0; + std::string::size_type stop = source.find(split); + while (stop != std::string::npos) { + target.push_back(source.substr(start, stop - start)); + start = stop + 1; + stop = source.find(split, start); + } + target.push_back(source.substr(start)); + return target; + } + +} + +template<typename Number> +std::string ProgramOptions::NumberOptionParser<Number>::getStringValue(Number n) +{ + std::ostringstream ost; + ost << n; + return ost.str(); +} + +template<typename Number> +void ProgramOptions::NumberOptionParser<Number>::set(const std::vector<std::string>& arguments) +{ + try{ + _number = boost::lexical_cast<Number>(arguments[0]); + } catch (const boost::bad_lexical_cast& e) { + std::ostringstream ost; + ost << "The argument '" << arguments[0] + << "' can not be interpreted as a number of type " + << getTypeName<Number>() << "."; + throw InvalidCommandLineArgumentsException( + ost.str(), VESPA_STRLOC); + } +} + +ProgramOptions::FlagOptionParser::~FlagOptionParser() = default; + +ProgramOptions::StringOptionParser::~StringOptionParser() = default; + +ProgramOptions::OptionParser::OptionParser( + const std::string& nameList, uint32_t argCount, const std::string& desc) + : _names(splitString(nameList, ' ')), + _hiddenNames(), + _argCount(argCount), + _argTypes(argCount), + _hasDefault(false), + _invalidDefault(false), + _defaultString(), + _description(desc) +{ + if (nameList == "") _names.clear(); +} + +ProgramOptions::OptionParser::OptionParser( + const std::string& nameList, uint32_t argCount, + const std::string& defString, const std::string& desc) + : _names(splitString(nameList, ' ')), + _hiddenNames(), + _argCount(argCount), + _argTypes(argCount), + _hasDefault(true), + _invalidDefault(false), + _defaultString(defString), + _description(desc) +{ } + +ProgramOptions::OptionParser::~OptionParser() { } + +void +ProgramOptions::OptionParser::setInvalidDefault() +{ + _invalidDefault = true; +} + +std::string ProgramOptions::OptionParser::getOptSyntaxString() const +{ + std::ostringstream ost; + for (uint32_t i=0; i<_names.size(); ++i) { + ost << (_names[i].size() == 1 ? " -" : " --"); + ost << _names[i]; + } + for (uint32_t i=0; i<_argCount; ++i) { + std::string type = (_argTypes[i] != "" ? _argTypes[i] : getArgType(i)); + ost << " <" << type << ">"; + } + return ost.str(); +} + +ProgramOptions::ProgramOptions() + : _argc(0), + _argv(0), + _options(), + _optionMap(), + _setOptions(), + _syntaxMessage(), + _maxLeftColumnSize(30), + _defaultsSet(false) +{ } + +ProgramOptions::ProgramOptions(int argc, const char* const* argv) + : _argc(argc), + _argv(argv), + _options(), + _optionMap(), + _setOptions(), + _syntaxMessage(), + _maxLeftColumnSize(30), + _defaultsSet(false) +{ } + +ProgramOptions::~ProgramOptions() { } + +void +ProgramOptions::clear() { + _configurables.clear(); + _options.clear(); + _optionMap.clear(); + _setOptions.clear(); + _arguments.clear(); +} + +void +ProgramOptions::setCommandLineArguments(int argc, const char* const* argv) +{ + _argc = argc; + _argv = argv; +} + +void +ProgramOptions::setSyntaxMessage(const std::string& msg) +{ + _syntaxMessage = msg; +} + +void +ProgramOptions::addHiddenIdentifiers(const std::string& optionNameList) +{ + if (_options.size() == 0) { + throw InvalidCommandLineArgumentsException( + "Cannot add hidden identifier to last " + "option as no option has been added yet.", VESPA_STRLOC); + } + OptionParser::SP opt = _options.back(); + if (opt->isHeader()) { + throw InvalidCommandLineArgumentsException( + "Cannot add option arguments to option header.", VESPA_STRLOC); + } + std::vector<std::string> newIds(splitString(optionNameList, ' ')); + for (uint32_t i=0; i<newIds.size(); ++i) { + std::map<std::string, OptionParser::SP>::const_iterator it( + _optionMap.find(newIds[i])); + if (it != _optionMap.end()) { + throw InvalidCommandLineArgumentsException( + "Option '" + newIds[i] + "' is already registered.", + VESPA_STRLOC); + } + } + for (uint32_t i=0; i<newIds.size(); ++i) { + _optionMap[newIds[i]] = opt; + opt->_hiddenNames.push_back(newIds[i]); + } +} + +void +ProgramOptions::setArgumentTypeName(const std::string& name, uint32_t index) +{ + if (_options.size() == 0) { + throw InvalidCommandLineArgumentsException( + "Cannot add hidden identifier to last " + "option as no option has been added yet.", VESPA_STRLOC); + } + OptionParser::SP opt = _options.back(); + if (opt->isHeader()) { + throw InvalidCommandLineArgumentsException( + "Cannot add option arguments to option header.", VESPA_STRLOC); + } + opt->_argTypes[index] = name; +} + +void +ProgramOptions::addOptionHeader(const std::string& description) +{ + _options.push_back(OptionParser::SP(new OptionHeader(description))); +} + +namespace { + bool isNumber(const std::string& arg) { + if (arg.size() > 1 && arg[0] == '-' + && arg[1] >= '0' && arg[1] <= '9') + { + return true; + } + return false; + } +} + +void +ProgramOptions::parse() +{ + try{ + std::ostringstream ost; + ost << "Parsing options:\n"; + for (int i=0; i<_argc; ++i) { + ost << " " << i << ": '" << _argv[i] << "'\n"; + } + LOG(debug, "%s", ost.str().c_str()); + uint32_t argPos = 0; + uint32_t optPos = 1; + for (; optPos < static_cast<uint32_t>(_argc); ++optPos) { + std::string s(_argv[optPos]); + // Skip arguments + if (s.size() < 2 || s[0] != '-' || s == "--" || isNumber(s)) { + if (argPos <= optPos) { // No more options to parse + break; + } else { // This has already been consumed as argument + continue; + } + } + if (argPos <= optPos) { argPos = optPos + 1; } + if (s.substr(0, 2) == "--") { + std::string id(s.substr(2)); + LOG(debug, "Parsing long option %s at pos %u, arg pos is now " + "%u", id.c_str(), optPos, argPos); + std::map<std::string, OptionParser::SP>::const_iterator it( + _optionMap.find(id)); + if (it == _optionMap.end()) { + throw InvalidCommandLineArgumentsException( + "Invalid option '" + id + "'.", VESPA_STRLOC); + } + parseOption(id, *it->second, argPos); + _setOptions.insert(it->second); + } else { + LOG(debug, "Parsing short options %s at pos %u, arg pos is now " + "%u.", s.c_str() + 1, optPos, argPos); + for (uint32_t shortPos = 1; shortPos < s.size(); ++shortPos) { + std::string id(s.substr(shortPos, 1)); + LOG(debug, "Parsing short option %s, arg pos is %u.", + id.c_str(), argPos); + std::map<std::string, OptionParser::SP>::const_iterator it( + _optionMap.find(id)); + if (it == _optionMap.end()) { + throw InvalidCommandLineArgumentsException( + "Invalid option '" + id + "'.", VESPA_STRLOC); + } + parseOption(id, *it->second, argPos); + _setOptions.insert(it->second); + } + } + } + if (!_defaultsSet) setDefaults(true); + for (uint32_t i=0; i<_arguments.size(); ++i) { + OptionParser& opt(*_arguments[i]); + if (opt._argCount == 0) { + LOG(debug, "Parsing list argument %s. Pos is %u.", + opt.getArgName().c_str(), optPos); + std::vector<std::string> arguments; + for (uint32_t j=optPos; j<static_cast<uint32_t>(_argc); ++j) { + arguments.push_back(_argv[j]); + } + opt.set(arguments); + optPos = _argc; + LOG(debug, "Done. Pos is now %u.", optPos); + } else if (optPos + opt._argCount > static_cast<uint32_t>(_argc)) { + if (!opt.isRequired()) { + LOG(debug, "Setting default for argument %u.", i); + opt.setDefault(); + } else { + throw InvalidCommandLineArgumentsException( + "Insufficient data is given to set required argument '" + + opt.getArgName() + "'.", VESPA_STRLOC); + } + } else { + parseArgument(opt, optPos); + } + } + } catch (const vespalib::Exception& e) { + throw; + } + for (uint32_t i=0; i<_configurables.size(); ++i) { + _configurables[i]->finalizeOptions(); + } +} + +void +ProgramOptions::setDefaults(bool failUnsetRequired) +{ + for (uint32_t i=0; i<_options.size(); ++i) { + OptionParser::SP opt(_options[i]); + if (opt->isHeader()) { + continue; + } + std::set<OptionParser::SP>::const_iterator it(_setOptions.find(opt)); + if (it == _setOptions.end()) { + if (opt->_hasDefault) { + opt->setDefault(); + } else if (failUnsetRequired) { + throw InvalidCommandLineArgumentsException( + "Option '" + opt->_names[0] + + "' has no default and must be set.", VESPA_STRLOC); + } + } + } + _defaultsSet = true; +} + +void +ProgramOptions::parseOption( + const std::string& id, OptionParser& opt, uint32_t& argPos) +{ + LOG(debug, "Parsing option %s. Argpos is %u.", id.c_str(), argPos); + std::vector<std::string> arguments; + for (; arguments.size() != opt._argCount; ++argPos) { + if (argPos >= static_cast<uint32_t>(_argc)) { + throw InvalidCommandLineArgumentsException( + vespalib::make_string( + "Option '%s' needs %u arguments. Only %u available.", + id.c_str(), opt._argCount, (uint32_t) arguments.size()), + VESPA_STRLOC); + } + if (strlen(_argv[argPos]) >= 2 && _argv[argPos][0] == '-' + && !isNumber(_argv[argPos])) + { + continue; + } + arguments.push_back(_argv[argPos]); + } + opt.set(arguments); + LOG(debug, "Done. Argpos is now %u.", argPos); +} + +void +ProgramOptions::parseArgument(OptionParser& opt, uint32_t& pos) +{ + LOG(debug, "Parsing argument %s. Pos is %u.", + opt.getArgName().c_str(), pos); + std::vector<std::string> arguments; + for (; arguments.size() != opt._argCount; ++pos) { + assert(pos < static_cast<uint32_t>(_argc)); + arguments.push_back(_argv[pos]); + } + opt.set(arguments); + LOG(debug, "Done. Pos is now %u.", pos); +} + +namespace { + std::vector<std::string> breakText(const std::vector<std::string>& source, + uint32_t maxLen, + int preserveWordSpaceLimit = -1) + { + if (preserveWordSpaceLimit < 0) { + preserveWordSpaceLimit = maxLen / 5; + } + std::vector<std::string> result; + for (uint32_t i=0; i<source.size(); ++i) { + std::vector<std::string> split(splitString(source[i], '\n')); + for (uint32_t j=0; j<split.size(); ++j) { + // Process each line of input here to possible break + // it. + std::string line = split[j]; + while (true) { + // If the line is already short enough, we're done. + if (line.size() <= maxLen) { + result.push_back(line); + break; + } + // Otherwise, find the last space before max len + std::string::size_type pos( + line.rfind(" ", maxLen)); + if (pos != std::string::npos + && pos > maxLen - preserveWordSpaceLimit) + { + // If the space comes late enough, add the line up + // to that point, and let the remainder go to a new + // line. + result.push_back(line.substr(0, pos)); + line = line.substr(pos+1); + } else { + // If the space is not late enough, force break + // inside a word and add a dash. + result.push_back(line.substr(0, maxLen - 1) + "-"); + line = line.substr(maxLen - 1); + } + } + } + } + return result; + } + + std::vector<std::string> breakText(const std::string& source, + uint32_t maxLen, + int preserveWordSpaceLimit = -1) + { + std::vector<std::string> v(1); + v[0] = source; + return breakText(v, maxLen, preserveWordSpaceLimit); + } +} + +void +ProgramOptions::writeSyntaxPage(std::ostream& out, bool showDefaults) +{ + bool hasOptions = false; + for(uint32_t i=0; i<_options.size(); ++i) { + if (!_options[i]->isHeader() && !_options[i]->hideFromSyntaxPage()) { + hasOptions = true; + } + } + if (!_syntaxMessage.empty()) { + out << "\n"; + std::vector<std::string> text(breakText(_syntaxMessage, 80)); + for (uint32_t i=0; i<text.size(); ++i) { + out << text[i] << "\n"; + } + } + if (_argc > 0) { + std::string progName = _argv[0]; + out << "\nUsage: "; + std::string::size_type pos = progName.rfind("/"); + if (pos != std::string::npos) { + out << progName.substr(pos+1); + } else { + out << progName; + } + if (hasOptions) { + out << " [options]"; + } + for (uint32_t i=0; i<_arguments.size(); ++i) { + OptionParser& opt(*_arguments[i]); + out << (opt.isRequired() ? " <" : " [") + << opt.getArgName(); + if (opt._argCount == 0) out << "..."; + out << (opt.isRequired() ? ">" : "]"); + } + out << "\n"; + } + if (_arguments.size() > 0) { + out << "\nArguments:\n"; + // To reuse option parser objects, argument names get split on + // whitespace. Concatenating to show nice text + std::vector<std::string> argNames(_arguments.size()); + uint32_t argSize = 10; + for(uint32_t i=0; i<_arguments.size(); ++i) { + OptionParser& opt(*_arguments[i]); + argNames[i] = opt.getArgName() + + " (" + opt.getArgType(0) + ")"; + if (argNames[i].size() <= _maxLeftColumnSize) { + argSize = std::max(argSize, + static_cast<uint32_t>(argNames[i].size())); + } + } + // Calculate indent used for extra lines + std::string indent = " "; // 1 space indent + " : " in between. + for (uint32_t i=0; i<argSize; ++i) indent += ' '; + + for(uint32_t i=0; i<_arguments.size(); ++i) { + OptionParser& opt(*_arguments[i]); + out << " " << argNames[i]; + if (argNames[i].size() > _maxLeftColumnSize) { + out << "\n "; + for (uint32_t j=0; j<argSize; ++j) { + out << " "; + } + } else { + for (uint32_t j=argNames[i].size(); j<argSize; ++j) { + out << " "; + } + } + std::vector<std::string> message( + breakText(opt._description, 80 - indent.size())); + if (opt._hasDefault) { + if (message.back().size() + indent.size() + 11 <= 80) { + message.back() += " (optional)"; + } else { + message.push_back("(optional)"); + } + } + for (uint32_t j=0; j<message.size(); ++j) { + out << (j == 0 ? " : " : indent) << message[j] << "\n"; + } + } + } + if (hasOptions) { + if (!_options[0]->isHeader()) { + out << "\nOptions:\n"; + } + uint32_t argSize = 10; + for(uint32_t i=0; i<_options.size(); ++i) { + OptionParser& opt(*_options[i]); + if (opt.isHeader() || opt.hideFromSyntaxPage()) continue; + argSize = std::max(argSize, static_cast<uint32_t>( + opt.getOptSyntaxString().size())); + } + // If too much space is used in first colo, calculate inset again, + // such that we don't need to push to max just because one is + // too long + if (argSize > _maxLeftColumnSize) { + argSize = 10; + for(uint32_t i=0; i<_options.size(); ++i) { + OptionParser& opt(*_options[i]); + if (opt.isHeader() || opt.hideFromSyntaxPage()) continue; + uint32_t size = static_cast<uint32_t>( + opt.getOptSyntaxString().size()); + if (size <= _maxLeftColumnSize) { + argSize = std::max(argSize, size); + } + } + } + std::string indent = " "; + for (uint32_t i=0; i<argSize; ++i) indent += ' '; + for(uint32_t i=0; i<_options.size(); ++i) { + OptionParser& opt(*_options[i]); + if (opt.isHeader()) { + out << "\n" << opt._description << ":\n"; + continue; + } + if (opt.hideFromSyntaxPage()) continue; + std::string optStr = opt.getOptSyntaxString(); + out << optStr; + for (uint32_t j=optStr.size(); j<argSize; ++j) { + out << " "; + } + std::vector<std::string> message( + breakText(opt._description, 80 - indent.size())); + if (showDefaults) { + std::string s; + if (!opt._hasDefault) { + s = "(required)"; + } else if (!opt._invalidDefault + && opt._defaultString != UNSET_TOKEN) + { + s = "(default " + opt._defaultString + ")"; + } + if (s.size() > 0) { + if (message.back().size() + indent.size() + + 1 + s.size() <= 80) + { + message.back() += " " + s; + } else { + message.push_back(s); + } + } + } + for (uint32_t j=0; j<message.size(); ++j) { + out << (j == 0 ? " : " : indent) << message[j] << "\n"; + } + } + } +} + +ProgramOptions::OptionParser& +ProgramOptions::addOption(OptionParser::SP && opt) +{ + for (uint32_t i=0; i<opt->_names.size(); ++i) { + std::map<std::string, OptionParser::SP>::const_iterator it( + _optionMap.find(opt->_names[i])); + if (it != _optionMap.end()) { + throw InvalidCommandLineArgumentsException( + "Option '" + opt->_names[i] + "' is already registered.", + VESPA_STRLOC); + } + } + _options.push_back(opt); + for (uint32_t i=0; i<opt->_names.size(); ++i) { + _optionMap[opt->_names[i]] = opt; + } + return *opt; +} + +ProgramOptions::OptionParser& +ProgramOptions::addArgument(std::shared_ptr<OptionParser> arg) +{ + if (!_arguments.empty() && !_arguments.back()->isRequired()) { + if (arg->isRequired()) { + throw InvalidCommandLineArgumentsException( + "Argument '" + arg->_names[0] + "' is required and cannot " + "follow an optional argument.", VESPA_STRLOC); + } + } + if (!_arguments.empty() && _arguments.back()->_argCount == 0) { + throw InvalidCommandLineArgumentsException( + "Argument '" + arg->_names[0] + "' cannot follow a list " + "argument that will consume all remaining arguments.", + VESPA_STRLOC); + + } + _arguments.push_back(arg); + return *arg; +} + +ProgramOptions::OptionParser& +ProgramOptions::getOptionParser(const std::string& id) +{ + std::map<std::string, OptionParser::SP>::const_iterator it( + _optionMap.find(id)); + if (it == _optionMap.end()) { + throw InvalidCommandLineArgumentsException( + "No option registered with id '" + id + "'.", VESPA_STRLOC); + } + return *it->second; +} + +ProgramOptions::OptionParser& +ProgramOptions::getArgumentParser(uint32_t argIndex) +{ + if (argIndex >= _arguments.size()) { + std::ostringstream ost; + ost << "Only " << _arguments.size() + << " arguments registered. Thus argument " << argIndex + << " does not exist."; + throw InvalidCommandLineArgumentsException(ost.str(), VESPA_STRLOC); + } + return *_arguments[argIndex]; +} + +ProgramOptions::BoolOptionParser::BoolOptionParser( + const std::string& nameList, bool& value, const std::string& desc) + : OptionParser(nameList, 0, UNSET_TOKEN, desc), + _value(value), + _defaultValue(false) +{ +} + +ProgramOptions::FlagOptionParser::FlagOptionParser( + const std::string& nameList, bool& value, const std::string& desc) + : OptionParser(nameList, 0, UNSET_TOKEN, desc), + _value(value), + _unsetValue(false) +{ + _invalidDefault = true; +} + +ProgramOptions::FlagOptionParser::FlagOptionParser( + const std::string& nameList, bool& value, const bool& unsetValue, + const std::string& desc) + : OptionParser(nameList, 0, unsetValue ? "true" : "false", desc), + _value(value), + _unsetValue(unsetValue) +{ + _invalidDefault = true; +} + +ProgramOptions::StringOptionParser::StringOptionParser( + const std::string& nameList, std::string& val, const std::string& desc) + : OptionParser(nameList, 1, desc), + _value(val), + _defaultValue() +{ +} + +ProgramOptions::StringOptionParser::StringOptionParser( + const std::string& nameList, std::string& value, + const std::string& defValue, const std::string& desc) + : OptionParser(nameList, 1, '"' + defValue + '"', desc), + _value(value), + _defaultValue(defValue) +{ +} + +ProgramOptions::MapOptionParser::MapOptionParser( + const std::string& nameList, std::map<std::string, std::string>& value, + const std::string& description) + : OptionParser(nameList, 2, "empty", description), + _value(value) +{ +} + +#define VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDOPTION(type, parsertype) \ +template<> \ +ProgramOptions::OptionParser& \ +ProgramOptions::addOption(const std::string& optionNameList, \ + type& value, const std::string& desc) \ +{ \ + return addOption(OptionParser::SP( \ + new parsertype(optionNameList, value, desc))); \ +} + +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDOPTION(bool, FlagOptionParser); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDOPTION(std::string, StringOptionParser); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDOPTION(int32_t, NumberOptionParser<int32_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDOPTION(uint32_t, NumberOptionParser<uint32_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDOPTION(int64_t, NumberOptionParser<int64_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDOPTION(uint64_t, NumberOptionParser<uint64_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDOPTION(float, NumberOptionParser<float>); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDOPTION(double, NumberOptionParser<double>); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDOPTION(MapOptionParser::MapType, MapOptionParser); + +#define VESPALIB_PROGRAMOPTIONS_IMPL_ADDOPTION(type, parsertype) \ +template<> \ +ProgramOptions::OptionParser& \ +ProgramOptions::addOption(const std::string& optionNameList, \ + type& value, const type& defVal, \ + const std::string& desc) \ +{ \ + return addOption(OptionParser::SP( \ + new parsertype(optionNameList, value, defVal, desc))); \ +} + +VESPALIB_PROGRAMOPTIONS_IMPL_ADDOPTION(bool, FlagOptionParser); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDOPTION(std::string, StringOptionParser); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDOPTION(int32_t, NumberOptionParser<int32_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDOPTION(uint32_t, NumberOptionParser<uint32_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDOPTION(int64_t, NumberOptionParser<int64_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDOPTION(uint64_t, NumberOptionParser<uint64_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDOPTION(float, NumberOptionParser<float>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDOPTION(double, NumberOptionParser<double>); + +#define VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDARGUMENT(type, parsertype) \ +template<> \ +ProgramOptions::OptionParser& \ +ProgramOptions::addArgument(const std::string& name, \ + type& value, \ + const std::string& desc) \ +{ \ + return addArgument(OptionParser::SP( \ + new parsertype(name, value, desc))); \ +} + +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDARGUMENT(bool, BoolOptionParser); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDARGUMENT(std::string, StringOptionParser); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDARGUMENT(int32_t, NumberOptionParser<int32_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDARGUMENT(uint32_t, NumberOptionParser<uint32_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDARGUMENT(int64_t, NumberOptionParser<int64_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDARGUMENT(uint64_t, NumberOptionParser<uint64_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDARGUMENT(float, NumberOptionParser<float>); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDARGUMENT(double, NumberOptionParser<double>); + +#define VESPALIB_PROGRAMOPTIONS_IMPL_ADDARGUMENT(type, parsertype) \ +template<> \ +ProgramOptions::OptionParser& \ +ProgramOptions::addArgument(const std::string& name, \ + type& value, const type& defVal, \ + const std::string& desc) \ +{ \ + return addArgument(OptionParser::SP( \ + new parsertype(name, value, defVal, desc))); \ +} + +VESPALIB_PROGRAMOPTIONS_IMPL_ADDARGUMENT(std::string, StringOptionParser); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDARGUMENT(int32_t, NumberOptionParser<int32_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDARGUMENT(uint32_t, NumberOptionParser<uint32_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDARGUMENT(int64_t, NumberOptionParser<int64_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDARGUMENT(uint64_t, NumberOptionParser<uint64_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDARGUMENT(float, NumberOptionParser<float>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDARGUMENT(double, NumberOptionParser<double>); + +#define VESPALIB_PROGRAMOPTIONS_IMPL_ADDLISTARGUMENT(type, parsertype) \ +template<> \ +ProgramOptions::OptionParser& \ +ProgramOptions::addListArgument(const std::string& name, \ + std::vector<type>& value, \ + const std::string& desc) \ +{ \ + ListOptionParser<type>* listParser( \ + new ListOptionParser<type>(name, value, desc)); \ + OptionParser::UP entryParser( \ + new parsertype(name, listParser->getSingleValue(), desc)); \ + listParser->setEntryParser(std::move(entryParser)); \ + return addArgument(OptionParser::SP(listParser)); \ +} + +VESPALIB_PROGRAMOPTIONS_IMPL_ADDLISTARGUMENT(std::string, StringOptionParser); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDLISTARGUMENT(int32_t, NumberOptionParser<int32_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDLISTARGUMENT(uint32_t, NumberOptionParser<uint32_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDLISTARGUMENT(int64_t, NumberOptionParser<int64_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDLISTARGUMENT(uint64_t, NumberOptionParser<uint64_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDLISTARGUMENT(float, NumberOptionParser<float>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDLISTARGUMENT(double, NumberOptionParser<double>); + +template struct ProgramOptions::NumberOptionParser<int32_t>; +template struct ProgramOptions::NumberOptionParser<uint32_t>; +template struct ProgramOptions::NumberOptionParser<int64_t>; +template struct ProgramOptions::NumberOptionParser<uint64_t>; +template struct ProgramOptions::NumberOptionParser<float>; +template struct ProgramOptions::NumberOptionParser<double>; + +} // vespalib diff --git a/vespalib/src/vespa/vespalib/util/programoptions.h b/vespalib/src/vespa/vespalib/util/programoptions.h new file mode 100644 index 00000000000..0c75f9b2c72 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/programoptions.h @@ -0,0 +1,367 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * \class vespalib::ProgramOptions + * \ingroup util + * + * \brief Utility class for easy parsing of program options. + * + * This class makes it easy to parse program options, and to write a decent + * syntax page. + * + * Just call addOption to register options, and call parseOptions to do the + * parsing. There's also a function for writing the syntax page, such that this + * is automatically updated. + * + * Stuff to come later: + * + * Support for arguments (So you can do stuff like ./myprog file.txt, and not + * only stuff like ./myprog -f file.txt) + * Setting min and max values for numbers. + * Support for multiargument options. + * Automatic man page writing. + */ + +#pragma once + +#include <vespa/vespalib/util/exception.h> +#include <map> +#include <set> +#include <vector> +#include <memory> + +namespace vespalib { + +VESPA_DEFINE_EXCEPTION(InvalidCommandLineArgumentsException, Exception); + +struct ProgramOptions { + /** Utility class used by command line configurable utility. */ + class LifetimeToken { + ProgramOptions& o; + public: + typedef std::unique_ptr<LifetimeToken> UP; + LifetimeToken(ProgramOptions& op) : o(op) {} + ~LifetimeToken() { o.clear(); } + }; + /** + * Utility class used to deletage stuff to be configured into multiple + * units. + */ + struct Configurable { + virtual ~Configurable() {} + /** + * Called on configurables to have it register its options. + * Unit must hang onto lifetimetoken until command line parsing have + * completed. Lifetimetoken should be deleted before stuff registered + * to be configured is deleted. + */ + virtual void registerCommandLineOptions(ProgramOptions&, LifetimeToken::UP) = 0; + + /** + * Called after command line parsing is complete, in order for + * components to ensure validity of options and throw exceptionse on + * failures. + */ + virtual void finalizeOptions() = 0; + }; + + struct OptionParser; + + int _argc; + const char* const* _argv; + std::vector<std::shared_ptr<OptionParser> > _options; + std::map<std::string, std::shared_ptr<OptionParser> > _optionMap; + std::set<std::shared_ptr<OptionParser> > _setOptions; + std::vector<std::shared_ptr<OptionParser> > _arguments; + std::string _syntaxMessage; + uint32_t _maxLeftColumnSize; + bool _defaultsSet; + std::vector<Configurable*> _configurables; + + ProgramOptions(const ProgramOptions&); + ProgramOptions& operator=(const ProgramOptions&); + +public: + /** + * If using empty constructor, setCommandLineArguments() must be called + * before parse() and writeSyntaxPage(). + */ + ProgramOptions(); + ProgramOptions(int argc, const char* const* argv); + virtual ~ProgramOptions(); + + void addConfigurable(Configurable& c) { + _configurables.push_back(&c); + c.registerCommandLineOptions( + *this, LifetimeToken::UP(new LifetimeToken(*this))); + } + + void setCommandLineArguments(int argc, const char* const* argv); + + /** + * In bool case, add an optional option that will be true if used. + * In all other cases, adds a required option as there are no default. + * Parsing will fail if required option is not set. + */ + template<typename Type> + OptionParser& addOption(const std::string& optionNameList, + Type& value, + const std::string& description); + + /** Add an optional option. Default value will be used if not set. */ + template<typename Type> + OptionParser& addOption(const std::string& optionNameList, + Type& value, + const Type& defaultValue, + const std::string& description); + + template<typename Type> + OptionParser& addArgument(const std::string& optionNameList, + Type& value, + const std::string& description); + + template<typename Type> + OptionParser& addArgument(const std::string& optionNameList, + Type& value, + const Type& defaultValue, + const std::string& description); + + template<typename Type> + OptionParser& addListArgument(const std::string& optionNameList, + std::vector<Type>& value, + const std::string& description); + + OptionParser& getOptionParser(const std::string& id); + OptionParser& getArgumentParser(uint32_t argIndex); + + void addHiddenIdentifiers(const std::string& optionNameList); + void setArgumentTypeName(const std::string& type, uint32_t index = 0); + + void addOptionHeader(const std::string& description); + + void setSyntaxPageMaxLeftColumnSize(uint32_t cols) + { _maxLeftColumnSize = cols; } + + /** + * Parses the command line arguments. Enable vespa debug logging if you want + * to see details. + * + * @throws InvalidCommandLineArgumentsException on any failures. + */ + void parse(); + + /** Writes a syntax page to fit an 80 column screen. */ + void writeSyntaxPage(std::ostream& out, bool showDefaults = true); + + /** Sets some textual description added to syntax page. */ + void setSyntaxMessage(const std::string& msg); + + /** + * Can be used after having added all the options to initialize all the + * parameters to default values. Useful if you want to set defaults first, + * override defaults with config of some kind, and then parse, such that + * command line parameters override config. + */ + void setDefaults() { setDefaults(false); } + + /** + * Useful to clear out all options before shutdown if this class outlives + * a class defining options. + */ + void clear(); + +private: + void parseOption(const std::string& id, OptionParser&, uint32_t& argPos); + void parseArgument(OptionParser& opt, uint32_t& pos); + OptionParser& addOption(std::shared_ptr<OptionParser> && opt); + OptionParser& addArgument(std::shared_ptr<OptionParser> arg); + void setDefaults(bool failUnsetRequired); + + struct OptionHeader; + template<typename Number> struct NumberOptionParser; + struct BoolOptionParser; + struct FlagOptionParser; + struct StringOptionParser; + struct MapOptionParser; + template<typename T> struct ListOptionParser; + +}; + +// ---------------------------------------------------------------------------- + +// Implementation of templates and inner classes +// Not a part of the public interface + +template<typename T> const char* getTypeName(); +template<> inline const char* getTypeName<int32_t>() { return "int"; } +template<> inline const char* getTypeName<uint32_t>() { return "uint"; } +template<> inline const char* getTypeName<int64_t>() { return "long"; } +template<> inline const char* getTypeName<uint64_t>() { return "ulong"; } +template<> inline const char* getTypeName<float>() { return "float"; } +template<> inline const char* getTypeName<double>() { return "double"; } + +struct ProgramOptions::OptionParser { + typedef std::unique_ptr<OptionParser> UP; + typedef std::shared_ptr<OptionParser> SP; + + std::vector<std::string> _names; + std::vector<std::string> _hiddenNames; + uint32_t _argCount; + std::vector<std::string> _argTypes; + bool _hasDefault; + bool _invalidDefault; + std::string _defaultString; + std::string _description; + + OptionParser(const std::string& nameList, uint32_t argCount, + const std::string& desc); + OptionParser(const std::string& nameList, uint32_t argCount, + const std::string& defString, const std::string& desc); + virtual ~OptionParser(); + + virtual bool isRequired() const { return !_hasDefault; } + virtual void set(const std::vector<std::string>& arguments) = 0; + virtual void setDefault() = 0; + virtual void setInvalidDefault(); + virtual std::string getArgType(uint32_t /* index */) const { return "val"; } + std::string getOptSyntaxString() const; + std::string getArgName() const { + std::string name = _names[0]; + for (uint32_t i=1; i<_names.size(); ++i) name += " " + _names[i]; + return name; + } + virtual bool isHeader() const { return false; } + virtual bool hideFromSyntaxPage() const + { return !isHeader() && _names.empty(); } +}; + +struct ProgramOptions::OptionHeader : public OptionParser { + OptionHeader(const std::string& desc) : OptionParser("", 0, desc) {} + void set(const std::vector<std::string>&) override {} + void setDefault() override {} + bool isHeader() const override { return true; } +}; + +template<typename Number> +struct ProgramOptions::NumberOptionParser : public OptionParser { + Number& _number; + Number _defaultValue; + + std::string getStringValue(Number n); + + NumberOptionParser(const std::string& nameList, Number& number, + const std::string& description) + : OptionParser(nameList, 1, description), + _number(number), + _defaultValue(number) + { + } + + NumberOptionParser(const std::string& nameList, Number& number, + const Number& defValue, const std::string& desc) + : OptionParser(nameList, 1, getStringValue(defValue), desc), + _number(number), + _defaultValue(defValue) + {} + ~NumberOptionParser() override; + + void set(const std::vector<std::string>& arguments) override; + void setDefault() override { _number = _defaultValue; } + std::string getArgType(uint32_t /* index */) const override { + return getTypeName<Number>(); + } +}; + +template<typename Number> +ProgramOptions::NumberOptionParser<Number>::~NumberOptionParser() = default; + +struct ProgramOptions::BoolOptionParser : public OptionParser { + bool& _value; + bool _defaultValue; + + BoolOptionParser(const std::string& nameList, bool& value, const std::string& description); + void set(const std::vector<std::string>&) override { _value = true; } + void setDefault() override { _value = false; } +}; + +struct ProgramOptions::FlagOptionParser : public OptionParser { + bool& _value; + bool _unsetValue; + + FlagOptionParser(const std::string& nameList, bool& value, const std::string& description); + FlagOptionParser(const std::string& nameList, bool& value, const bool& unsetValue, const std::string& description); + ~FlagOptionParser() override; + void set(const std::vector<std::string>&) override { _value = !_unsetValue; } + void setDefault() override { _value = _unsetValue; } +}; + + +struct ProgramOptions::StringOptionParser : public OptionParser { + std::string& _value; + std::string _defaultValue; + + StringOptionParser(const std::string& nameList, std::string& value, const std::string& description); + StringOptionParser(const std::string& nameList, std::string& value, + const std::string& defVal, const std::string& desc); + ~StringOptionParser() override; + + void set(const std::vector<std::string>& arguments) override { _value = arguments[0]; } + void setDefault() override { _value = _defaultValue; } + std::string getArgType(uint32_t /* index */) const override { return "string"; } +}; + +struct ProgramOptions::MapOptionParser : public OptionParser { + typedef std::map<std::string, std::string> MapType; + std::map<std::string, std::string>& _value; + + MapOptionParser(const std::string& nameList, + std::map<std::string, std::string>& value, + const std::string& description); + + void set(const std::vector<std::string>& arguments) override { + _value[arguments[0]] = arguments[1]; + } + + std::string getArgType(uint32_t /* index */) const override { return "string"; } + + // Default of map is just an empty map. + void setDefault() override { _value.clear(); } +}; + +template<typename T> +struct ProgramOptions::ListOptionParser : public OptionParser { + std::vector<T>& _value; + T _singleValue; + OptionParser::UP _entryParser; + + ListOptionParser(const std::string& nameList, + std::vector<T>& value, + const std::string& description) + : OptionParser(nameList, 0, description), + _value(value) + { + } + + T& getSingleValue() { return _singleValue; } + + void setEntryParser(OptionParser::UP entryParser) { + _entryParser = std::move(entryParser); + } + bool isRequired() const override { return false; } + void set(const std::vector<std::string>& arguments) override { + for (uint32_t i=0; i<arguments.size(); ++i) { + std::vector<std::string> v; + v.push_back(arguments[i]); + _entryParser->set(v); + _value.push_back(_singleValue); + } + } + void setDefault() override { + _value.clear(); + } + std::string getArgType(uint32_t index) const override { + return _entryParser->getArgType(index) + "[]"; + } +}; + +} // vespalib + diff --git a/vespalib/src/vespa/vespalib/util/rusage.cpp b/vespalib/src/vespa/vespalib/util/rusage.cpp new file mode 100644 index 00000000000..ed0ec125dcf --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/rusage.cpp @@ -0,0 +1,135 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "rusage.h" +#include <stdexcept> +#include <cerrno> +#include <vespa/vespalib/util/stringfmt.h> + +namespace vespalib { + +RUsage::RUsage() : + rusage(), + _time(0) +{ + ru_utime.tv_sec = 0; + ru_utime.tv_usec = 0; + ru_stime.tv_sec = 0; + ru_stime.tv_usec = 0; + ru_maxrss = 0; + ru_ixrss = 0; + ru_idrss = 0; + ru_isrss = 0; + ru_minflt = 0; + ru_majflt = 0; + ru_nswap = 0; + ru_inblock = 0; + ru_oublock = 0; + ru_msgsnd = 0; + ru_msgrcv = 0; + ru_nsignals = 0; + ru_nvcsw = 0; + ru_nivcsw = 0; +} + +RUsage +RUsage::createSelf() +{ + return createSelf(vespalib::steady_time()); +} + +RUsage +RUsage::createChildren() +{ + return createChildren(vespalib::steady_time()); +} + +RUsage +RUsage::createSelf(vespalib::steady_time since) +{ + RUsage r; + r._time = vespalib::steady_clock::now() - since; + if (getrusage(RUSAGE_SELF, &r) != 0) { + throw std::runtime_error(vespalib::make_string("getrusage failed with errno = %d", errno).c_str()); + } + return r; +} + +RUsage +RUsage::createChildren(vespalib::steady_time since) +{ + RUsage r; + r._time = vespalib::steady_clock::now() - since; + if (getrusage(RUSAGE_CHILDREN, &r) != 0) { + throw std::runtime_error(vespalib::make_string("getrusage failed with errno = %d", errno).c_str()); + } + return r; +} + +vespalib::string +RUsage::toString() +{ + vespalib::string s; + if (_time != duration::zero()) s += make_string("duration = %1.6f\n", vespalib::to_s(_time)); + if (from_timeval(ru_utime) > duration::zero()) s += make_string("user time = %1.6f\n", to_s(from_timeval(ru_utime))); + if (from_timeval(ru_stime) > duration::zero()) s += make_string("system time = %1.6f\n", to_s(from_timeval(ru_stime))); + if (ru_maxrss != 0) s += make_string("ru_maxrss = %ld\n", ru_maxrss); + if (ru_ixrss != 0) s += make_string("ru_ixrss = %ld\n", ru_ixrss); + if (ru_idrss != 0) s += make_string("ru_idrss = %ld\n", ru_idrss); + if (ru_isrss != 0) s += make_string("ru_isrss = %ld\n", ru_isrss); + if (ru_minflt != 0) s += make_string("ru_minflt = %ld\n", ru_minflt); + if (ru_majflt != 0) s += make_string("ru_majflt = %ld\n", ru_majflt); + if (ru_nswap != 0) s += make_string("ru_nswap = %ld\n", ru_nswap); + if (ru_inblock != 0) s += make_string("ru_inblock = %ld\n", ru_inblock); + if (ru_oublock != 0) s += make_string("ru_oublock = %ld\n", ru_oublock); + if (ru_msgsnd != 0) s += make_string("ru_msgsnd = %ld\n", ru_msgsnd); + if (ru_msgrcv != 0) s += make_string("ru_msgrcv = %ld\n", ru_msgrcv); + if (ru_nsignals != 0) s += make_string("ru_nsignals = %ld\n", ru_nsignals); + if (ru_nvcsw != 0) s += make_string("ru_nvcsw = %ld\n", ru_nvcsw); + if (ru_nivcsw != 0) s += make_string("ru_nivcsw = %ld", ru_nivcsw); + return s; +} + +RUsage & +RUsage::operator -= (const RUsage & b) +{ + _time -= b._time; + ru_utime = ru_utime - b.ru_utime; + ru_stime = ru_stime - b.ru_stime; + ru_maxrss -= b.ru_maxrss; + ru_ixrss -= b.ru_ixrss; + ru_idrss -= b.ru_idrss; + ru_isrss -= b.ru_isrss; + ru_minflt -= b.ru_minflt; + ru_majflt -= b.ru_majflt; + ru_nswap -= b.ru_nswap; + ru_inblock -= b.ru_inblock; + ru_oublock -= b.ru_oublock; + ru_msgsnd -= b.ru_msgsnd; + ru_msgrcv -= b.ru_msgrcv; + ru_nsignals -= b.ru_nsignals; + ru_nvcsw -= b.ru_nvcsw; + ru_nivcsw -= b.ru_nivcsw; + return *this; +} + +timeval +operator - (const timeval & a, const timeval & b) +{ + timeval tmp; + if (a.tv_usec >= b.tv_usec) { + tmp.tv_usec = a.tv_usec - b.tv_usec; + tmp.tv_sec = a.tv_sec - b.tv_sec; + } else { + tmp.tv_usec = (a.tv_usec + 1000000) - b.tv_usec; + tmp.tv_sec = a.tv_sec - 1 - b.tv_sec; + } + return tmp; +} + +RUsage operator -(const RUsage & a, const RUsage & b) +{ + RUsage tmp(a); + tmp -= b; + return tmp; +} + +} diff --git a/vespalib/src/vespa/vespalib/util/rusage.h b/vespalib/src/vespa/vespalib/util/rusage.h new file mode 100644 index 00000000000..4c741f7699d --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/rusage.h @@ -0,0 +1,39 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/time.h> +#include <sys/resource.h> + +namespace vespalib { + +class RUsage : private rusage { +public: + /** + * Create an rusage with all member set to zero. + **/ + RUsage(); + /** + * Will create an RUsage and initialize member with RUSAGE_SELF + **/ + static RUsage createSelf(); + static RUsage createSelf(vespalib::steady_time since); + /** + * Will create an RUsage and initialize member with RUSAGE_CHILDREN + **/ + static RUsage createChildren(); + static RUsage createChildren(vespalib::steady_time since); + /** + * Will create an RUsage and initialize member with RUSAGE_CHILDREN + **/ + vespalib::string toString(); + RUsage & operator -= (const RUsage & rhs); +private: + vespalib::duration _time; +}; + +RUsage operator -(const RUsage & a, const RUsage & b); +timeval operator -(const timeval & a, const timeval & b); + +} + diff --git a/vespalib/src/vespa/vespalib/util/shutdownguard.cpp b/vespalib/src/vespa/vespalib/util/shutdownguard.cpp new file mode 100644 index 00000000000..12e58898c06 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/shutdownguard.cpp @@ -0,0 +1,40 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "shutdownguard.h" +#include <unistd.h> +#include <thread> + +#include <vespa/log/log.h> +LOG_SETUP(".vespalib.shutdownguard"); + +namespace vespalib { + +namespace { +enum { STACK_SIZE = (1u << 16) }; +} +void ShutdownGuard::Run(FastOS_ThreadInterface *, void *) +{ + while (_dieAtTime > steady_clock::now() && ! GetThread()->GetBreakFlag()) { + std::this_thread::sleep_for(5ms); + } + if (_dieAtTime <= steady_clock::now()) { + LOG(warning, "ShutdownGuard is now forcing an exit of the process."); + _exit(EXIT_FAILURE); + } +} + +ShutdownGuard::ShutdownGuard(duration millis) : + FastOS_Runnable(), + _pool(STACK_SIZE, 1), + _dieAtTime(steady_clock::now() + millis) +{ + _pool.NewThread(this); +} + +ShutdownGuard::~ShutdownGuard() +{ + GetThread()->SetBreakFlag(); + GetThread()->Join(); + _pool.Close(); +} + +} diff --git a/vespalib/src/vespa/vespalib/util/shutdownguard.h b/vespalib/src/vespa/vespalib/util/shutdownguard.h new file mode 100644 index 00000000000..d76d4deb5d2 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/shutdownguard.h @@ -0,0 +1,38 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/util/time.h> +#include <vespa/fastos/thread.h> + +namespace vespalib { + +/** + * Class to ensure that the current process finishes within a given time. + * Construct with the number of milliseconds before triggering _exit(); + * destruct the ShutdownGuard object to dismiss the automatic process + * termination. + * A separate shutdown thread will perform the actual _exit() call. + **/ +class ShutdownGuard : public FastOS_Runnable +{ + FastOS_ThreadPool _pool; + steady_time _dieAtTime; + + void Run(FastOS_ThreadInterface *, void *) override; + +public: + /** + * Construct a shutdown guard with a given lifetime. + * @arg millis the number of milliseconds before process automatically exits + **/ + ShutdownGuard(duration millis); + + /** + * Destructor that dismisses the guard and collects the shutdown thread. + **/ + ~ShutdownGuard(); + +}; + +} // namespace vespalib + diff --git a/vespalib/src/vespa/vespalib/util/sort.h b/vespalib/src/vespa/vespalib/util/sort.h new file mode 100644 index 00000000000..ce3f6772ef1 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/sort.h @@ -0,0 +1,281 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/objects/nbo.h> +#include <functional> +#include <limits> +#include <algorithm> + +namespace vespalib { + +template<typename T, bool asc=true> +class convertForSort +{ +}; + +template<> +class convertForSort<float, true> +{ +public: + typedef float InputType; + typedef int32_t IntType; + typedef uint32_t UIntType; + typedef std::less<InputType> Compare; + static inline UIntType convert(float value) + { + union { float f; UIntType u; } val; + val.f=value; + return (static_cast<IntType>(val.u) >= 0) + ? (val.u ^ (UIntType(std::numeric_limits<IntType>::max()) + 1)) + : (val.u ^ std::numeric_limits<UIntType>::max()); + } +}; + +template<> +class convertForSort<float, false> +{ +public: + typedef float InputType; + typedef int32_t IntType; + typedef uint32_t UIntType; + typedef std::greater<InputType> Compare; + static inline UIntType convert(float value) + { + union { float f; UIntType u; } val; + val.f=value; + return (static_cast<IntType>(val.u) >= 0) + ? (val.u ^ std::numeric_limits<IntType>::max()) + : val.u; + } +}; + + +template<> +class convertForSort<double, true> +{ +public: + typedef double InputType; + typedef int64_t IntType; + typedef uint64_t UIntType; + typedef std::less<InputType> Compare; + static inline UIntType convert(double value) + { + union { double f; UIntType u; } val; + val.f=value; + return (static_cast<IntType>(val.u) >= 0) + ? (val.u ^ (UIntType(std::numeric_limits<IntType>::max()) + 1)) + : (val.u ^ std::numeric_limits<UIntType>::max()); + } +}; + +template<> +class convertForSort<double, false> +{ +public: + typedef double InputType; + typedef int64_t IntType; + typedef uint64_t UIntType; + typedef std::greater<InputType> Compare; + static inline UIntType convert(double value) + { + union { double f; UIntType u; } val; + val.f=value; + return (static_cast<IntType>(val.u) >= 0) + ? (val.u ^ std::numeric_limits<IntType>::max()) + : val.u; + } +}; + +template<> +class convertForSort<uint8_t, true> +{ +public: + typedef uint8_t InputType; + typedef int8_t IntType; + typedef uint8_t UIntType; + typedef std::less<InputType> Compare; + static inline UIntType convert(UIntType value) { return value; } +}; + +template<> +class convertForSort<uint8_t, false> +{ +public: + typedef uint8_t InputType; + typedef int8_t IntType; + typedef uint8_t UIntType; + typedef std::greater<InputType> Compare; + static inline UIntType convert(UIntType value) { return ~value; } +}; +template<> +class convertForSort<uint16_t, true> +{ +public: + typedef uint16_t InputType; + typedef int16_t IntType; + typedef uint16_t UIntType; + typedef std::less<InputType> Compare; + static inline UIntType convert(UIntType value) { return value; } +}; +template<> +class convertForSort<uint16_t, false> +{ +public: + typedef uint16_t InputType; + typedef int16_t IntType; + typedef uint16_t UIntType; + typedef std::greater<InputType> Compare; + static inline UIntType convert(UIntType value) { return ~value; } +}; +template<> +class convertForSort<uint32_t, true> +{ +public: + typedef uint32_t InputType; + typedef int32_t IntType; + typedef uint32_t UIntType; + typedef std::less<InputType> Compare; + static inline UIntType convert(UIntType value) { return value; } +}; +template<> +class convertForSort<uint32_t, false> +{ +public: + typedef uint32_t InputType; + typedef int32_t IntType; + typedef uint32_t UIntType; + typedef std::greater<InputType> Compare; + static inline UIntType convert(UIntType value) { return ~value; } +}; +template<> +class convertForSort<uint64_t, true> +{ +public: + typedef uint64_t InputType; + typedef int64_t IntType; + typedef uint64_t UIntType; + typedef std::less<InputType> Compare; + static inline UIntType convert(UIntType value) { return value; } +}; +template<> +class convertForSort<uint64_t, false> +{ +public: + typedef uint64_t InputType; + typedef int64_t IntType; + typedef uint64_t UIntType; + typedef std::greater<InputType> Compare; + static inline UIntType convert(UIntType value) { return ~value; } +}; + +template<> +class convertForSort<bool, true> +{ +public: + typedef bool InputType; + typedef bool IntType; + typedef bool UIntType; + typedef std::less<InputType> Compare; + static inline UIntType convert(IntType value) { return value; } +}; +template<> +class convertForSort<bool, false> +{ +public: + typedef bool InputType; + typedef bool IntType; + typedef bool UIntType; + typedef std::less<InputType> Compare; + static inline UIntType convert(IntType value) { return !value; } +}; + +template<> +class convertForSort<int8_t, true> +{ +public: + typedef int8_t InputType; + typedef int8_t IntType; + typedef uint8_t UIntType; + typedef std::less<InputType> Compare; + static inline UIntType convert(IntType value) { return value ^ (std::numeric_limits<IntType>::max() + 1); } +}; +template<> +class convertForSort<int8_t, false> +{ +public: + typedef int8_t InputType; + typedef int8_t IntType; + typedef uint8_t UIntType; + typedef std::greater<InputType> Compare; + static inline UIntType convert(IntType value) { return value ^ std::numeric_limits<IntType>::max(); } +}; +template<> +class convertForSort<int16_t, true> +{ +public: + typedef int16_t InputType; + typedef int16_t IntType; + typedef uint16_t UIntType; + typedef std::less<InputType> Compare; + static inline UIntType convert(IntType value) { return value ^ (std::numeric_limits<IntType>::max() + 1); } +}; +template<> +class convertForSort<int16_t, false> +{ +public: + typedef int16_t InputType; + typedef int16_t IntType; + typedef uint16_t UIntType; + typedef std::greater<InputType> Compare; + static inline UIntType convert(IntType value) { return value ^ std::numeric_limits<IntType>::max(); } +}; +template<> +class convertForSort<int32_t, true> +{ +public: + typedef int32_t InputType; + typedef int32_t IntType; + typedef uint32_t UIntType; + typedef std::less<InputType> Compare; + static inline UIntType convert(IntType value) { return value ^ (UIntType(std::numeric_limits<IntType>::max()) + 1); } +}; +template<> +class convertForSort<int32_t, false> +{ +public: + typedef int32_t InputType; + typedef int32_t IntType; + typedef uint32_t UIntType; + typedef std::greater<InputType> Compare; + static inline UIntType convert(IntType value) { return value ^ std::numeric_limits<IntType>::max(); } +}; +template<> +class convertForSort<int64_t, true> +{ +public: + typedef int64_t InputType; + typedef int64_t IntType; + typedef uint64_t UIntType; + typedef std::less<InputType> Compare; + static inline UIntType convert(IntType value) { return value ^ (UIntType(std::numeric_limits<IntType>::max()) + 1); } +}; +template<> +class convertForSort<int64_t, false> +{ +public: + typedef int64_t InputType; + typedef int64_t IntType; + typedef uint64_t UIntType; + typedef std::greater<InputType> Compare; + static inline UIntType convert(IntType value) { return value ^ std::numeric_limits<IntType>::max(); } +}; + +template<typename C> +uint32_t serializeForSort(typename C::InputType v, void * dst) { + typename C::UIntType nbo(vespalib::nbo::n2h(C::convert(v))); + memcpy(dst, &nbo, sizeof(nbo)); + return sizeof(nbo); +} + +} // namespace vespalib + diff --git a/vespalib/src/vespa/vespalib/util/testclock.cpp b/vespalib/src/vespa/vespalib/util/testclock.cpp new file mode 100644 index 00000000000..bc5d37ca437 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/testclock.cpp @@ -0,0 +1,16 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "testclock.h" +#include <vespa/vespalib/util/invokeserviceimpl.h> + +namespace vespalib { + +TestClock::TestClock() + : _ticker(std::make_unique<InvokeServiceImpl>(10ms)), + _clock(_ticker->nowRef()) +{ +} + +TestClock::~TestClock() = default; + +} diff --git a/vespalib/src/vespa/vespalib/util/testclock.h b/vespalib/src/vespa/vespalib/util/testclock.h new file mode 100644 index 00000000000..9446ff32cb2 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/testclock.h @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "clock.h" + +namespace vespalib { + +class InvokeServiceImpl; + +/** + * Self contained clock useable for testing that provides a backing for the vespalib::Clock interface. + */ + +class TestClock +{ +private: + std::unique_ptr<InvokeServiceImpl> _ticker; + Clock _clock; +public: + TestClock(); + TestClock(const TestClock &) = delete; + TestClock & operator =(const TestClock &) = delete; + TestClock(TestClock &&) = delete; + TestClock & operator =(TestClock &&) = delete; + ~TestClock(); + const Clock & clock() { return _clock; } +}; + +} + diff --git a/vespalib/src/vespa/vespalib/util/varholder.h b/vespalib/src/vespa/vespalib/util/varholder.h new file mode 100644 index 00000000000..d92c00e0081 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/varholder.h @@ -0,0 +1,38 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <mutex> + +namespace vespalib { + +template <typename T> +class VarHolder +{ + T _v; + mutable std::mutex _lock; +public: + VarHolder() : _v(), _lock() {} + explicit VarHolder(const T &v) : _v(v), _lock() {} + VarHolder(const VarHolder &) = delete; + VarHolder & operator = (const VarHolder &) = delete; + ~VarHolder() {} + + void set(const T &v) { + T old; + { + std::lock_guard guard(_lock); + old = _v; + _v = v; + } + } + + void clear() { set(T()); } + + T get() const { + std::lock_guard guard(_lock); + return _v; + } +}; + +} diff --git a/vespalib/src/vespa/vespalib/util/xmlserializable.cpp b/vespalib/src/vespa/vespalib/util/xmlserializable.cpp new file mode 100644 index 00000000000..c687609df74 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/xmlserializable.cpp @@ -0,0 +1,18 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "xmlserializable.h" +#include "xmlstream.h" +#include <sstream> + +namespace vespalib::xml { + +std::string +XmlSerializable::toXml(const std::string& indent) const +{ + std::ostringstream ost; + XmlOutputStream xos(ost, indent); + printXml(xos); + return ost.str(); +} + +} diff --git a/vespalib/src/vespa/vespalib/util/xmlserializable.h b/vespalib/src/vespa/vespalib/util/xmlserializable.h new file mode 100644 index 00000000000..76f0ad1fa3a --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/xmlserializable.h @@ -0,0 +1,40 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <string> + +namespace vespalib::xml { + +class XmlOutputStream; + +/** + * @class document::XmlSerializable + * + * Base class for classes that can be converted into XML. + */ +class XmlSerializable +{ +public: + XmlSerializable() {} + virtual ~XmlSerializable() = default; + + virtual void printXml(XmlOutputStream& out) const = 0; + + /** Utility function, using printXml() to create a string. */ + virtual std::string toXml(const std::string& indent = "") const; +}; + +} + +namespace vespalib { +// The XmlSerializable and XmlOutputStream is often used in header files +// and is thus available in the vespalib namespace. To not pollute the +// vespalib namespace with all the other classes, use +// "using namespace vespalib::xml" within your printXml functions + +using XmlSerializable = vespalib::xml::XmlSerializable; +using XmlOutputStream = vespalib::xml::XmlOutputStream; + +} // vespalib + diff --git a/vespalib/src/vespa/vespalib/util/xmlstream.cpp b/vespalib/src/vespa/vespalib/util/xmlstream.cpp new file mode 100644 index 00000000000..bdc09da127b --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/xmlstream.cpp @@ -0,0 +1,465 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "xmlstream.hpp" +#include <vespa/vespalib/encoding/base64.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <cassert> +#include <vector> + +namespace vespalib::xml { + +namespace { + + std::vector<bool> getLegalIdentifierFirstCharacters() { + std::vector<bool> vec(256, false); + for (uint32_t i='a'; i<='z'; ++i) vec[i] = true; + for (uint32_t i='A'; i<='Z'; ++i) vec[i] = true; + vec[':'] = true; + vec['_'] = true; + return vec; + } + + std::vector<bool> getLegalIdentifierCharacters() { + std::vector<bool> vec(getLegalIdentifierFirstCharacters()); + vec['-'] = true; + vec['.'] = true; + for (uint32_t i='0'; i<='9'; ++i) { + vec[i] = true; + } + return vec; + } + + std::vector<bool> getBinaryCharacters() { + std::vector<bool> vec(256, false); + for (uint32_t i=0; i<32; ++i) { + vec[i] = true; + } + vec['\t'] = false; + vec['\n'] = false; + vec['\r'] = false; + vec['\f'] = false; + return vec; + } + + std::vector<bool> getEscapedXmlCharacters() { + std::vector<bool> vec(256, false); + for (uint32_t i=0; i<32; ++i) { + vec[i] = true; + } + vec['\n'] = false; + vec['<'] = true; + vec['>'] = true; + vec['&'] = true; + return vec; + } + + std::vector<bool> legalIdentifierFirstChar( + getLegalIdentifierFirstCharacters()); + std::vector<bool> legalIdentifierChars = getLegalIdentifierCharacters(); + std::vector<bool> binaryChars = getBinaryCharacters(); + std::vector<bool> escapedXmlChars = getEscapedXmlCharacters(); + + bool containsBinaryCharacters(const std::string& s) { + for (int i=0, n=s.size(); i<n; ++i) { + if (binaryChars[static_cast<uint8_t>(s[i])]) return true; + } + return false; + } + + const std::string xmlAttributeEscape(const std::string& s) { + vespalib::asciistream ost; + for (uint32_t i=0, n=s.size(); i<n; ++i) { + if (s[i] == '"' || s[i] == '\n' + || escapedXmlChars[static_cast<uint8_t>(s[i])]) + { + if (s[i] == '<') ost << "<"; + else if (s[i] == '>') ost << ">"; + else if (s[i] == '&') ost << "&"; + else if (s[i] == '"') ost << """; + else { + ost << "&#" << (int) s[i] << ";"; + } + } else { + ost << s[i]; + } + } + return ost.str(); + } + + void writeEscaped(std::ostream& out, const std::string& s) { + for (uint32_t i=0, n=s.size(); i<n; ++i) { + if (escapedXmlChars[static_cast<uint8_t>(s[i])]) { + if (s[i] == '<') out << "<"; + else if (s[i] == '>') out << ">"; + else if (s[i] == '&') out << "&"; + else { + out << "&#" << (int) s[i] << ";"; + } + } else { + out << s[i]; + } + } + } + + void writeBase64Encoded(std::ostream& out, const std::string& s) { + out << vespalib::Base64::encode(&s[0], s.size()); + } +} + +bool isLegalName(const std::string& name) { + if (name.size() == 0) return false; + if (!legalIdentifierFirstChar[static_cast<uint8_t>(name[0])]) return false; + for (int i=1, n=name.size(); i<n; ++i) { + if (!legalIdentifierChars[static_cast<uint8_t>(name[i])]) return false; + } + return true; +} + +void convertToLegalName(std::string& name) { + if (name.size() == 0) { + name = "__no_name__"; + } else { + if (!legalIdentifierFirstChar[static_cast<uint8_t>(name[0])]) { + name[0] = '_'; + } + for (int i=1, n=name.size(); i<n; ++i) { + if (!legalIdentifierChars[static_cast<uint8_t>(name[i])]) { + name[i] = '_'; + } + } + } +} + +XmlOutputStream::XmlOutputStream(std::ostream& ostream, + const std::string& indent) + : _indent(indent), + _wrappedStream(ostream), + _tagStack(), + _cachedTag(), + _cachedAttributes(), + _cachedContent() +{ +} + +XmlAttribute::~XmlAttribute() +{ +} + +XmlContent::~XmlContent() +{ +} + +XmlOutputStream::~XmlOutputStream() +{ +} + +XmlOutputStream& +XmlOutputStream::operator<<(const XmlTag& tag) +{ + //std::cerr << "Trying to add tag " << tag.getName() << ". cached tag is " + // << (void*) _cachedTag.get() << "\n"; + if (_cachedTag.get() != 0) flush(false); + _cachedTag.reset(new XmlTag(tag)); + _cachedContentType = XmlContent::AUTO; + //std::cerr << "Added tag " << _cachedTag->getName() << "\n"; + return *this; +} + +XmlOutputStream& +XmlOutputStream::operator<<(const XmlAttribute& attribute) +{ + //std::cerr << "Adding attribute\n"; + if (_cachedTag.get() == 0) { + throw vespalib::IllegalStateException("Cannot add attribute " + + attribute.getName() + ", as no tag is open"); + } + _cachedAttributes.push_back(attribute); + return *this; +} + +XmlOutputStream& +XmlOutputStream::operator<<(const XmlEndTag&) +{ + //std::cerr << "Adding endtag\n"; + if (_cachedTag.get()) { + flush(true); + _cachedContentType = XmlContent::ESCAPED; + } else if (_tagStack.empty()) { + throw vespalib::IllegalStateException("No open tags left to end"); + } else { + for (uint32_t i=1; i<_tagStack.size(); ++i) { + _wrappedStream << _indent; + } + _wrappedStream << "</" << _tagStack.back() << ">"; + _tagStack.pop_back(); + if (!_tagStack.empty()) _wrappedStream << '\n'; + _cachedContentType = XmlContent::ESCAPED; + } + return *this; +} + +XmlOutputStream& +XmlOutputStream::operator<<(const XmlContent& content) +{ + //std::cerr << "Adding content\n"; + if (_cachedTag.get() == 0 && _tagStack.empty()) { + throw vespalib::IllegalStateException( + "No open tag to write content in"); + } + if (_cachedTag.get() != 0) { + //std::cerr << "Content is '" << content.getContent() << "'\n"; + if (content.getType() == XmlContent::AUTO) { // Do nothing.. Always ok + } else if (_cachedContentType == XmlContent::AUTO) { + _cachedContentType = content.getType(); + } else if (_cachedContentType != content.getType()) { + throw vespalib::IllegalStateException( + "Have already added content of different type"); + } + _cachedContent.push_back(content); + } else { + if (content.getType() == XmlContent::BASE64) { + throw vespalib::IllegalStateException( + "Cannot add Base64 encoded content after tag content"); + } + for (uint32_t i=0; i<_tagStack.size(); ++i) { + _wrappedStream << _indent; + } + _wrappedStream << content.getContent() << '\n'; + } + return *this; +} + +XmlOutputStream& +XmlOutputStream::operator<<(const XmlSerializable& serializable) +{ + //std::cerr << "Adding serializable\n"; + serializable.printXml(*this); + return *this; +} + +XmlOutputStream& +XmlOutputStream::operator<<(const std::string& content) +{ + //std::cerr << "Adding content string\n"; + return *this << XmlContent(content); +} + +XmlOutputStream& +XmlOutputStream::operator<<(char c) +{ + return *this << XmlContent(std::string(&c, 1)); +} + +XmlOutputStream& +XmlOutputStream::operator<<(int32_t i) +{ + return *this << XmlContent(vespalib::make_string("%d", i)); +} + +XmlOutputStream& +XmlOutputStream::operator<<(int64_t i) +{ + return *this << XmlContent(vespalib::make_string("%" PRId64, i)); +} + +XmlOutputStream& +XmlOutputStream::operator<<(float f) +{ + return *this << XmlContent(vespalib::make_string("%g", f)); +} + +XmlOutputStream& +XmlOutputStream::operator<<(double d) +{ + return *this << XmlContent(vespalib::make_string("%g", d)); +} + +void +XmlOutputStream::flush(bool endTag) +{ + //std::cerr << "Flushing\n"; + if (_cachedTag.get() == 0) { + throw vespalib::IllegalStateException("Cannot write non-existing tag"); + } + for (uint32_t i=0; i<_tagStack.size(); ++i) { + _wrappedStream << _indent; + } + _wrappedStream << '<' << _cachedTag->getName(); + for (std::list<XmlAttribute>::const_iterator it = _cachedAttributes.begin(); + it != _cachedAttributes.end(); ++it) + { + _wrappedStream << ' ' << it->getName() << "=\"" + << xmlAttributeEscape(it->getValue()) << '"'; + } + _cachedAttributes.clear(); + if (_cachedContent.empty() && endTag) { + _wrappedStream << "/>\n"; + } else if (_cachedContent.empty()) { + _wrappedStream << ">\n"; + _tagStack.push_back(_cachedTag->getName()); + } else { + if (_cachedContentType == XmlContent::AUTO) { + _cachedContentType = XmlContent::ESCAPED; + for (std::list<XmlContent>::const_iterator it + = _cachedContent.begin(); it != _cachedContent.end(); ++it) + { + if (containsBinaryCharacters(it->getContent())) { + _cachedContentType = XmlContent::BASE64; + break; + } + } + } + if (_cachedContentType == XmlContent::BASE64) { + _wrappedStream << " binaryencoding=\"base64\""; + } + _wrappedStream << '>'; + for (std::list<XmlContent>::const_iterator it = _cachedContent.begin(); + it != _cachedContent.end(); ++it) + { + if (!endTag) { + _wrappedStream << '\n'; + for (uint32_t i=0; i<=_tagStack.size(); ++i) { + _wrappedStream << _indent; + } + } + switch (_cachedContentType) { + case XmlContent::ESCAPED: { + writeEscaped(_wrappedStream, it->getContent()); + break; + } + case XmlContent::BASE64: { + writeBase64Encoded(_wrappedStream, it->getContent()); + break; + } + default: assert(false); + } + } + _cachedContent.clear(); + if (endTag) { + _wrappedStream << "</" << _cachedTag->getName() << ">\n"; + } else { + _wrappedStream << '\n'; + _tagStack.push_back(_cachedTag->getName()); + } + } + _cachedTag.reset(0); +} + +XmlTag::XmlTag(const XmlTag& tag) + : _name(tag._name), + _attributes(), + _content(), + _flags(tag._flags) +{ +} + +XmlTag::~XmlTag() {} + +XmlTag::XmlTag(const std::string& name, XmlTagFlags flags) + : _name(name), + _attributes(), + _content(), + _flags(flags) +{ + if (_flags == XmlTagFlags::CONVERT_ILLEGAL_CHARACTERS) { + convertToLegalName(_name); + } + if (!isLegalName(_name)) { + throw vespalib::IllegalArgumentException("Name '" + _name + "' contains " + "illegal XML characters and cannot be used as tag name"); + } +} + +XmlAttribute::XmlAttribute(const XmlAttribute& attribute) + : _name(attribute._name), + _value(attribute._value), + _next() +{ +} + +XmlAttribute::XmlAttribute(const std::string& name, const char * value, uint32_t flags) + : _name(name), + _value(), + _next() +{ + vespalib::asciistream ost; + if (flags & HEX) ost << vespalib::hex << "0x"; + ost << value; + _value = ost.str(); + if (!isLegalName(name)) { + throw vespalib::IllegalArgumentException("Name '" + name + "' contains " + "illegal XML characters and cannot be used as attribute name"); + } +} + +XmlEndTag::XmlEndTag() +{ +} + +XmlContent::XmlContent(Type type) + : _type(type), + _content(), + _nextContent(), + _nextTag() +{ +} + +XmlContent::XmlContent() + : _type(AUTO), + _content(), + _nextContent(), + _nextTag() +{ +} + +XmlContent::XmlContent(const XmlContent& content) + : _type(content._type), + _content(content._content), + _nextContent(), + _nextTag() +{ +} + +XmlContent::XmlContent(const std::string& value) + : _type(AUTO), + _content(value), + _nextContent(), + _nextTag() +{ +} + +XmlContentWrapper::XmlContentWrapper(const XmlContentWrapper& wrapper) + : XmlContent(wrapper) +{ +} + +XmlContentWrapper::XmlContentWrapper(const char* value) + : XmlContent(std::string(value)) +{ +} + +XmlContentWrapper::XmlContentWrapper(const char* value, uint32_t size) + : XmlContent(std::string(value, size)) +{ +} + +using CharP = char *; +using ConstCharP = const char *; + +template XmlAttribute::XmlAttribute(const std::string &, std::string, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, vespalib::string, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, vespalib::stringref, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, CharP, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, bool, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, short, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, int, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, long, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, long long, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, unsigned short, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, unsigned int, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, unsigned long, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, unsigned long long, unsigned int); +template XmlAttribute::XmlAttribute(const std::string &, double, unsigned int); + +} diff --git a/vespalib/src/vespa/vespalib/util/xmlstream.h b/vespalib/src/vespa/vespalib/util/xmlstream.h new file mode 100644 index 00000000000..b197ff5c7c8 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/xmlstream.h @@ -0,0 +1,210 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @file xmlserializable.h + * @ingroup util + * + * @brief Interfaces to be used for XML serialization. + * + * This file contains XML utility classes, to make XML serialization simple. + * Rather than users writing their own XML, these tools let you define a tree + * structure, and this library builds the XML for you. This ensures that you + * write legal XML and that stuff that needs to be escaped is. + * <p> + * It defines a superclass for XML serializable classes, called XmlSerializable. + * This is what classes that should be XML serializable will inherit. + * <p> + * When implementing the printXml() function in XmlSerializable, one will + * use the various XML helper classes defined here to build a tree structure + * creating the XML. These are: XmlTag, XmlEndTag, XmlAttribute, and XmlContent. + * Some subclasses exist of XmlContent to facilitate various types of content. + * <p> + * The XmlOutputStream wraps a regular std::ostream. You write XML objects to it + * and it is responsible for writing all the XML code. This way, the XML + * serialization is done without interfering with regular output operators. + * <p> + * For example usage, refer to the unit test: + * vespalib/tests/xmlserializable/xmlserializabletest.cpp + * + */ + +#pragma once + +#include "xmlserializable.h" +#include <iosfwd> +#include <list> +#include <memory> + +namespace vespalib::xml { + +class XmlAttribute; +class XmlContent; +class XmlOutputStream; + +bool isLegalName(const std::string& name); + +enum class XmlTagFlags { NONE = 0, CONVERT_ILLEGAL_CHARACTERS = 1 }; + +/** + * @class document::XmlTag + * + * @brief Start a new tag with given name. + */ +class XmlTag { + std::string _name; + std::unique_ptr<XmlAttribute> _attributes; + std::unique_ptr<XmlContent> _content; + XmlTagFlags _flags; +public: + XmlTag(const XmlTag&); + XmlTag(const std::string& name, XmlTagFlags = XmlTagFlags::NONE); + ~XmlTag(); + + const std::string& getName() const { return _name; } +}; + +/** + * @class document::XmlEndTag + * + * @brief Indicates that current tag is closed. + */ +class XmlEndTag { +public: + XmlEndTag(); +}; + +/** + * @class document::XmlAttribute + * + * @brief Defined a single attribute within an XML tag. + * + * When adding an XML to an XML stream, the attribute will be added to the last + * tag added. This can not be called after the last tag opened in the stream is + * closed, so add all attributes before starting to add new XML child tags. + */ +class XmlAttribute { + std::string _name; + std::string _value; + std::unique_ptr<XmlAttribute> _next; +public: + enum Flag { NONE = 0x0, HEX = 0x1 }; + XmlAttribute(const XmlAttribute&); + /** Add any value that can be written to an ostringstream. */ + template<typename T> + XmlAttribute(const std::string& name, T value, uint32_t flags = NONE); + XmlAttribute(const std::string& name, const char * value, uint32_t flags = NONE); + ~XmlAttribute(); + + const std::string& getName() const { return _name; } + const std::string& getValue() const { return _value; } +}; + + +/** + * @class document::XmlContent + * + * XML content to be written to stream. By default it will autodetect whether to + * escape or base64 encode content. XmlOutputStream functions taking primitives + * will generate XmlContent instances. + */ +class XmlContent { +public: + enum Type { AUTO, ESCAPED, BASE64 }; +protected: + XmlContent(Type type); +private: + Type _type; + std::string _content; + std::unique_ptr<XmlContent> _nextContent; + std::unique_ptr<XmlTag> _nextTag; + +public: + XmlContent(); + XmlContent(const XmlContent&); + XmlContent(const std::string& value); + ~XmlContent(); + + Type getType() const { return _type; } + const std::string& getContent() const { return _content; } +}; + +/** + * @class document::XmlEscapedContent + * + * Token used to tell that this content field should only be XML escaped. + */ +class XmlEscapedContent : public XmlContent { +public: + XmlEscapedContent() : XmlContent(ESCAPED) {} +}; + +/** + * @class document::XmlBase64Content + * + * Token used to tell that this content field should always be base64 encoded. + */ +class XmlBase64Content : public XmlContent { +public: + XmlBase64Content() : XmlContent(BASE64) {} +}; + +/** + * @class document::XmlContentWrapper + * + * A wrapper class for content that one doesn't want to copy or release + * ownership of. This wrapper merely takes pointer to data, and assumes it + * will stay alive as long as needed. + */ +class XmlContentWrapper : public XmlContent { +public: + XmlContentWrapper(const XmlContentWrapper&); + XmlContentWrapper(const char* value); + XmlContentWrapper(const char* value, uint32_t size); +}; + +/** + * @class document::XmlOutputStream + * + * @brief std::ostream wrapper, only accepting data that will become XML. + * + * After XmlEndTag() has been sent to the stream, the tag is guarantueed to have + * been written. Call isFinalized() to ensure that you have closed all the tags + * that have been opened. Within a tag, the stream will cache some information, + * as more information might be required before knowing what to print. + */ +class XmlOutputStream { + const std::string _indent; + std::ostream& _wrappedStream; + std::list<std::string> _tagStack; + std::unique_ptr<XmlTag> _cachedTag; + std::list<XmlAttribute> _cachedAttributes; + std::list<XmlContent> _cachedContent; + XmlContent::Type _cachedContentType; + + void flush(bool endTag); + +public: + + XmlOutputStream(std::ostream& ostream, const std::string& indent = ""); + ~XmlOutputStream(); + + bool isFinalized() const + { return (_tagStack.empty() && _cachedTag.get() == 0); } + + std::ostream& getWrappedStream() { return _wrappedStream; } + + XmlOutputStream& operator<<(const XmlTag& tag); + XmlOutputStream& operator<<(const XmlAttribute& attribute); + XmlOutputStream& operator<<(const XmlEndTag& endtag); + XmlOutputStream& operator<<(const XmlContent& content); + XmlOutputStream& operator<<(const XmlSerializable& serializable); + + XmlOutputStream& operator<<(const std::string& content); + XmlOutputStream& operator<<(char c); + XmlOutputStream& operator<<(int32_t i); + XmlOutputStream& operator<<(int64_t i); + XmlOutputStream& operator<<(float f); + XmlOutputStream& operator<<(double d); +}; + +} + diff --git a/vespalib/src/vespa/vespalib/util/xmlstream.hpp b/vespalib/src/vespa/vespalib/util/xmlstream.hpp new file mode 100644 index 00000000000..82f1146df29 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/xmlstream.hpp @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "xmlstream.h" +#include <vespa/vespalib/util/exceptions.h> +#include <sstream> + +namespace vespalib::xml { + +template<typename T> +XmlAttribute::XmlAttribute(const std::string& name, T value, uint32_t flags) + : _name(name), + _value(), + _next() +{ + std::ostringstream ost; + if (flags & HEX) ost << std::hex << "0x"; + ost << value; + _value = ost.str(); + if (!isLegalName(name)) { + throw IllegalArgumentException("Name '" + name + "' contains " + "illegal XML characters and cannot be used as attribute name"); + } +} + +} |