diff options
| author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
|---|---|---|
| committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
| commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
| tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /staging_vespalib | |
Publish
Diffstat (limited to 'staging_vespalib')
289 files changed, 19466 insertions, 0 deletions
diff --git a/staging_vespalib/.gitignore b/staging_vespalib/.gitignore new file mode 100644 index 00000000000..727ed8ec9e8 --- /dev/null +++ b/staging_vespalib/.gitignore @@ -0,0 +1,5 @@ +bin +doc +lib +Makefile +Testing diff --git a/staging_vespalib/CMakeLists.txt b/staging_vespalib/CMakeLists.txt new file mode 100644 index 00000000000..5184d98d1ce --- /dev/null +++ b/staging_vespalib/CMakeLists.txt @@ -0,0 +1,51 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_define_module( + DEPENDS + fastos + vespalog + fastlib_fast + vespalib + fnet + + TESTS + src/tests/array + src/tests/benchmark + src/tests/bits + src/tests/clock + src/tests/cpu + src/tests/crc + src/tests/databuffer + src/tests/directio + src/tests/dotproduct + src/tests/encoding/base64 + src/tests/fileheader + src/tests/floatingpointtype + src/tests/growablebytebuffer + src/tests/json + src/tests/librarypool + src/tests/memorydatastore + src/tests/objectdump + src/tests/objects + src/tests/objectselection + src/tests/polymorphicarray + src/tests/programoptions + src/tests/rusage + src/tests/shutdownguard + src/tests/state_server + src/tests/stllike + src/tests/timer + src/tests/trace + src/tests/util/process_memory_stats + src/tests/xmlserializable + + LIBS + src/vespa/vespalib + src/vespa/vespalib/data + src/vespa/vespalib/encoding + src/vespa/vespalib/hwaccelrated + src/vespa/vespalib/net + src/vespa/vespalib/objects + src/vespa/vespalib/stllike + src/vespa/vespalib/trace + src/vespa/vespalib/util +) diff --git a/staging_vespalib/OWNERS b/staging_vespalib/OWNERS new file mode 100644 index 00000000000..30c53a630b0 --- /dev/null +++ b/staging_vespalib/OWNERS @@ -0,0 +1,3 @@ +arnej27959 +havardpe +balder diff --git a/staging_vespalib/README b/staging_vespalib/README new file mode 100644 index 00000000000..4cb562d8b79 --- /dev/null +++ b/staging_vespalib/README @@ -0,0 +1,3 @@ +Vespalib is the location for common and generic vespa utility classes. +This is the staging location for classes that are intended to be added +to vespalib, but aren't finished enough yet.
\ No newline at end of file diff --git a/staging_vespalib/src/.gitignore b/staging_vespalib/src/.gitignore new file mode 100644 index 00000000000..eadbd940e1c --- /dev/null +++ b/staging_vespalib/src/.gitignore @@ -0,0 +1,6 @@ +*.dsp +*.mak +Makefile.ini +config_command.sh +project.dsw +/module.conf diff --git a/staging_vespalib/src/Doxyfile b/staging_vespalib/src/Doxyfile new file mode 100644 index 00000000000..f16908c8204 --- /dev/null +++ b/staging_vespalib/src/Doxyfile @@ -0,0 +1,225 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Doxyfile 1.4.6 + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- +PROJECT_NAME = vespalib +PROJECT_NUMBER = +OUTPUT_DIRECTORY = ../doc/doxygen +CREATE_SUBDIRS = NO +OUTPUT_LANGUAGE = English +USE_WINDOWS_ENCODING = NO +BRIEF_MEMBER_DESC = YES +REPEAT_BRIEF = YES +ABBREVIATE_BRIEF = +ALWAYS_DETAILED_SEC = NO +INLINE_INHERITED_MEMB = NO +FULL_PATH_NAMES = YES +STRIP_FROM_PATH = +STRIP_FROM_INC_PATH = ./ +SHORT_NAMES = NO +JAVADOC_AUTOBRIEF = NO +MULTILINE_CPP_IS_BRIEF = NO +DETAILS_AT_TOP = NO +INHERIT_DOCS = YES +SEPARATE_MEMBER_PAGES = NO +TAB_SIZE = 8 +ALIASES = +OPTIMIZE_OUTPUT_FOR_C = NO +OPTIMIZE_OUTPUT_JAVA = NO +BUILTIN_STL_SUPPORT = NO +DISTRIBUTE_GROUP_DOC = NO +SUBGROUPING = YES +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- +EXTRACT_ALL = YES +EXTRACT_PRIVATE = NO +EXTRACT_STATIC = NO +EXTRACT_LOCAL_CLASSES = YES +EXTRACT_LOCAL_METHODS = NO +HIDE_UNDOC_MEMBERS = YES +HIDE_UNDOC_CLASSES = YES +HIDE_FRIEND_COMPOUNDS = NO +HIDE_IN_BODY_DOCS = NO +INTERNAL_DOCS = NO +CASE_SENSE_NAMES = YES +HIDE_SCOPE_NAMES = NO +SHOW_INCLUDE_FILES = YES +INLINE_INFO = YES +SORT_MEMBER_DOCS = YES +SORT_BRIEF_DOCS = NO +SORT_BY_SCOPE_NAME = NO +GENERATE_TODOLIST = YES +GENERATE_TESTLIST = YES +GENERATE_BUGLIST = YES +GENERATE_DEPRECATEDLIST= YES +ENABLED_SECTIONS = +MAX_INITIALIZER_LINES = 30 +SHOW_USED_FILES = YES +SHOW_DIRECTORIES = NO +FILE_VERSION_FILTER = +#--------------------------------------------------------------------------- +# configuration options related to warning and progress messages +#--------------------------------------------------------------------------- +QUIET = NO +WARNINGS = YES +WARN_IF_UNDOCUMENTED = YES +WARN_IF_DOC_ERROR = YES +WARN_NO_PARAMDOC = NO +WARN_FORMAT = "$file:$line: $text" +WARN_LOGFILE = +#--------------------------------------------------------------------------- +# configuration options related to the input files +#--------------------------------------------------------------------------- +INPUT = vespalib/util \ + vespalib/testkit +FILE_PATTERNS = +RECURSIVE = NO +EXCLUDE = +EXCLUDE_SYMLINKS = NO +EXCLUDE_PATTERNS = +EXAMPLE_PATH = +EXAMPLE_PATTERNS = +EXAMPLE_RECURSIVE = NO +IMAGE_PATH = +INPUT_FILTER = +FILTER_PATTERNS = +FILTER_SOURCE_FILES = NO +#--------------------------------------------------------------------------- +# configuration options related to source browsing +#--------------------------------------------------------------------------- +SOURCE_BROWSER = YES +INLINE_SOURCES = NO +STRIP_CODE_COMMENTS = YES +REFERENCED_BY_RELATION = NO +REFERENCES_RELATION = NO +USE_HTAGS = NO +VERBATIM_HEADERS = YES +#--------------------------------------------------------------------------- +# configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- +ALPHABETICAL_INDEX = YES +COLS_IN_ALPHA_INDEX = 5 +IGNORE_PREFIX = +#--------------------------------------------------------------------------- +# configuration options related to the HTML output +#--------------------------------------------------------------------------- +GENERATE_HTML = YES +HTML_OUTPUT = html +HTML_FILE_EXTENSION = .html +HTML_HEADER = +HTML_FOOTER = +HTML_STYLESHEET = +HTML_ALIGN_MEMBERS = YES +GENERATE_HTMLHELP = NO +CHM_FILE = +HHC_LOCATION = +GENERATE_CHI = NO +BINARY_TOC = NO +TOC_EXPAND = NO +DISABLE_INDEX = NO +ENUM_VALUES_PER_LINE = 4 +GENERATE_TREEVIEW = NO +TREEVIEW_WIDTH = 250 +#--------------------------------------------------------------------------- +# configuration options related to the LaTeX output +#--------------------------------------------------------------------------- +GENERATE_LATEX = NO +LATEX_OUTPUT = latex +LATEX_CMD_NAME = latex +MAKEINDEX_CMD_NAME = makeindex +COMPACT_LATEX = NO +PAPER_TYPE = a4wide +EXTRA_PACKAGES = +LATEX_HEADER = +PDF_HYPERLINKS = NO +USE_PDFLATEX = NO +LATEX_BATCHMODE = NO +LATEX_HIDE_INDICES = NO +#--------------------------------------------------------------------------- +# configuration options related to the RTF output +#--------------------------------------------------------------------------- +GENERATE_RTF = NO +RTF_OUTPUT = rtf +COMPACT_RTF = NO +RTF_HYPERLINKS = NO +RTF_STYLESHEET_FILE = +RTF_EXTENSIONS_FILE = +#--------------------------------------------------------------------------- +# configuration options related to the man page output +#--------------------------------------------------------------------------- +GENERATE_MAN = NO +MAN_OUTPUT = man +MAN_EXTENSION = .3 +MAN_LINKS = NO +#--------------------------------------------------------------------------- +# configuration options related to the XML output +#--------------------------------------------------------------------------- +GENERATE_XML = NO +XML_OUTPUT = xml +XML_SCHEMA = +XML_DTD = +XML_PROGRAMLISTING = YES +#--------------------------------------------------------------------------- +# configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- +GENERATE_AUTOGEN_DEF = NO +#--------------------------------------------------------------------------- +# configuration options related to the Perl module output +#--------------------------------------------------------------------------- +GENERATE_PERLMOD = NO +PERLMOD_LATEX = NO +PERLMOD_PRETTY = YES +PERLMOD_MAKEVAR_PREFIX = +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- +ENABLE_PREPROCESSING = YES +MACRO_EXPANSION = NO +EXPAND_ONLY_PREDEF = NO +SEARCH_INCLUDES = YES +INCLUDE_PATH = +INCLUDE_FILE_PATTERNS = +PREDEFINED = IAM_DOXYGEN +EXPAND_AS_DEFINED = +SKIP_FUNCTION_MACROS = YES +#--------------------------------------------------------------------------- +# Configuration::additions related to external references +#--------------------------------------------------------------------------- +TAGFILES = +GENERATE_TAGFILE = ../doc/doxygen/vespalib.tag +ALLEXTERNALS = NO +EXTERNAL_GROUPS = YES +PERL_PATH = /usr/bin/perl +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- +CLASS_DIAGRAMS = YES +HIDE_UNDOC_RELATIONS = YES +HAVE_DOT = YES +CLASS_GRAPH = YES +COLLABORATION_GRAPH = YES +GROUP_GRAPHS = YES +UML_LOOK = NO +TEMPLATE_RELATIONS = NO +INCLUDE_GRAPH = YES +INCLUDED_BY_GRAPH = YES +CALL_GRAPH = NO +GRAPHICAL_HIERARCHY = YES +DIRECTORY_GRAPH = YES +DOT_IMAGE_FORMAT = png +DOT_PATH = +DOTFILE_DIRS = +MAX_DOT_GRAPH_WIDTH = 1024 +MAX_DOT_GRAPH_HEIGHT = 1024 +MAX_DOT_GRAPH_DEPTH = 0 +DOT_TRANSPARENT = NO +DOT_MULTI_TARGETS = NO +GENERATE_LEGEND = YES +DOT_CLEANUP = YES +#--------------------------------------------------------------------------- +# Configuration::additions related to the search engine +#--------------------------------------------------------------------------- +SEARCHENGINE = NO diff --git a/staging_vespalib/src/module.conf.rhel6 b/staging_vespalib/src/module.conf.rhel6 new file mode 100644 index 00000000000..5867773215b --- /dev/null +++ b/staging_vespalib/src/module.conf.rhel6 @@ -0,0 +1,8 @@ +module staging_vespalib +require module fastos shared library fastos +require module log shared library vespalog +require module fastlib shared library fast +require module vespalib shared library vespalib +require module fnet shared library fnet +require module boost +provide library staging_vespalib diff --git a/staging_vespalib/src/testlist.txt b/staging_vespalib/src/testlist.txt new file mode 100644 index 00000000000..e67614239b2 --- /dev/null +++ b/staging_vespalib/src/testlist.txt @@ -0,0 +1,29 @@ +tests/array +tests/benchmark +tests/bits +tests/clock +tests/cpu +tests/crc +tests/databuffer +tests/directio +tests/dotproduct +tests/encoding/base64 +tests/fileheader +tests/floatingpointtype +tests/growablebytebuffer +tests/json +tests/librarypool +tests/memorydatastore +tests/objectdump +tests/objects +tests/objectselection +tests/programoptions +tests/polymorphicarray +tests/rusage +tests/shutdownguard +tests/state_server +tests/stllike +tests/timer +tests/trace +tests/util/process_memory_stats +tests/xmlserializable diff --git a/staging_vespalib/src/tests/.gitignore b/staging_vespalib/src/tests/.gitignore new file mode 100644 index 00000000000..c473b24b98a --- /dev/null +++ b/staging_vespalib/src/tests/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +testrunner +*_test diff --git a/staging_vespalib/src/tests/array/.gitignore b/staging_vespalib/src/tests/array/.gitignore new file mode 100644 index 00000000000..154e6464c50 --- /dev/null +++ b/staging_vespalib/src/tests/array/.gitignore @@ -0,0 +1,5 @@ +/sort_benchmark +/allocinarray_benchmark +staging_vespalib_allocinarray_test_app +staging_vespalib_allocinarray_benchmark_app +staging_vespalib_sort_benchmark_app diff --git a/staging_vespalib/src/tests/array/CMakeLists.txt b/staging_vespalib/src/tests/array/CMakeLists.txt new file mode 100644 index 00000000000..7b9841d474a --- /dev/null +++ b/staging_vespalib/src/tests/array/CMakeLists.txt @@ -0,0 +1,22 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_allocinarray_test_app + SOURCES + allocinarray_test.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_allocinarray_test_app COMMAND staging_vespalib_allocinarray_test_app) +vespa_add_executable(staging_vespalib_sort_benchmark_app + SOURCES + sort_benchmark.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_sort_benchmark_app COMMAND staging_vespalib_sort_benchmark_app BENCHMARK) +vespa_add_executable(staging_vespalib_allocinarray_benchmark_app + SOURCES + allocinarray_benchmark.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_allocinarray_benchmark_app COMMAND staging_vespalib_allocinarray_benchmark_app BENCHMARK) diff --git a/staging_vespalib/src/tests/array/DESC b/staging_vespalib/src/tests/array/DESC new file mode 100644 index 00000000000..59c59c2c50a --- /dev/null +++ b/staging_vespalib/src/tests/array/DESC @@ -0,0 +1 @@ +Array test. Take a look at array_test.cpp for details. diff --git a/staging_vespalib/src/tests/array/FILES b/staging_vespalib/src/tests/array/FILES new file mode 100644 index 00000000000..fa3825f832f --- /dev/null +++ b/staging_vespalib/src/tests/array/FILES @@ -0,0 +1 @@ +array_test.cpp diff --git a/staging_vespalib/src/tests/array/allocinarray_benchmark.cpp b/staging_vespalib/src/tests/array/allocinarray_benchmark.cpp new file mode 100644 index 00000000000..9f365ff937c --- /dev/null +++ b/staging_vespalib/src/tests/array/allocinarray_benchmark.cpp @@ -0,0 +1,124 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/rusage.h> +#include <vespa/vespalib/util/optimized.h> +#include <vespa/vespalib/util/allocinarray.h> +#include <vespa/log/log.h> +LOG_SETUP("allocinarray_benchmark"); + +using namespace vespalib; + +class Test : public TestApp +{ +public: +private: + void benchmarkTree(size_t count); + void benchmarkTreeInArray(size_t count); + int Main(); +}; + +template <typename T> +class TreeNode +{ +public: + typedef TreeNode * P; + TreeNode(const T & p) :_left(NULL), _right(NULL), _payLoad(p) { } + ~TreeNode() { + if (_left) { + delete _left; + } + if (_right) { + delete _right; + } + } + P left() { return _left; } + P right() { return _right; } + void left(P l) { _left = l; } + void right(P l) { _right = l; } +private: + P _left; + P _right; + T _payLoad; +}; + +template <typename T> +class RefTreeNode +{ +public: + typedef uint32_t P; + RefTreeNode(const T & p) :_left(-1), _right(-1), _payLoad(p) { } + P left() { return _left; } + P right() { return _right; } + void left(P l) { _left = l; } + void right(P l) { _right = l; } +private: + P _left; + P _right; + T _payLoad; +}; + +typedef TreeNode<long> N; +typedef RefTreeNode<long> R; +typedef AllocInArray<R, vespalib::Array<R, MMapAlloc> > Store; + +void populate(Store & store, uint32_t parent, size_t depth) +{ + if (depth > 0) { + store[parent].left(store.alloc(R(0))); + populate(store, store[parent].left(), depth-1); + store[parent].right(store.alloc(R(1))); + populate(store, store[parent].right(), depth-1); + } +} + +void populate(N * parent, size_t depth) +{ + if (depth > 0) { + parent->left(new N(0)); + populate(parent->left(), depth-1); + parent->right(new N(1)); + populate(parent->right(), depth-1); + } +} + +void Test::benchmarkTree(size_t count) +{ + N root(0); + size_t depth = Optimized::msbIdx(count); + populate(&root, depth); +} + +void Test::benchmarkTreeInArray(size_t count) +{ + Store store; + store.alloc(R(0)); + size_t depth = Optimized::msbIdx(count); + populate(store, 0, depth); +} + +int +Test::Main() +{ + std::string type("direct"); + size_t count = 1000000; + if (_argc > 1) { + type = _argv[1]; + } + if (_argc > 2) { + count = strtol(_argv[2], NULL, 0); + } + TEST_INIT("allocinarray_benchmark"); + fastos::TimeStamp start(fastos::ClockSystem::now()); + if (type == "direct") { + benchmarkTree(count); + } else { + benchmarkTreeInArray(count); + } + LOG(info, "rusage = {\n%s\n}", vespalib::RUsage::createSelf(start).toString().c_str()); + ASSERT_EQUAL(0, kill(0, SIGPROF)); + TEST_DONE(); +} + +TEST_APPHOOK(Test); + diff --git a/staging_vespalib/src/tests/array/allocinarray_test.cpp b/staging_vespalib/src/tests/array/allocinarray_test.cpp new file mode 100644 index 00000000000..3a40a71f288 --- /dev/null +++ b/staging_vespalib/src/tests/array/allocinarray_test.cpp @@ -0,0 +1,75 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +#include <vespa/vespalib/util/array.h> +#include <vespa/vespalib/util/allocinarray.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/testkit/testapp.h> +#include <deque> + +LOG_SETUP("array_test"); + +using namespace vespalib; + +class Test : public TestApp +{ +public: + int Main(); +private: + template <typename T, typename V> + void testAllocInArray(); +}; + +int +Test::Main() +{ + TEST_INIT("allocinarray_test"); + + testAllocInArray<int64_t, vespalib::Array<int64_t> >(); + testAllocInArray<int64_t, vespalib::Array<int64_t, vespalib::DefaultAlloc> >(); + testAllocInArray<int64_t, std::vector<int64_t> >(); + testAllocInArray<int64_t, std::deque<int64_t> >(); + + TEST_DONE(); +} + +template <typename T, typename V> +void Test::testAllocInArray() +{ + typedef AllocInArray<T, V> AA; + AA alloc; + EXPECT_EQUAL(0ul, alloc.size()); + EXPECT_EQUAL(0ul, alloc.alloc(1)); + EXPECT_EQUAL(1ul, alloc.size()); + EXPECT_EQUAL(1, alloc[0]); + alloc.free(0); + EXPECT_EQUAL(0ul, alloc.size()); + alloc.free(0); + EXPECT_EQUAL(0ul, alloc.size()); + alloc.free(1); + EXPECT_EQUAL(0ul, alloc.size()); + + alloc.alloc(7); + alloc.alloc(17); + alloc.alloc(-17); + EXPECT_EQUAL(3ul, alloc.size()); + EXPECT_EQUAL(7, alloc[0]); + EXPECT_EQUAL(17, alloc[1]); + EXPECT_EQUAL(-17, alloc[2]); + alloc[1] = 99; + EXPECT_EQUAL(99, alloc[1]); + alloc.free(1); + EXPECT_EQUAL(2ul, alloc.size()); + EXPECT_EQUAL(7, alloc[0]); + EXPECT_EQUAL(-17, alloc[2]); + EXPECT_EQUAL(1ul, alloc.alloc(103)); + EXPECT_EQUAL(3ul, alloc.size()); + EXPECT_EQUAL(7, alloc[0]); + EXPECT_EQUAL(103, alloc[1]); + EXPECT_EQUAL(-17, alloc[2]); + + alloc.clear(); + EXPECT_EQUAL(0ul, alloc.size()); +} + +TEST_APPHOOK(Test) diff --git a/staging_vespalib/src/tests/array/sort_benchmark.cpp b/staging_vespalib/src/tests/array/sort_benchmark.cpp new file mode 100644 index 00000000000..bb304f92d58 --- /dev/null +++ b/staging_vespalib/src/tests/array/sort_benchmark.cpp @@ -0,0 +1,182 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/rusage.h> +#include <vespa/vespalib/util/array.h> +#include <vespa/log/log.h> +LOG_SETUP("sort_benchmark"); + +using namespace vespalib; + +class Test : public TestApp +{ +public: +private: + template<typename T> + vespalib::Array<T> create(size_t count); + template<typename T> + void sortDirect(size_t count); + template<typename T> + void sortInDirect(size_t count); + int Main(); +}; + +template<size_t N> +class TT +{ +public: + TT(uint64_t v) : _v(v) { } + bool operator < (const TT & rhs) const { return _v < rhs._v; } +private: + uint64_t _v; + uint8_t _payLoad[N - sizeof(uint64_t)]; +}; + +template <typename T> +class I +{ +public: + I(const T * p) : _p(p) { } + bool operator < (const I & rhs) const { return *_p < *rhs._p; } +private: + const T * _p; +}; + +template<typename T> +vespalib::Array<T> +Test::create(size_t count) +{ + vespalib::Array<T> v; + v.reserve(count); + srand(0); + for (size_t i(0); i < count; i++) { + v.push_back(rand()); + } + return v; +} + +template<typename T> +void Test::sortDirect(size_t count) +{ + vespalib::Array<T> v(create<T>(count)); + LOG(info, "Running sortDirect with %ld count and payload of %ld", v.size(), sizeof(T)); + for (size_t j=0; j < 10; j++) { + vespalib::Array<T> t(v); + std::sort(t.begin(), t.end()); + } +} + +template<typename T> +void Test::sortInDirect(size_t count) +{ + vespalib::Array<T> k(create<T>(count)); + LOG(info, "Running sortInDirect with %ld count and payload of %ld", k.size(), sizeof(T)); + vespalib::Array< I<T> > v; + v.reserve(k.size()); + for (size_t i(0), m(k.size()); i < m; i++) { + v.push_back(&k[i]); + } + for (size_t j=0; j < 10; j++) { + vespalib::Array< I<T> > t(v); + std::sort(t.begin(), t.end()); + } +} + +int +Test::Main() +{ + std::string type("sortdirect"); + size_t count = 1000000; + size_t payLoad = 0; + if (_argc > 1) { + type = _argv[1]; + } + if (_argc > 2) { + count = strtol(_argv[2], NULL, 0); + } + if (_argc > 3) { + payLoad = strtol(_argv[3], NULL, 0); + } + TEST_INIT("sort_benchmark"); + fastos::TimeStamp start(fastos::ClockSystem::now()); + if (payLoad < 8) { + typedef TT<8> T; + if (type == "sortdirect") { + sortDirect<T>(count); + } else if (type == "sortindirect") { + sortInDirect<T>(count); + } else { + LOG(warning, "type '%s' is unknown", type.c_str()); + } + } else if (payLoad < 16) { + typedef TT<16> T; + if (type == "sortdirect") { + sortDirect<T>(count); + } else if (type == "sortindirect") { + sortInDirect<T>(count); + } else { + LOG(warning, "type '%s' is unknown", type.c_str()); + } + } else if (payLoad < 32) { + typedef TT<32> T; + if (type == "sortdirect") { + sortDirect<T>(count); + } else if (type == "sortindirect") { + sortInDirect<T>(count); + } else { + LOG(warning, "type '%s' is unknown", type.c_str()); + } + } else if (payLoad < 64) { + typedef TT<64> T; + if (type == "sortdirect") { + sortDirect<T>(count); + } else if (type == "sortindirect") { + sortInDirect<T>(count); + } else { + LOG(warning, "type '%s' is unknown", type.c_str()); + } + } else if (payLoad < 128) { + typedef TT<128> T; + if (type == "sortdirect") { + sortDirect<T>(count); + } else if (type == "sortindirect") { + sortInDirect<T>(count); + } else { + LOG(warning, "type '%s' is unknown", type.c_str()); + } + } else if (payLoad < 256) { + typedef TT<256> T; + if (type == "sortdirect") { + sortDirect<T>(count); + } else if (type == "sortindirect") { + sortInDirect<T>(count); + } else { + LOG(warning, "type '%s' is unknown", type.c_str()); + } + } else if (payLoad < 512) { + typedef TT<512> T; + if (type == "sortdirect") { + sortDirect<T>(count); + } else if (type == "sortindirect") { + sortInDirect<T>(count); + } else { + LOG(warning, "type '%s' is unknown", type.c_str()); + } + } else { + typedef TT<1024> T; + LOG(info, "Payload %ld is too big to make any sense. Using %ld.", payLoad, sizeof(T)); + if (type == "sortdirect") { + sortDirect<T>(count); + } else if (type == "sortindirect") { + sortInDirect<T>(count); + } else { + LOG(warning, "type '%s' is unknown", type.c_str()); + } + } + LOG(info, "rusage = {\n%s\n}", vespalib::RUsage::createSelf(start).toString().c_str()); + ASSERT_EQUAL(0, kill(0, SIGPROF)); + TEST_DONE(); +} + +TEST_APPHOOK(Test); + diff --git a/staging_vespalib/src/tests/benchmark/.gitignore b/staging_vespalib/src/tests/benchmark/.gitignore new file mode 100644 index 00000000000..789e52c1370 --- /dev/null +++ b/staging_vespalib/src/tests/benchmark/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +benchmark_test +staging_vespalib_benchmark_test_app diff --git a/staging_vespalib/src/tests/benchmark/CMakeLists.txt b/staging_vespalib/src/tests/benchmark/CMakeLists.txt new file mode 100644 index 00000000000..be8ea976c1d --- /dev/null +++ b/staging_vespalib/src/tests/benchmark/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_benchmark_test_app + SOURCES + benchmark.cpp + testbase.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_benchmark_test NO_VALGRIND COMMAND sh benchmark_test.sh BENCHMARK) diff --git a/staging_vespalib/src/tests/benchmark/DESC b/staging_vespalib/src/tests/benchmark/DESC new file mode 100644 index 00000000000..cc3b175dace --- /dev/null +++ b/staging_vespalib/src/tests/benchmark/DESC @@ -0,0 +1 @@ +This is so far a test to show benchmarks of different programming techniques. diff --git a/staging_vespalib/src/tests/benchmark/FILES b/staging_vespalib/src/tests/benchmark/FILES new file mode 100644 index 00000000000..30525d50897 --- /dev/null +++ b/staging_vespalib/src/tests/benchmark/FILES @@ -0,0 +1 @@ +benchmark.cpp diff --git a/staging_vespalib/src/tests/benchmark/benchmark.cpp b/staging_vespalib/src/tests/benchmark/benchmark.cpp new file mode 100644 index 00000000000..1c4c5a59c63 --- /dev/null +++ b/staging_vespalib/src/tests/benchmark/benchmark.cpp @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("benchmark_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include "testbase.h" + +using namespace vespalib; + +TEST_SETUP(Test) + +int +Test::Main() +{ + TEST_INIT("benchmark_test"); + + if (_argc > 1) { + size_t concurrency(1); + size_t numRuns(1000); + if (_argc > 2) { + numRuns = strtoul(_argv[2], NULL, 0); + if (_argc > 3) { + concurrency = strtoul(_argv[3], NULL, 0); + } + } + Benchmark::run(_argv[1], numRuns, concurrency); + } + + TEST_DONE(); +} diff --git a/staging_vespalib/src/tests/benchmark/benchmark_test.sh b/staging_vespalib/src/tests/benchmark/benchmark_test.sh new file mode 100755 index 00000000000..ce0a56ee545 --- /dev/null +++ b/staging_vespalib/src/tests/benchmark/benchmark_test.sh @@ -0,0 +1,19 @@ +#!/bin/bash +TIME=time + +$TIME ./staging_vespalib_benchmark_test_app vespalib::ParamByReferenceVectorInt 200000 1 +$TIME ./staging_vespalib_benchmark_test_app vespalib::ParamByValueVectorInt 4000 1 +$TIME ./staging_vespalib_benchmark_test_app vespalib::ParamByReferenceVectorString 30000 1 +$TIME ./staging_vespalib_benchmark_test_app vespalib::ParamByValueVectorString 40 1 +$TIME ./staging_vespalib_benchmark_test_app vespalib::ReturnByReferenceVectorString 10 1 +$TIME ./staging_vespalib_benchmark_test_app vespalib::ReturnByValueVectorString 10 1 +$TIME ./staging_vespalib_benchmark_test_app vespalib::ReturnByValueMultiVectorString 10 1 +$TIME ./staging_vespalib_benchmark_test_app vespalib::ClockSystem 1000 1 +$TIME ./staging_vespalib_benchmark_test_app vespalib::ClockGToD 1000 1 +$TIME ./staging_vespalib_benchmark_test_app vespalib::ClockGToD 20000 1 +$TIME./staging_vespalib_benchmark_test_app vespalib::ClockREALTIME 1000 1 +$TIME ./staging_vespalib_benchmark_test_app vespalib::ClockMONOTONIC 1000 1 +$TIME ./staging_vespalib_benchmark_test_app vespalib::ClockMONOTONIC_RAW 1000 1 +$TIME ./staging_vespalib_benchmark_test_app vespalib::ClockPROCESS_CPUTIME_ID 2500 1 +$TIME ./staging_vespalib_benchmark_test_app vespalib::ClockTHREAD_CPUTIME_ID 2500 1 +$TIME ./staging_vespalib_benchmark_test_app vespalib::CreateVespalibString 20000 1 diff --git a/staging_vespalib/src/tests/benchmark/testbase.cpp b/staging_vespalib/src/tests/benchmark/testbase.cpp new file mode 100644 index 00000000000..9ac20638218 --- /dev/null +++ b/staging_vespalib/src/tests/benchmark/testbase.cpp @@ -0,0 +1,278 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +//#include <linux/time.h> +#include <vespa/log/log.h> +#include <vespa/fastos/timestamp.h> +#include "testbase.h" + +LOG_SETUP(".testbase"); + +namespace vespalib { + +IMPLEMENT_IDENTIFIABLE_ABSTRACT_NS(vespalib, Benchmark, Identifiable); + +IMPLEMENT_BENCHMARK(ParamByReferenceVectorInt, Benchmark); +IMPLEMENT_BENCHMARK(ParamByValueVectorInt, Benchmark); +IMPLEMENT_BENCHMARK(ParamByReferenceVectorString, Benchmark); +IMPLEMENT_BENCHMARK(ParamByValueVectorString, Benchmark); +IMPLEMENT_BENCHMARK(ReturnByReferenceVectorString, Benchmark); +IMPLEMENT_BENCHMARK(ReturnByValueVectorString, Benchmark); +IMPLEMENT_BENCHMARK(ReturnByValueMultiVectorString, Benchmark); +IMPLEMENT_BENCHMARK(ClockSystem, Benchmark); +IMPLEMENT_BENCHMARK(ClockREALTIME, Benchmark); +IMPLEMENT_BENCHMARK(ClockMONOTONIC, Benchmark); +IMPLEMENT_BENCHMARK(ClockMONOTONIC_RAW, Benchmark); +IMPLEMENT_BENCHMARK(ClockPROCESS_CPUTIME_ID, Benchmark); +IMPLEMENT_BENCHMARK(ClockTHREAD_CPUTIME_ID, Benchmark); +IMPLEMENT_BENCHMARK(CreateVespalibString, Benchmark); + +void Benchmark::run(const char *name, size_t numRuns, size_t concurrency) +{ + const Identifiable::RuntimeClass * cInfo = Identifiable::classFromName(name); + if (cInfo) { + std::unique_ptr<Benchmark> test(static_cast<Benchmark *>(cInfo->create())); + test->run(numRuns, concurrency); + } else { + LOG(warning, "Could not find any test with the name %s", name); + } +} +void Benchmark::run(size_t numRuns, size_t concurrency) +{ + LOG(info, "Starting benchmark %s with %ld threads and %ld rep", getClass().name(), concurrency, numRuns); + for (size_t i(0); i < numRuns; i++) { + onRun(); + } + LOG(info, "Stopping benchmark %s", getClass().name()); +} + +size_t ParamByReferenceVectorInt::callByReference(const Vector & values) const +{ + return values.size(); +} + +size_t ParamByReferenceVectorInt::onRun() +{ + Vector values(1000); + size_t sum(0); + for (size_t i=0; i < 1000; i++) { + sum += callByReference(values); + } + return sum; +} + +size_t ParamByValueVectorInt::callByValue(Vector values) const +{ + return values.size(); +} + +size_t ParamByValueVectorInt::onRun() +{ + Vector values(1000); + size_t sum(0); + for (size_t i=0; i < 1000; i++) { + sum += callByValue(values); + } + return sum; +} + +size_t ParamByReferenceVectorString::callByReference(const Vector & values) const +{ + return values.size(); +} + +size_t ParamByReferenceVectorString::onRun() +{ + Vector values(1000, "This is a simple string copy test"); + size_t sum(0); + for (size_t i=0; i < 1000; i++) { + sum += callByReference(values); + } + return sum; +} + +size_t ParamByValueVectorString::callByValue(Vector values) const +{ + return values.size(); +} + +size_t ParamByValueVectorString::onRun() +{ + Vector values(1000, "This is a simple string copy test"); + size_t sum(0); + for (size_t i=0; i < 1000; i++) { + sum += callByValue(values); + } + return sum; +} + +const ReturnByReferenceVectorString::Vector & ReturnByReferenceVectorString::returnByReference(Vector & param) const +{ + Vector values(1000, "return by value"); + param.swap(values); + return param; +} + +size_t ReturnByReferenceVectorString::onRun() +{ + size_t sum(0); + for (size_t i=0; i < 1000; i++) { + Vector values; + sum += returnByReference(values).size(); + } + return sum; +} + +ReturnByValueVectorString::Vector ReturnByValueVectorString::returnByValue() const +{ + Vector values; + if (rand() % 7) { + Vector tmp(1000, "return by value"); + values.swap(tmp); + } else { + Vector tmp(1000, "Return by value"); + values.swap(tmp); + } + return values; +} + +size_t ReturnByValueVectorString::onRun() +{ + size_t sum(0); + for (size_t i=0; i < 1000; i++) { + sum += returnByValue().size(); + } + return sum; +} + +ReturnByValueMultiVectorString::Vector ReturnByValueMultiVectorString::returnByValue() const +{ + if (rand() % 7) { + Vector values(1000, "return by value"); + return values; + } else { + Vector values(1000, "Return by value"); + return values; + } +} + +size_t ReturnByValueMultiVectorString::onRun() +{ + size_t sum(0); + for (size_t i=0; i < 1000; i++) { + sum += returnByValue().size(); + } + return sum; +} + +size_t ClockSystem::onRun() +{ + fastos::TimeStamp start(fastos::ClockSystem::now()); + fastos::TimeStamp end(start); + for (size_t i=0; i < 1000; i++) { + end = fastos::ClockSystem::now(); + } + return (start - end).ns(); +} + +size_t ClockREALTIME::onRun() +{ + struct timespec ts; + int foo = clock_gettime(CLOCK_REALTIME, &ts); + assert(foo == 0); + (void) foo; + fastos::TimeStamp start(ts.tv_sec*1000L*1000L*1000L + ts.tv_nsec); + fastos::TimeStamp end(start); + for (size_t i=0; i < 1000; i++) { + clock_gettime(CLOCK_REALTIME, &ts); + end = ts.tv_sec*1000L*1000L*1000L + ts.tv_nsec; + } + return (start - end).ns(); +} + +size_t ClockMONOTONIC::onRun() +{ + struct timespec ts; + int foo = clock_gettime(CLOCK_MONOTONIC, &ts); + assert(foo == 0); + (void) foo; + fastos::TimeStamp start(ts.tv_sec*1000L*1000L*1000L + ts.tv_nsec); + fastos::TimeStamp end(start); + for (size_t i=0; i < 1000; i++) { + clock_gettime(CLOCK_MONOTONIC, &ts); + end = ts.tv_sec*1000L*1000L*1000L + ts.tv_nsec; + } + return (start - end).ns(); +} + +ClockMONOTONIC_RAW::ClockMONOTONIC_RAW() +{ +#ifndef CLOCK_MONOTONIC_RAW + LOG(warning, "CLOCK_MONOTONIC_RAW is not defined, using CLOCK_MONOTONIC instead."); +#endif +} + +#ifndef CLOCK_MONOTONIC_RAW + #define CLOCK_MONOTONIC_RAW CLOCK_MONOTONIC +#endif + +size_t ClockMONOTONIC_RAW::onRun() +{ + struct timespec ts; + int foo = clock_gettime(CLOCK_MONOTONIC_RAW, &ts); + assert(foo == 0); + (void) foo; + fastos::TimeStamp start(ts.tv_sec*1000L*1000L*1000L + ts.tv_nsec); + fastos::TimeStamp end(start); + for (size_t i=0; i < 1000; i++) { + clock_gettime(CLOCK_MONOTONIC_RAW, &ts); + end = ts.tv_sec*1000L*1000L*1000L + ts.tv_nsec; + } + return (start - end).ns(); +} + +size_t ClockPROCESS_CPUTIME_ID::onRun() +{ + struct timespec ts; + int foo = clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts); + assert(foo == 0); + (void) foo; + fastos::TimeStamp start(ts.tv_sec*1000L*1000L*1000L + ts.tv_nsec); + fastos::TimeStamp end(start); + for (size_t i=0; i < 1000; i++) { + clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts); + end = ts.tv_sec*1000L*1000L*1000L + ts.tv_nsec; + } + return (start - end).ns(); +} + +size_t ClockTHREAD_CPUTIME_ID::onRun() +{ + struct timespec ts; + int foo = clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts); + assert(foo == 0); + (void) foo; + fastos::TimeStamp start(ts.tv_sec*1000L*1000L*1000L + ts.tv_nsec); + fastos::TimeStamp end(start); + for (size_t i=0; i < 1000; i++) { + clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts); + end = ts.tv_sec*1000L*1000L*1000L + ts.tv_nsec; + } + return (start - end).ns(); +} + +size_t CreateVespalibString::onRun() +{ + size_t sum(0); + const char * text1("Dette er en passe"); + const char * text2(" kort streng som passer paa stacken"); + char text[100]; + strcpy(text, text1); + strcat(text, text2); + for (size_t i=0; i < 1000; i++) { + string s(text); + sum += s.size(); + } + return sum; +} + +} diff --git a/staging_vespalib/src/tests/benchmark/testbase.h b/staging_vespalib/src/tests/benchmark/testbase.h new file mode 100644 index 00000000000..45b0e30019e --- /dev/null +++ b/staging_vespalib/src/tests/benchmark/testbase.h @@ -0,0 +1,169 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/objects/identifiable.h> +#include <vector> +#include <string> + +#define BENCHMARK_BASE_CID(n) (0x1000000 + n) + +#define CID_vespalib_Benchmark BENCHMARK_BASE_CID(0) +#define CID_vespalib_ParamByReferenceVectorInt BENCHMARK_BASE_CID(1) +#define CID_vespalib_ParamByReferenceVectorString BENCHMARK_BASE_CID(2) +#define CID_vespalib_ParamByValueVectorInt BENCHMARK_BASE_CID(3) +#define CID_vespalib_ParamByValueVectorString BENCHMARK_BASE_CID(4) +#define CID_vespalib_ReturnByReferenceVectorString BENCHMARK_BASE_CID(5) +#define CID_vespalib_ReturnByValueVectorString BENCHMARK_BASE_CID(6) +#define CID_vespalib_ReturnByValueMultiVectorString BENCHMARK_BASE_CID(7) +#define CID_vespalib_ClockSystem BENCHMARK_BASE_CID(8) +#define CID_vespalib_ClockREALTIME BENCHMARK_BASE_CID(10) +#define CID_vespalib_ClockMONOTONIC BENCHMARK_BASE_CID(11) +#define CID_vespalib_ClockMONOTONIC_RAW BENCHMARK_BASE_CID(12) +#define CID_vespalib_ClockPROCESS_CPUTIME_ID BENCHMARK_BASE_CID(13) +#define CID_vespalib_ClockTHREAD_CPUTIME_ID BENCHMARK_BASE_CID(14) +#define CID_vespalib_CreateVespalibString BENCHMARK_BASE_CID(15) + +#define DECLARE_BENCHMARK(a) DECLARE_IDENTIFIABLE_NS(vespalib, a) +#define IMPLEMENT_BENCHMARK(a, d) IMPLEMENT_IDENTIFIABLE_NS(vespalib, a, d) + +namespace vespalib { + +class Benchmark : public Identifiable +{ +public: + DECLARE_IDENTIFIABLE_ABSTRACT_NS(vespalib, Benchmark); + void run(size_t numRuns, size_t concurrency=1); + static void run(const char * testName, size_t numRuns, size_t concurrency); +private: + virtual size_t onRun() = 0; +}; + +class ParamByReferenceVectorInt : public Benchmark +{ +public: + DECLARE_BENCHMARK(ParamByReferenceVectorInt); +private: + typedef std::vector<int> Vector; + size_t callByReference(const Vector & values) const __attribute__((noinline)); + virtual size_t onRun(); +}; + +class ParamByValueVectorInt : public Benchmark +{ +public: + DECLARE_BENCHMARK(ParamByValueVectorInt); +private: + typedef std::vector<int> Vector; + size_t callByValue(Vector values) const __attribute__((noinline)); + virtual size_t onRun(); +}; + +class ParamByReferenceVectorString : public Benchmark +{ +public: + DECLARE_BENCHMARK(ParamByReferenceVectorString); +private: + typedef std::vector<std::string> Vector; + size_t callByReference(const Vector & values) const __attribute__((noinline)); + virtual size_t onRun(); +}; + +class ParamByValueVectorString : public Benchmark +{ +public: + DECLARE_BENCHMARK(ParamByValueVectorString); +private: + typedef std::vector<std::string> Vector; + size_t callByValue(Vector values) const __attribute__((noinline)); + virtual size_t onRun(); +}; + +class ReturnByReferenceVectorString : public Benchmark +{ +public: + DECLARE_BENCHMARK(ReturnByReferenceVectorString); +private: + typedef std::vector<std::string> Vector; + const Vector & returnByReference(Vector & values) const __attribute__((noinline)); + virtual size_t onRun(); +}; + +class ReturnByValueVectorString : public Benchmark +{ +public: + DECLARE_BENCHMARK(ReturnByValueVectorString); +private: + typedef std::vector<std::string> Vector; + Vector returnByValue() const __attribute__((noinline)); + virtual size_t onRun(); +}; + +class ReturnByValueMultiVectorString : public Benchmark +{ +public: + DECLARE_BENCHMARK(ReturnByValueMultiVectorString); +private: + typedef std::vector<std::string> Vector; + Vector returnByValue() const __attribute__((noinline)); + virtual size_t onRun(); +}; + +class CreateVespalibString : public Benchmark +{ +public: + DECLARE_BENCHMARK(CreateVespalibString); +private: + virtual size_t onRun(); +}; + +class ClockSystem : public Benchmark +{ +public: + DECLARE_BENCHMARK(ClockSystem); +private: + virtual size_t onRun(); +}; + +class ClockREALTIME : public Benchmark +{ +public: + DECLARE_BENCHMARK(ClockREALTIME); +private: + virtual size_t onRun(); +}; + +class ClockMONOTONIC : public Benchmark +{ +public: + DECLARE_BENCHMARK(ClockMONOTONIC); +private: + virtual size_t onRun(); +}; + +class ClockMONOTONIC_RAW : public Benchmark +{ +public: + DECLARE_BENCHMARK(ClockMONOTONIC_RAW); + ClockMONOTONIC_RAW(); +private: + virtual size_t onRun(); +}; + +class ClockPROCESS_CPUTIME_ID : public Benchmark +{ +public: + DECLARE_BENCHMARK(ClockPROCESS_CPUTIME_ID); +private: + virtual size_t onRun(); +}; + +class ClockTHREAD_CPUTIME_ID : public Benchmark +{ +public: + DECLARE_BENCHMARK(ClockTHREAD_CPUTIME_ID); +private: + virtual size_t onRun(); +}; + +} + diff --git a/staging_vespalib/src/tests/bits/.gitignore b/staging_vespalib/src/tests/bits/.gitignore new file mode 100644 index 00000000000..82481fe3c30 --- /dev/null +++ b/staging_vespalib/src/tests/bits/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +bits_test +staging_vespalib_bits_test_app diff --git a/staging_vespalib/src/tests/bits/CMakeLists.txt b/staging_vespalib/src/tests/bits/CMakeLists.txt new file mode 100644 index 00000000000..693b6cbb6cc --- /dev/null +++ b/staging_vespalib/src/tests/bits/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_bits_test_app + SOURCES + bits_test.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_bits_test_app COMMAND staging_vespalib_bits_test_app) diff --git a/staging_vespalib/src/tests/bits/DESC b/staging_vespalib/src/tests/bits/DESC new file mode 100644 index 00000000000..2ea242366ff --- /dev/null +++ b/staging_vespalib/src/tests/bits/DESC @@ -0,0 +1 @@ +bit manipultaion test. Take a look at bits.cpp for details. diff --git a/staging_vespalib/src/tests/bits/FILES b/staging_vespalib/src/tests/bits/FILES new file mode 100644 index 00000000000..95ad604b592 --- /dev/null +++ b/staging_vespalib/src/tests/bits/FILES @@ -0,0 +1 @@ +bits.cpp diff --git a/staging_vespalib/src/tests/bits/bits_test.cpp b/staging_vespalib/src/tests/bits/bits_test.cpp new file mode 100644 index 00000000000..83e60f0f214 --- /dev/null +++ b/staging_vespalib/src/tests/bits/bits_test.cpp @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("bits_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/bits.h> +#include <boost/crc.hpp> + +using namespace vespalib; + +class Test : public TestApp +{ +public: + int Main(); + template <typename T> + void testFixed(const T * v, size_t sz); + void testBuffer(); +}; + +int +Test::Main() +{ + TEST_INIT("bits_test"); + uint8_t u8[5] = { 0, 1, 127, 135, 255 }; + testFixed(u8, sizeof(u8)/sizeof(u8[0])); + uint16_t u16[5] = { 0, 1, 127, 135, 255 }; + testFixed(u16, sizeof(u16)/sizeof(u16[0])); + uint32_t u32[5] = { 0, 1, 127, 135, 255 }; + testFixed(u32, sizeof(u32)/sizeof(u32[0])); + uint64_t u64[5] = { 0, 1, 127, 135, 255 }; + testFixed(u64, sizeof(u64)/sizeof(u64[0])); + testBuffer(); + TEST_DONE(); +} + +template <typename T> +void Test::testFixed(const T * v, size_t sz) +{ + EXPECT_EQUAL(0u, Bits::reverse(static_cast<T>(0))); + EXPECT_EQUAL(1ul << (sizeof(T)*8 - 1), Bits::reverse(static_cast<T>(1))); + EXPECT_EQUAL(static_cast<T>(-1), Bits::reverse(static_cast<T>(-1))); + for (size_t i(0); i < sz; i++) { + EXPECT_EQUAL(Bits::reverse(v[i]), boost::detail::reflector<sizeof(T)*8>::reflect(v[i])); + EXPECT_EQUAL(Bits::reverse(Bits::reverse(v[i])), v[i]); + } +} + +void Test::testBuffer() +{ + uint64_t a(0x0102040810204080ul); + uint64_t b(a); + Bits::reverse(&a, sizeof(a)); + EXPECT_EQUAL(a, Bits::reverse(b)); + Bits::reverse(&a, sizeof(a)); + EXPECT_EQUAL(a, b); +} + +TEST_APPHOOK(Test) diff --git a/staging_vespalib/src/tests/clock/.gitignore b/staging_vespalib/src/tests/clock/.gitignore new file mode 100644 index 00000000000..1826ba1563b --- /dev/null +++ b/staging_vespalib/src/tests/clock/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +clock_test +staging_vespalib_clock_test_app diff --git a/staging_vespalib/src/tests/clock/CMakeLists.txt b/staging_vespalib/src/tests/clock/CMakeLists.txt new file mode 100644 index 00000000000..8717a6411ab --- /dev/null +++ b/staging_vespalib/src/tests/clock/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_clock_test_app + SOURCES + clock_test.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_clock_test_app COMMAND staging_vespalib_clock_test_app) diff --git a/staging_vespalib/src/tests/clock/DESC b/staging_vespalib/src/tests/clock/DESC new file mode 100644 index 00000000000..061f9af2d0a --- /dev/null +++ b/staging_vespalib/src/tests/clock/DESC @@ -0,0 +1 @@ +Unit tests for the clock. diff --git a/staging_vespalib/src/tests/clock/FILES b/staging_vespalib/src/tests/clock/FILES new file mode 100644 index 00000000000..c0a2e8e47ea --- /dev/null +++ b/staging_vespalib/src/tests/clock/FILES @@ -0,0 +1 @@ +clock.cpp diff --git a/staging_vespalib/src/tests/clock/clock_test.cpp b/staging_vespalib/src/tests/clock/clock_test.cpp new file mode 100644 index 00000000000..f7f15dd3d24 --- /dev/null +++ b/staging_vespalib/src/tests/clock/clock_test.cpp @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("sharedptr_test"); +#include <vespa/vespalib/util/clock.h> +#include <vespa/vespalib/testkit/testapp.h> + +using vespalib::Clock; +using fastos::TimeStamp; + +class Test : public vespalib::TestApp +{ +public: + int Main(); +}; + + +int +Test::Main() +{ + TEST_INIT("clock_test"); + + Clock clock(0.050); + FastOS_ThreadPool pool(0x10000); + ASSERT_TRUE(pool.NewThread(&clock, NULL) != NULL); + uint64_t start = clock.getTimeNS(); + FastOS_Thread::Sleep(5000); + uint64_t stop = clock.getTimeNS(); + EXPECT_TRUE(stop > start); + FastOS_Thread::Sleep(6000); + clock.stop(); + uint64_t stop2 = clock.getTimeNS(); + EXPECT_TRUE(stop2 > stop); + EXPECT_TRUE((stop2 - stop)/TimeStamp::MICRO > 1000); + TEST_DONE(); +} + +TEST_APPHOOK(Test) diff --git a/staging_vespalib/src/tests/cpu/.gitignore b/staging_vespalib/src/tests/cpu/.gitignore new file mode 100644 index 00000000000..eea6ced9753 --- /dev/null +++ b/staging_vespalib/src/tests/cpu/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +cpu_test +staging_vespalib_cpu_test_app diff --git a/staging_vespalib/src/tests/cpu/CMakeLists.txt b/staging_vespalib/src/tests/cpu/CMakeLists.txt new file mode 100644 index 00000000000..13bc4014d85 --- /dev/null +++ b/staging_vespalib/src/tests/cpu/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_cpu_test_app + SOURCES + cpu_test.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_cpu_test_app COMMAND staging_vespalib_cpu_test_app) diff --git a/staging_vespalib/src/tests/cpu/DESC b/staging_vespalib/src/tests/cpu/DESC new file mode 100644 index 00000000000..19de1b44450 --- /dev/null +++ b/staging_vespalib/src/tests/cpu/DESC @@ -0,0 +1 @@ +cpu special instructions test. Take a look at cpu.cpp for details. diff --git a/staging_vespalib/src/tests/cpu/FILES b/staging_vespalib/src/tests/cpu/FILES new file mode 100644 index 00000000000..6abb996b510 --- /dev/null +++ b/staging_vespalib/src/tests/cpu/FILES @@ -0,0 +1 @@ +cpu.cpp diff --git a/staging_vespalib/src/tests/cpu/cpu_test.cpp b/staging_vespalib/src/tests/cpu/cpu_test.cpp new file mode 100644 index 00000000000..56f109e47a4 --- /dev/null +++ b/staging_vespalib/src/tests/cpu/cpu_test.cpp @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("cpu_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/cpu.h> + +using namespace vespalib; + +class Test : public vespalib::TestApp +{ +public: + int Main(); +}; + +int Test::Main() +{ + TEST_INIT("cpu_test"); + + const X86CpuInfo & cpu = X86CpuInfo::cpuInfo(); + EXPECT_TRUE(cpu.hasMMX()); + EXPECT_TRUE(cpu.hasSSE()); + EXPECT_TRUE(cpu.hasSSE2()); + EXPECT_TRUE(cpu.hasSSE3()); + EXPECT_TRUE(cpu.hasCX16()); + + X86CpuInfo::print(stdout); + + TEST_FLUSH(); + TEST_DONE(); +} + +TEST_APPHOOK(Test) diff --git a/staging_vespalib/src/tests/crc/.gitignore b/staging_vespalib/src/tests/crc/.gitignore new file mode 100644 index 00000000000..448fa141b03 --- /dev/null +++ b/staging_vespalib/src/tests/crc/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +crc_test +staging_vespalib_crc_test_app diff --git a/staging_vespalib/src/tests/crc/CMakeLists.txt b/staging_vespalib/src/tests/crc/CMakeLists.txt new file mode 100644 index 00000000000..cc8e828d0d4 --- /dev/null +++ b/staging_vespalib/src/tests/crc/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_crc_test_app + SOURCES + crc_test.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_crc_test_app COMMAND staging_vespalib_crc_test_app boost) diff --git a/staging_vespalib/src/tests/crc/DESC b/staging_vespalib/src/tests/crc/DESC new file mode 100644 index 00000000000..587969f7773 --- /dev/null +++ b/staging_vespalib/src/tests/crc/DESC @@ -0,0 +1 @@ +crc test. Take a look at crc.cpp for details. diff --git a/staging_vespalib/src/tests/crc/FILES b/staging_vespalib/src/tests/crc/FILES new file mode 100644 index 00000000000..2e867a09e10 --- /dev/null +++ b/staging_vespalib/src/tests/crc/FILES @@ -0,0 +1 @@ +crc.cpp diff --git a/staging_vespalib/src/tests/crc/crc_test.cpp b/staging_vespalib/src/tests/crc/crc_test.cpp new file mode 100644 index 00000000000..6c028cc09ce --- /dev/null +++ b/staging_vespalib/src/tests/crc/crc_test.cpp @@ -0,0 +1,80 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("crc_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/crc.h> +#include <boost/crc.hpp> +#include <vector> + +using namespace vespalib; + +class Test : public TestApp +{ +public: + int Main(); + void testCorrectNess(); + void testBenchmark(bool our, size_t bufSz, size_t numRep); +}; + +int +Test::Main() +{ + TEST_INIT("crc_test"); + testCorrectNess(); + if (_argc >= 2) { + testBenchmark(false, 1024, 1000*1000); + } else { + testBenchmark(true, 1024, 1000*1000); + } + TEST_DONE(); +} + +void Test::testCorrectNess() +{ + const char *a[7] = { "", "a", "ab", "abc", "abcd", "abcde", "doc:crawler:http://www.ntnu.no/" }; + for (size_t i(0); i < sizeof(a)/sizeof(a[0]); i++) { + uint32_t vespaCrc32 = crc_32_type::crc(a[i], strlen(a[i])); + boost::crc_32_type calculator; + calculator.process_bytes(a[i], strlen(a[i])); + EXPECT_EQUAL(vespaCrc32, calculator.checksum()); + vespalib::crc_32_type calculator2; + calculator2.process_bytes(a[i], strlen(a[i])); + EXPECT_EQUAL(vespaCrc32, calculator2.checksum()); + EXPECT_EQUAL(calculator.checksum(), calculator2.checksum()); + } + vespalib::crc_32_type calculator2; + boost::crc_32_type calculator; + for (size_t i(0); i < sizeof(a)/sizeof(a[0]); i++) { + calculator.process_bytes(a[i], strlen(a[i])); + calculator2.process_bytes(a[i], strlen(a[i])); + EXPECT_EQUAL(calculator.checksum(), calculator2.checksum()); + } + EXPECT_EQUAL(calculator.checksum(), calculator2.checksum()); +} + +void Test::testBenchmark(bool our, size_t bufSz, size_t numRep) +{ + std::vector<char> a(numRep+bufSz); + for(size_t i(0), m(a.size()); i < m; i++) { + a[i] = i&0xff; + } + uint32_t sum(0); + if (our) { + for (size_t i(0); i < (numRep); i++) { + //sum ^= crc_32_type::crc(&a[i], bufSz); + vespalib::crc_32_type calculator; + calculator.process_bytes(&a[i], bufSz); + sum ^=calculator.checksum(); + } + } else { + for (size_t i(0); i < (numRep); i++) { + boost::crc_32_type calculator; + calculator.process_bytes(&a[i], bufSz); + sum ^=calculator.checksum(); + } + } + printf("sum = %x\n", sum); +} + +TEST_APPHOOK(Test) diff --git a/staging_vespalib/src/tests/create-test.sh b/staging_vespalib/src/tests/create-test.sh new file mode 100755 index 00000000000..56234dbab30 --- /dev/null +++ b/staging_vespalib/src/tests/create-test.sh @@ -0,0 +1,65 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +gen_project_file() { + echo "generating '$1' ..." + echo "APPLICATION ${test}_test" > $1 + echo "OBJS ${test}_test" >> $1 + echo "LIBS vespalib/staging_vespalib" >> $1 + echo "EXTERNALLIBS vespalib vespalog" >> $1 + echo "" >> $1 + echo "CUSTOMMAKE" >> $1 + echo "test: depend ${test}_test" >> $1 + echo -e "\t@./${test}_test" >> $1 +} + +gen_source() { + echo "generating '$1' ..." + echo "#include <vespa/log/log.h>" > $1 + echo "LOG_SETUP(\"${test}_test\");" >> $1 + echo "#include <vespa/fastos/fastos.h>" >> $1 + echo "#include <vespa/vespalib/testkit/testapp.h>" >> $1 + echo "" >> $1 + echo "using namespace vespalib;" >> $1 + echo "" >> $1 + echo "TEST_SETUP(Test);" >> $1 + echo "" >> $1 + echo "int" >> $1 + echo "Test::Main()" >> $1 + echo "{" >> $1 + echo " TEST_INIT(\"${test}_test\");" >> $1 + echo " TEST_DONE();" >> $1 + echo "}" >> $1 +} + +gen_desc() { + echo "generating '$1' ..." + echo "$test test. Take a look at ${test}_test.cpp for details." > $1 +} + +gen_file_list() { + echo "generating '$1' ..." + echo "${test}_test.cpp" > $1 +} + +if [ $# -ne 1 ]; then + echo "usage: $0 <name>" + echo " name: name of the test to create" + exit 1 +fi + +test=$1 +if [ -e $test ]; then + echo "$test already present, don't want to mess it up..." + exit 1 +fi + +echo "creating directory '$test' ..." +mkdir -p $test || exit 1 +cd $test || exit 1 +test=`basename $test` + +gen_project_file fastos.project +gen_source ${test}_test.cpp +gen_desc DESC +gen_file_list FILES diff --git a/staging_vespalib/src/tests/databuffer/.gitignore b/staging_vespalib/src/tests/databuffer/.gitignore new file mode 100644 index 00000000000..e7b0e69c372 --- /dev/null +++ b/staging_vespalib/src/tests/databuffer/.gitignore @@ -0,0 +1 @@ +staging_vespalib_databuffer_test_app diff --git a/staging_vespalib/src/tests/databuffer/CMakeLists.txt b/staging_vespalib/src/tests/databuffer/CMakeLists.txt new file mode 100644 index 00000000000..65eae47a502 --- /dev/null +++ b/staging_vespalib/src/tests/databuffer/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_databuffer_test_app + SOURCES + databuffer_test.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_databuffer_test_app COMMAND staging_vespalib_databuffer_test_app) diff --git a/staging_vespalib/src/tests/databuffer/databuffer_test.cpp b/staging_vespalib/src/tests/databuffer/databuffer_test.cpp new file mode 100644 index 00000000000..3793af27aa7 --- /dev/null +++ b/staging_vespalib/src/tests/databuffer/databuffer_test.cpp @@ -0,0 +1,145 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("databuffer_test"); + +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/data/databuffer.h> +#include <iostream> + +using namespace vespalib; + +class Test : public vespalib::TestApp { +private: + void testBasic(); +public: + int Main() { + TEST_INIT("databuffer_test"); + + testBasic(); TEST_FLUSH(); + + TEST_DONE(); + } +}; + +TEST_APPHOOK(Test); + +void +Test::testBasic() +{ + DataBuffer a(50); + EXPECT_EQUAL(256u, a.getBufSize()); + EXPECT_EQUAL(a.getFreeLen(), a.getBufSize()); + a.ensureFree(1000); + EXPECT_EQUAL(1024u, a.getBufSize()); + EXPECT_EQUAL(a.getFreeLen(), a.getBufSize()); + EXPECT_EQUAL(0u, a.getDeadLen()); + EXPECT_EQUAL(0u, a.getDataLen()); + EXPECT_EQUAL(a.getData(), a.getDead()); + EXPECT_EQUAL(a.getData(), a.getFree()); + EXPECT_EQUAL(a.getBufSize(), a.getFreeLen()); + a.assertValid(); + + a.writeInt16(7); + EXPECT_EQUAL(0u, a.getDeadLen()); + EXPECT_EQUAL(2u, a.getDataLen()); + EXPECT_EQUAL(a.getBufSize()-2, a.getFreeLen()); + EXPECT_EQUAL(a.getData(), a.getDead()); + EXPECT_EQUAL(a.getData()+2, a.getFree()); + a.clear(); + EXPECT_EQUAL(0u, a.getDeadLen()); + EXPECT_EQUAL(0u, a.getDataLen()); + EXPECT_EQUAL(a.getBufSize(), a.getFreeLen()); + + a.writeInt8(0xaau); + EXPECT_EQUAL(1u, a.getDataLen()); + EXPECT_EQUAL(0xaau, a.peekInt8(0)); + EXPECT_EQUAL(1u, a.getDataLen()); + EXPECT_EQUAL(0xaau, a.readInt8()); + EXPECT_EQUAL(0u, a.getDataLen()); + + a.writeInt16(0xaabbu); + EXPECT_EQUAL(2u, a.getDataLen()); + EXPECT_EQUAL(0xaabbu, a.peekInt16(0)); + EXPECT_EQUAL(2u, a.getDataLen()); + EXPECT_EQUAL(0xaabbu, a.readInt16()); + EXPECT_EQUAL(0u, a.getDataLen()); + a.writeInt16(0xaabbu); + EXPECT_EQUAL(2u, a.getDataLen()); + EXPECT_EQUAL(0xbbaau, a.peekInt16Reverse(0)); + EXPECT_EQUAL(2u, a.getDataLen()); + EXPECT_EQUAL(0xbbaau, a.readInt16Reverse()); + EXPECT_EQUAL(0u, a.getDataLen()); + + a.writeInt32(0xaabbccddu); + EXPECT_EQUAL(4u, a.getDataLen()); + EXPECT_EQUAL(0xaabbccddu, a.peekInt32(0)); + EXPECT_EQUAL(4u, a.getDataLen()); + EXPECT_EQUAL(0xaabbccddu, a.readInt32()); + EXPECT_EQUAL(0u, a.getDataLen()); + a.writeInt32(0xaabbccddu); + EXPECT_EQUAL(4u, a.getDataLen()); + EXPECT_EQUAL(0xddccbbaau, a.peekInt32Reverse(0)); + EXPECT_EQUAL(4u, a.getDataLen()); + EXPECT_EQUAL(0xddccbbaau, a.readInt32Reverse()); + EXPECT_EQUAL(0u, a.getDataLen()); + + a.writeInt64(0xaabbccddeeff9988ul); + EXPECT_EQUAL(8u, a.getDataLen()); + EXPECT_EQUAL(0xaabbccddeeff9988ul, a.peekInt64(0)); + EXPECT_EQUAL(8u, a.getDataLen()); + EXPECT_EQUAL(0xaabbccddeeff9988ul, a.readInt64()); + EXPECT_EQUAL(0u, a.getDataLen()); + a.writeInt64(0xaabbccddeeff9988ul); + EXPECT_EQUAL(8u, a.getDataLen()); + EXPECT_EQUAL(0x8899ffeeddccbbaaul, a.peekInt64Reverse(0)); + EXPECT_EQUAL(8u, a.getDataLen()); + EXPECT_EQUAL(0x8899ffeeddccbbaaul, a.readInt64Reverse()); + EXPECT_EQUAL(0u, a.getDataLen()); + + a.writeFloat(8.9f); + EXPECT_EQUAL(4u, a.getDataLen()); + EXPECT_EQUAL(8.9f, a.readFloat()); + EXPECT_EQUAL(0u, a.getDataLen()); + + a.writeDouble(8.9); + EXPECT_EQUAL(8u, a.getDataLen()); + EXPECT_EQUAL(8.9, a.readDouble()); + EXPECT_EQUAL(0u, a.getDataLen()); + + const char *c = "abc"; + char b[3]; + a.writeBytes(c, 3); + EXPECT_EQUAL(3u, a.getDataLen()); + EXPECT_EQUAL(0, memcmp(c, a.getData(), a.getDataLen())); + a.peekBytes(b, 3, 0); + EXPECT_EQUAL(3u, a.getDataLen()); + EXPECT_EQUAL(0, memcmp(c, b, sizeof(b))); + a.readBytes(b, sizeof(b)); + EXPECT_EQUAL(0u, a.getDataLen()); + EXPECT_EQUAL(0, memcmp(c, b, sizeof(b))); + + a.writeInt64(67); + EXPECT_EQUAL(8u, a.getDataLen()); + EXPECT_FALSE(a.shrink(1025)); + EXPECT_FALSE(a.shrink(7)); + EXPECT_TRUE(a.shrink(16)); + EXPECT_EQUAL(8u, a.getDataLen()); + EXPECT_EQUAL(16u, a.getBufSize()); + + a.writeInt64(89); + EXPECT_EQUAL(16u, a.getDataLen()); + EXPECT_EQUAL(16u, a.getBufSize()); + EXPECT_EQUAL(0u, a.getDeadLen()); + EXPECT_EQUAL(67u, a.readInt64()); + EXPECT_EQUAL(8u, a.getDataLen()); + EXPECT_EQUAL(8u, a.getDeadLen()); + EXPECT_EQUAL(16u, a.getBufSize()); + a.pack(16); + EXPECT_EQUAL(8u, a.getDataLen()); + EXPECT_EQUAL(0u, a.getDeadLen()); + EXPECT_EQUAL(256u, a.getBufSize()); + EXPECT_EQUAL(89u, a.readInt64()); + EXPECT_EQUAL(0u, a.getDataLen()); + EXPECT_EQUAL(256u, a.getBufSize()); +} diff --git a/staging_vespalib/src/tests/directio/.gitignore b/staging_vespalib/src/tests/directio/.gitignore new file mode 100644 index 00000000000..6e6dbe13324 --- /dev/null +++ b/staging_vespalib/src/tests/directio/.gitignore @@ -0,0 +1 @@ +staging_vespalib_directio_test_app diff --git a/staging_vespalib/src/tests/directio/CMakeLists.txt b/staging_vespalib/src/tests/directio/CMakeLists.txt new file mode 100644 index 00000000000..334ae825bf6 --- /dev/null +++ b/staging_vespalib/src/tests/directio/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_directio_test_app + SOURCES + directio.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_directio_test_app COMMAND staging_vespalib_directio_test_app) diff --git a/staging_vespalib/src/tests/directio/directio.cpp b/staging_vespalib/src/tests/directio/directio.cpp new file mode 100644 index 00000000000..570d7276bc9 --- /dev/null +++ b/staging_vespalib/src/tests/directio/directio.cpp @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("directio_test"); +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/data/databuffer.h> + +using namespace fastos; +using namespace vespalib; + +TEST("that DirectIOException propagates the correct information.") { + const char *msg("The buffer"); + DirectIOException e("file.a", msg, 10, 3); + EXPECT_EQUAL(10u, e.getLength()); + EXPECT_EQUAL(3u, e.getOffset()); + EXPECT_EQUAL(msg, e.getBuffer()); + EXPECT_EQUAL(0u, string(e.what()).find("DirectIO failed for file 'file.a' buffer=")); + EXPECT_EQUAL(string("file.a"), e.getFileName()); +} + +TEST("that DirectIOException is thrown on unaligned buf.") { + FastOS_File f("staging_vespalib_directio_test_app"); + f.EnableDirectIO(); + EXPECT_TRUE(f.OpenReadOnly()); + DataBuffer buf(10000, 4096); + bool caught(false); + try { + f.ReadBuf(buf.getFree()+1, 4096, 0); + } catch (const DirectIOException & e) { + EXPECT_EQUAL(4096u, e.getLength()); + EXPECT_EQUAL(0u, e.getOffset()); + EXPECT_EQUAL(buf.getFree()+1, e.getBuffer()); + EXPECT_EQUAL(string(f.GetFileName()), e.getFileName()); + caught = true; + } + EXPECT_TRUE(caught); +} + +TEST("that DirectIOException is thrown on unaligned offset.") { + FastOS_File f("staging_vespalib_directio_test_app"); + f.EnableDirectIO(); + EXPECT_TRUE(f.OpenReadOnly()); + DataBuffer buf(10000, 4096); + bool caught(false); + try { + f.ReadBuf(buf.getFree(), 4096, 1); + } catch (const DirectIOException & e) { + EXPECT_EQUAL(4096u, e.getLength()); + EXPECT_EQUAL(1u, e.getOffset()); + EXPECT_EQUAL(buf.getFree(), e.getBuffer()); + EXPECT_EQUAL(string(f.GetFileName()), e.getFileName()); + caught = true; + } + EXPECT_TRUE(caught); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/staging_vespalib/src/tests/dotproduct/.gitignore b/staging_vespalib/src/tests/dotproduct/.gitignore new file mode 100644 index 00000000000..5d9432fbd08 --- /dev/null +++ b/staging_vespalib/src/tests/dotproduct/.gitignore @@ -0,0 +1,2 @@ +dotproductbenchmark +staging_vespalib_dotproductbenchmark_app diff --git a/staging_vespalib/src/tests/dotproduct/CMakeLists.txt b/staging_vespalib/src/tests/dotproduct/CMakeLists.txt new file mode 100644 index 00000000000..30a02632f1c --- /dev/null +++ b/staging_vespalib/src/tests/dotproduct/CMakeLists.txt @@ -0,0 +1,21 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_dotproductbenchmark_app + SOURCES + dotproductbenchmark.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_dotproductbenchmark_app_sparse-ordered COMMAND staging_vespalib_dotproductbenchmark_app 10 10 1000 1000 BENCHMARK) +vespa_add_test(NAME staging_vespalib_dotproductbenchmark_app_sparse-unordered COMMAND staging_vespalib_dotproductbenchmark_app 10 10 1000 1000 BENCHMARK) +vespa_add_test(NAME staging_vespalib_dotproductbenchmark_app_full COMMAND staging_vespalib_dotproductbenchmark_app 10 10 1000 1000 BENCHMARK) + +# benchmark: dotproductbenchmark +# $(HIDE)$(LDL) time $(VALGRIND) ./dotproductbenchmark sparse-ordered 1000 1000 1000 1000 +# $(HIDE)$(LDL) time $(VALGRIND) ./dotproductbenchmark sparse-unordered 1000 1000 1000 1000 +# $(HIDE)$(LDL) time $(VALGRIND) ./dotproductbenchmark full 1000 1000 1000 1000 +# $(HIDE)$(LDL) time $(VALGRIND) ./dotproductbenchmark sparse-ordered 1000 1000 100 1000 +# $(HIDE)$(LDL) time $(VALGRIND) ./dotproductbenchmark sparse-unordered 1000 1000 100 1000 +# $(HIDE)$(LDL) time $(VALGRIND) ./dotproductbenchmark full 1000 1000 100 1000 +# $(HIDE)$(LDL) time $(VALGRIND) ./dotproductbenchmark sparse-ordered 1000 1000 1000 100 +# $(HIDE)$(LDL) time $(VALGRIND) ./dotproductbenchmark sparse-unordered 1000 1000 1000 100 +# $(HIDE)$(LDL) time $(VALGRIND) ./dotproductbenchmark full 1000 1000 1000 100 diff --git a/staging_vespalib/src/tests/dotproduct/dotproductbenchmark.cpp b/staging_vespalib/src/tests/dotproduct/dotproductbenchmark.cpp new file mode 100644 index 00000000000..9ef3d959f3b --- /dev/null +++ b/staging_vespalib/src/tests/dotproduct/dotproductbenchmark.cpp @@ -0,0 +1,179 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/hwaccelrated/iaccelrated.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/stllike/hash_map.h> +#include <iostream> + +using namespace vespalib; +using vespalib::hwaccelrated::IAccelrated; + +class Benchmark { +public: + virtual ~Benchmark() { } + virtual void compute(size_t docId) const = 0; +}; + +void +runBenchmark(size_t count, size_t docs, const Benchmark & benchmark) +{ + for (size_t i(0); i < count; i++) { + for (size_t docId(0); docId < docs; docId++) { + benchmark.compute(docId); + } + } +} + +template <typename T> +class FullBenchmark : public Benchmark +{ +public: + FullBenchmark(size_t numDocs, size_t numValues) : + _values(numDocs*numValues), + _query(numValues), + _dp(IAccelrated::getAccelrator()) + { + for (size_t i(0); i < numDocs; i++) { + for (size_t j(0); j < numValues; j++) { + _values[i*numValues + j] = j; + } + } + for (size_t j(0); j < numValues; j++) { + _query[j] = j; + } + } + virtual void compute(size_t docId) const { + _dp->dotProduct(&_query[0], &_values[docId * _query.size()], _query.size()); + } +private: + std::vector<T> _values; + std::vector<T> _query; + IAccelrated::UP _dp; +}; + +class SparseBenchmark : public Benchmark +{ +public: + SparseBenchmark(size_t numDocs, size_t numValues, size_t numQueryValues) : + _numValues(numValues), + _values(numDocs*numValues) + { + for (size_t i(0); i < numDocs; i++) { + for (size_t j(0); j < numValues; j++) { + size_t k(numValues < numQueryValues ? (j*numQueryValues)/numValues : j); + _values[i*numValues + j] = P(k, k); + } + } + } +protected: + struct P { + P(uint32_t key=0, int32_t value=0) : + _key(key), + _value(value) + { } + uint32_t _key; + int32_t _value; + }; + size_t _numValues; + std::vector<P> _values; +}; + +class UnorderedSparseBenchmark : public SparseBenchmark +{ +private: + typedef hash_map<uint32_t, int32_t> map; +public: + UnorderedSparseBenchmark(size_t numDocs, size_t numValues, size_t numQueryValues) : + SparseBenchmark(numDocs, numValues, numQueryValues) + { + for (size_t j(0); j < numQueryValues; j++) { + _query[j] = j; + } + } +private: + virtual void compute(size_t docId) const { + int64_t sum(0); + size_t offset(docId*_numValues); + const auto e(_query.end()); + for (size_t i(0); i < _numValues; i++) { + auto it = _query.find(_values[offset + i]._key); + if (it != e) { + sum += static_cast<int64_t>(_values[offset + i]._value) * it->second; + } + } + } + map _query; +}; + +class OrderedSparseBenchmark : public SparseBenchmark +{ +private: +public: + OrderedSparseBenchmark(size_t numDocs, size_t numValues, size_t numQueryValues) : + SparseBenchmark(numDocs, numValues, numQueryValues), + _query(numQueryValues) + { + for (size_t j(0); j < numQueryValues; j++) { + size_t k(numValues > numQueryValues ? j*numValues/numQueryValues : j); + _query[j] = P(k, k); + } + } +private: + virtual void compute(size_t docId) const { + int64_t sum(0); + size_t offset(docId*_numValues); + + for (size_t a(0), b(0); a < _query.size() && b < _numValues; b++) { + for (; a < _query.size() && (_query[a]._key <= _values[offset + b]._key); a++); + if (_query[a]._key == _values[offset + b]._key) { + sum += static_cast<int64_t>(_values[offset + b]._value) * _query[a]._value; + } + } + } + std::vector<P> _query; +}; + +int main(int argc, char *argv[]) +{ + size_t numDocs(1); + size_t numValues(1000); + size_t numQueryValues(1000); + size_t numQueries(1000000); + string type("full"); + if ( argc > 1) { + type = argv[1]; + } + if ( argc > 2) { + numQueries = strtoul(argv[2], NULL, 0); + } + if ( argc > 3) { + numDocs = strtoul(argv[3], NULL, 0); + } + if ( argc > 4) { + numValues = strtoul(argv[4], NULL, 0); + } + if ( argc > 5) { + numQueryValues = strtoul(argv[5], NULL, 0); + } + + std::cout << "type = " << type << std::endl; + std::cout << "numQueries = " << numQueries << std::endl; + std::cout << "numDocs = " << numDocs << std::endl; + std::cout << "numValues = " << numValues << std::endl; + std::cout << "numQueryValues = " << numQueryValues << std::endl; + if (type == "full") { + FullBenchmark<int32_t> bm(numDocs, numValues); + runBenchmark(numQueries, numDocs, bm); + } else if (type == "sparse-ordered") { + OrderedSparseBenchmark bm(numDocs, numValues, numQueryValues); + runBenchmark(numQueries, numDocs, bm); + } else if (type == "sparse-unordered") { + UnorderedSparseBenchmark bm(numDocs, numValues, numQueryValues); + runBenchmark(numQueries, numDocs, bm); + } else { + std::cerr << "type '" << type << "' is unknown." << std::endl; + } + + return 0; +} + diff --git a/staging_vespalib/src/tests/encoding/.gitignore b/staging_vespalib/src/tests/encoding/.gitignore new file mode 100644 index 00000000000..a3e9c375723 --- /dev/null +++ b/staging_vespalib/src/tests/encoding/.gitignore @@ -0,0 +1,3 @@ +.depend +Makefile +*_test diff --git a/staging_vespalib/src/tests/encoding/base64/.gitignore b/staging_vespalib/src/tests/encoding/base64/.gitignore new file mode 100644 index 00000000000..04f2994a28f --- /dev/null +++ b/staging_vespalib/src/tests/encoding/base64/.gitignore @@ -0,0 +1 @@ +staging_vespalib_base64_test_app diff --git a/staging_vespalib/src/tests/encoding/base64/CMakeLists.txt b/staging_vespalib/src/tests/encoding/base64/CMakeLists.txt new file mode 100644 index 00000000000..bc515aa26d6 --- /dev/null +++ b/staging_vespalib/src/tests/encoding/base64/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_base64_test_app + SOURCES + base64_test.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_base64_test_app COMMAND staging_vespalib_base64_test_app) diff --git a/staging_vespalib/src/tests/encoding/base64/DESC b/staging_vespalib/src/tests/encoding/base64/DESC new file mode 100644 index 00000000000..e9ece74e119 --- /dev/null +++ b/staging_vespalib/src/tests/encoding/base64/DESC @@ -0,0 +1 @@ +base64 test. Take a look at base64.cpp for details. diff --git a/staging_vespalib/src/tests/encoding/base64/FILES b/staging_vespalib/src/tests/encoding/base64/FILES new file mode 100644 index 00000000000..475b11151a3 --- /dev/null +++ b/staging_vespalib/src/tests/encoding/base64/FILES @@ -0,0 +1 @@ +base64.cpp diff --git a/staging_vespalib/src/tests/encoding/base64/base64_test.cpp b/staging_vespalib/src/tests/encoding/base64/base64_test.cpp new file mode 100644 index 00000000000..e765ac7d601 --- /dev/null +++ b/staging_vespalib/src/tests/encoding/base64/base64_test.cpp @@ -0,0 +1,89 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/encoding/base64.h> + +#include <vespa/log/log.h> +#include <iostream> +#include <vector> +#include <vespa/vespalib/testkit/testapp.h> + +LOG_SETUP("base64_test"); + +using namespace vespalib; + +TEST_SETUP(Test); + +int +Test::Main() +{ + TEST_INIT("base64_test"); + + // Basic test without padding + std::string source = "No need to pad this string."; + std::string encoded = Base64::encode(source); + std::string expected = "Tm8gbmVlZCB0byBwYWQgdGhpcyBzdHJpbmcu"; + std::string decoded = Base64::decode(encoded); + + EXPECT_EQUAL(expected, encoded); + EXPECT_EQUAL(source, decoded); + + EXPECT_TRUE(static_cast<uint32_t>( + Base64::getMaximumEncodeLength(source.size())) >= encoded.size()); + EXPECT_TRUE(static_cast<uint32_t>( + Base64::getMaximumDecodeLength(encoded.size())) >= source.size()); + + // Basic string that needs padding + source = "This string will need to be padded."; + encoded = Base64::encode(source); + expected = "VGhpcyBzdHJpbmcgd2lsbCBuZWVkIHRvIGJlIHBhZGRlZC4="; + decoded = Base64::decode(encoded); + + EXPECT_EQUAL(expected, encoded); + EXPECT_EQUAL(source, decoded); + + EXPECT_TRUE(static_cast<uint32_t>( + Base64::getMaximumEncodeLength(source.size())) >= encoded.size()); + EXPECT_TRUE(static_cast<uint32_t>( + Base64::getMaximumDecodeLength(encoded.size())) >= source.size()); + + // Check that max sizes are good for whatever input sizes + source = ""; + for (uint32_t i=0; i<100; ++i) { + source += "a"; + // Code will assert if -1 is returned from either + // getMaximumEncodeLength() or getMaximumDecodeLength(). + encoded = Base64::encode(source); + decoded = Base64::decode(encoded); + EXPECT_EQUAL(source, decoded); + } + + // Check that -1 is returned on too little space when encoding + source = "Checking that -1 is returned when not enough space to encode"; + std::vector<char> buffer(100, '\0'); + uint32_t minSizeNeeded = 81; + for (uint32_t i=0; i<minSizeNeeded; ++i) { + EXPECT_EQUAL(-1, Base64::encode(source.c_str(), source.size(), + &buffer[0], i)); + } + EXPECT_EQUAL(80, Base64::encode(source.c_str(), source.size(), + &buffer[0], minSizeNeeded)); + EXPECT_EQUAL(Base64::encode(source), std::string(&buffer[0], 80)); + EXPECT_TRUE(minSizeNeeded <= static_cast<uint32_t>( + Base64::getMaximumEncodeLength(source.size()))); + + EXPECT_TRUE(buffer[80] == '\0'); + + // Check that -1 is returned on too little space when decoding + encoded = Base64::encode(source); + minSizeNeeded = 60; + for (uint32_t i=0; i<minSizeNeeded; ++i) { + EXPECT_EQUAL(-1, Base64::decode(encoded.c_str(), encoded.size(), + &buffer[0], i)); + } + EXPECT_EQUAL(60, Base64::decode(encoded.c_str(), encoded.size(), + &buffer[0], minSizeNeeded)); + EXPECT_EQUAL(source, std::string(&buffer[0], 60)); + + TEST_DONE(); +} diff --git a/staging_vespalib/src/tests/fileheader/.gitignore b/staging_vespalib/src/tests/fileheader/.gitignore new file mode 100644 index 00000000000..909655ba711 --- /dev/null +++ b/staging_vespalib/src/tests/fileheader/.gitignore @@ -0,0 +1,6 @@ +*.So +.depend* +Makefile +fileheader.tmp +fileheader_test +staging_vespalib_fileheader_test_app diff --git a/staging_vespalib/src/tests/fileheader/CMakeLists.txt b/staging_vespalib/src/tests/fileheader/CMakeLists.txt new file mode 100644 index 00000000000..9e8e834147e --- /dev/null +++ b/staging_vespalib/src/tests/fileheader/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_fileheader_test_app + SOURCES + fileheader_test.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_fileheader_test_app COMMAND staging_vespalib_fileheader_test_app) diff --git a/staging_vespalib/src/tests/fileheader/DESC b/staging_vespalib/src/tests/fileheader/DESC new file mode 100644 index 00000000000..feb98035803 --- /dev/null +++ b/staging_vespalib/src/tests/fileheader/DESC @@ -0,0 +1 @@ +fileheader test. Take a look at fileheader.cpp for details. diff --git a/staging_vespalib/src/tests/fileheader/FILES b/staging_vespalib/src/tests/fileheader/FILES new file mode 100644 index 00000000000..f1ade37b054 --- /dev/null +++ b/staging_vespalib/src/tests/fileheader/FILES @@ -0,0 +1 @@ +fileheader.cpp diff --git a/staging_vespalib/src/tests/fileheader/fileheader.dat b/staging_vespalib/src/tests/fileheader/fileheader.dat Binary files differnew file mode 100644 index 00000000000..90660f64b98 --- /dev/null +++ b/staging_vespalib/src/tests/fileheader/fileheader.dat diff --git a/staging_vespalib/src/tests/fileheader/fileheader_test.cpp b/staging_vespalib/src/tests/fileheader/fileheader_test.cpp new file mode 100644 index 00000000000..2aaeeea58a8 --- /dev/null +++ b/staging_vespalib/src/tests/fileheader/fileheader_test.cpp @@ -0,0 +1,694 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("fileheader_test"); + +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/data/fileheader.h> +#include <iostream> + +using namespace vespalib; + +class Test : public vespalib::TestApp { +private: + void testTag(); + void testTagErrors(); + void testTagIteration(); + void testGenericHeader(); + void testBufferReader(); + void testBufferWriter(); + void testBufferAccess(); + void testFileReader(); + void testFileWriter(); + void testFileHeader(); + void testFileAlign(); + void testFileSize(); + void testReadErrors(); + bool testReadError(DataBuffer &buf, const std::string &expected); + void testWriteErrors(); + void testRewriteErrors(); + void testLayout(); + + void + testReadSize(bool mapped); + + void + testReadSizeErrors(bool mapped); + + bool + testReadSizeError(DataBuffer &buf, const std::string &expected, + bool mapped); + +public: + int Main() { + TEST_INIT("fileheader_test"); + + testTag(); TEST_FLUSH(); + testTagErrors(); TEST_FLUSH(); + testTagIteration(); TEST_FLUSH(); + testGenericHeader(); TEST_FLUSH(); + testBufferReader(); TEST_FLUSH(); + testBufferWriter(); TEST_FLUSH(); + testBufferAccess(); TEST_FLUSH(); + testFileReader(); TEST_FLUSH(); + testFileWriter(); TEST_FLUSH(); + testFileHeader(); TEST_FLUSH(); + testFileAlign(); TEST_FLUSH(); + testFileSize(); TEST_FLUSH(); + testReadErrors(); TEST_FLUSH(); + testWriteErrors(); TEST_FLUSH(); + testRewriteErrors(); TEST_FLUSH(); + testLayout(); TEST_FLUSH(); + testReadSize(false); TEST_FLUSH(); + testReadSizeErrors(false); TEST_FLUSH(); + testReadSize(true); TEST_FLUSH(); + testReadSizeErrors(true); TEST_FLUSH(); + + TEST_DONE(); + } +}; + +TEST_APPHOOK(Test); + +void +Test::testTag() +{ + { + std::vector<GenericHeader::Tag> tags; + tags.push_back(GenericHeader::Tag("foo", 6.9)); + tags.push_back(GenericHeader::Tag("foo", 6.9f)); + for (std::vector<GenericHeader::Tag>::iterator it = tags.begin(); + it != tags.end(); ++it) + { + GenericHeader::Tag tag = *it; + for (uint32_t i = 0; i < 2; ++i) { + EXPECT_EQUAL(GenericHeader::Tag::TYPE_FLOAT, tag.getType()); + EXPECT_EQUAL("foo", tag.getName()); + EXPECT_TRUE(tag.asString().empty()); + EXPECT_APPROX(6.9, tag.asFloat(), 1E-6); + EXPECT_EQUAL(0, tag.asInteger()); + + uint32_t len = tag.getSize(); + DataBuffer buf(len); + EXPECT_EQUAL(len, tag.write(buf)); + + GenericHeader::Tag tmp; + EXPECT_EQUAL(len, tmp.read(buf)); + tag = tmp; + } + } + } + { + std::vector<GenericHeader::Tag> tags; + tags.push_back(GenericHeader::Tag("foo", (int8_t)69)); + tags.push_back(GenericHeader::Tag("foo", (uint8_t)69)); + tags.push_back(GenericHeader::Tag("foo", (int16_t)69)); + tags.push_back(GenericHeader::Tag("foo", (uint16_t)69)); + tags.push_back(GenericHeader::Tag("foo", (int32_t)69)); + tags.push_back(GenericHeader::Tag("foo", (uint32_t)69)); + tags.push_back(GenericHeader::Tag("foo", (int64_t)69)); + for (std::vector<GenericHeader::Tag>::iterator it = tags.begin(); + it != tags.end(); ++it) + { + GenericHeader::Tag tag = *it; + for (uint32_t i = 0; i < 2; ++i) { + EXPECT_EQUAL(GenericHeader::Tag::TYPE_INTEGER, tag.getType()); + EXPECT_EQUAL("foo", tag.getName()); + EXPECT_TRUE(tag.asString().empty()); + EXPECT_EQUAL(0.0, tag.asFloat()); + EXPECT_EQUAL(69l, tag.asInteger()); + + uint32_t len = tag.getSize(); + DataBuffer buf(len); + EXPECT_EQUAL(len, tag.write(buf)); + + GenericHeader::Tag tmp; + EXPECT_EQUAL(len, tmp.read(buf)); + tag = tmp; + } + } + } + { + GenericHeader::Tag tag("foo", "bar"); + for (uint32_t i = 0; i < 2; ++i) { + EXPECT_EQUAL(GenericHeader::Tag::TYPE_STRING, tag.getType()); + EXPECT_EQUAL("foo", tag.getName()); + EXPECT_EQUAL("bar", tag.asString()); + EXPECT_EQUAL(0.0, tag.asFloat()); + EXPECT_EQUAL(0, tag.asInteger()); + + uint32_t len = tag.getSize(); + DataBuffer buf(len); + EXPECT_EQUAL(len, tag.write(buf)); + + GenericHeader::Tag tmp; + EXPECT_EQUAL(len, tmp.read(buf)); + tag = tmp; + } + } +} + +void +Test::testTagErrors() +{ + DataBuffer buf(1024); + buf.writeBytes("foo", 3); + buf.writeInt8(0); + buf.writeInt8((uint8_t)GenericHeader::Tag::TYPE_EMPTY); + + GenericHeader::Tag tag("bar", 6.9); + try { + tag.read(buf); + EXPECT_TRUE(false); + } catch (IllegalHeaderException &e) { + EXPECT_EQUAL("Can not deserialize empty tag.", e.getMessage()); + } + EXPECT_EQUAL("bar", tag.getName()); + EXPECT_EQUAL(GenericHeader::Tag::TYPE_FLOAT, tag.getType()); + EXPECT_EQUAL(6.9, tag.asFloat()); +} + +void +Test::testTagIteration() +{ + GenericHeader header; + header.putTag(GenericHeader::Tag("foo", 6.9)); + header.putTag(GenericHeader::Tag("bar", 6699)); + header.putTag(GenericHeader::Tag("baz", "666999")); + + EXPECT_EQUAL(3u, header.getNumTags()); + EXPECT_EQUAL("bar", header.getTag(0).getName()); + EXPECT_EQUAL("baz", header.getTag(1).getName()); + EXPECT_EQUAL("foo", header.getTag(2).getName()); +} + +void +Test::testGenericHeader() +{ + GenericHeader header; + EXPECT_TRUE(header.isEmpty()); + EXPECT_EQUAL(0u, header.getNumTags()); + EXPECT_TRUE(!header.hasTag("foo")); + EXPECT_TRUE(header.getTag("foo").isEmpty()); + EXPECT_TRUE(!header.hasTag("bar")); + EXPECT_TRUE(header.getTag("bar").isEmpty()); + EXPECT_TRUE(!header.hasTag("baz")); + EXPECT_TRUE(header.getTag("baz").isEmpty()); + + header.putTag(GenericHeader::Tag("foo", 6.9)); + EXPECT_TRUE(!header.isEmpty()); + EXPECT_EQUAL(1u, header.getNumTags()); + EXPECT_TRUE(header.hasTag("foo")); + EXPECT_EQUAL(6.9, header.getTag("foo").asFloat()); + EXPECT_TRUE(!header.hasTag("bar")); + EXPECT_TRUE(header.getTag("bar").isEmpty()); + EXPECT_TRUE(!header.hasTag("baz")); + EXPECT_TRUE(header.getTag("baz").isEmpty()); + + header.putTag(GenericHeader::Tag("bar", 6699)); + EXPECT_TRUE(!header.isEmpty()); + EXPECT_EQUAL(2u, header.getNumTags()); + EXPECT_TRUE(header.hasTag("foo")); + EXPECT_EQUAL(6.9, header.getTag("foo").asFloat()); + EXPECT_TRUE(header.hasTag("bar")); + EXPECT_EQUAL(6699, header.getTag("bar").asInteger()); + EXPECT_TRUE(!header.hasTag("baz")); + EXPECT_TRUE(header.getTag("baz").isEmpty()); + + header.putTag(GenericHeader::Tag("baz", "666999")); + EXPECT_TRUE(header.hasTag("foo")); + EXPECT_EQUAL(6.9, header.getTag("foo").asFloat()); + EXPECT_TRUE(header.hasTag("bar")); + EXPECT_EQUAL(6699, header.getTag("bar").asInteger()); + EXPECT_TRUE(header.hasTag("baz")); + EXPECT_EQUAL("666999", header.getTag("baz").asString()); + + header.removeTag("bar"); + EXPECT_TRUE(header.hasTag("foo")); + EXPECT_EQUAL(6.9, header.getTag("foo").asFloat()); + EXPECT_TRUE(!header.hasTag("bar")); + EXPECT_TRUE(header.getTag("bar").isEmpty()); + EXPECT_TRUE(header.hasTag("baz")); + EXPECT_EQUAL("666999", header.getTag("baz").asString()); + + header.removeTag("foo"); + EXPECT_TRUE(!header.hasTag("foo")); + EXPECT_TRUE(header.getTag("foo").isEmpty()); + EXPECT_TRUE(!header.hasTag("bar")); + EXPECT_TRUE(header.getTag("bar").isEmpty()); + EXPECT_TRUE(header.hasTag("baz")); + EXPECT_EQUAL("666999", header.getTag("baz").asString()); + + header.removeTag("baz"); + EXPECT_TRUE(!header.hasTag("foo")); + EXPECT_TRUE(header.getTag("foo").isEmpty()); + EXPECT_TRUE(!header.hasTag("bar")); + EXPECT_TRUE(header.getTag("bar").isEmpty()); + EXPECT_TRUE(!header.hasTag("baz")); + EXPECT_TRUE(header.getTag("baz").isEmpty()); +} + +void +Test::testBufferReader() +{ + DataBuffer src(256); + for (uint32_t i = 0; i < 256; ++i) { + src.writeInt8((uint8_t)i); + } + + GenericHeader::BufferReader reader(src); + + char dst[7]; + uint32_t sum = 0; + while (sum < 256) { + uint32_t len = (uint32_t)reader.getData(dst, 7); + for (uint32_t i = 0; i < len; ++i) { + EXPECT_EQUAL(sum + i, (uint8_t)dst[i]); + } + sum += len; + } + EXPECT_EQUAL(256u, sum); +} + +void +Test::testBufferWriter() +{ + DataBuffer dst(256); + GenericHeader::BufferWriter writer(dst); + + uint32_t sum = 0; + while(sum < 256) { + char src[7]; + for (uint32_t i = 0; i < 7; ++i) { + src[i] = (uint8_t)(sum + i); + } + uint32_t len = std::min(7u, 256 - sum); + EXPECT_EQUAL(len, (uint32_t)writer.putData(src, len)); + sum += len; + } + EXPECT_EQUAL(256u, sum); + + // flip dst + for (uint32_t i = 0; i < 256; ++i) { + uint8_t b = dst.readInt8(); + EXPECT_EQUAL(i, (uint32_t)b); + } +} + +void +Test::testBufferAccess() +{ + DataBuffer buf; + uint32_t len = 0; + { + GenericHeader header; + header.putTag(GenericHeader::Tag("foo", 6.9)); + header.putTag(GenericHeader::Tag("bar", 6699)); + header.putTag(GenericHeader::Tag("baz", "666999")); + + int64_t bval = 0x1234567890abcdefLL; + header.putTag(GenericHeader::Tag("big", bval)); + + len = header.getSize(); + buf.ensureFree(len); + GenericHeader::BufferWriter writer(buf); + EXPECT_EQUAL(len, header.write(writer)); + } + { + GenericHeader header; + GenericHeader::BufferReader reader(buf); + EXPECT_EQUAL(len, header.read(reader)); + + EXPECT_TRUE(header.hasTag("foo")); + EXPECT_EQUAL(6.9, header.getTag("foo").asFloat()); + EXPECT_TRUE(header.hasTag("bar")); + EXPECT_EQUAL(6699, header.getTag("bar").asInteger()); + EXPECT_TRUE(header.hasTag("baz")); + EXPECT_EQUAL("666999", header.getTag("baz").asString()); + EXPECT_TRUE(header.hasTag("big")); + EXPECT_EQUAL(0x1234567890abcdefLL, header.getTag("big").asInteger()); + } +} + +void +Test::testFileReader() +{ + { + FastOS_File file; + ASSERT_TRUE(file.OpenWriteOnlyTruncate("fileheader.tmp")); + + uint8_t buf[256]; + for (uint32_t i = 0; i < 256; ++i) { + buf[i] = (uint8_t)i; + } + EXPECT_EQUAL(256, file.Write2(buf, 256)); + + file.Close(); + } + { + FastOS_File file; + ASSERT_TRUE(file.OpenReadOnly("fileheader.tmp")); + FileHeader::FileReader reader(file); + + char buf[7]; + uint32_t sum = 0; + while(sum < 256) { + uint32_t len = (uint32_t)reader.getData(buf, 7); + for (uint32_t i = 0; i < len; ++i) { + EXPECT_EQUAL(sum + i, (uint8_t)buf[i]); + } + sum += len; + } + EXPECT_EQUAL(256u, sum); + + file.Close(); + file.Delete(); + } +} + +void +Test::testFileWriter() +{ + { + FastOS_File file; + ASSERT_TRUE(file.OpenWriteOnlyTruncate("fileheader.tmp")); + FileHeader::FileWriter writer(file); + + uint32_t sum = 0; + while(sum < 256) { + char src[7]; + for (uint32_t i = 0; i < 7; ++i) { + src[i] = (uint8_t)(sum + i); + } + uint32_t len = std::min(7u, 256 - sum); + EXPECT_EQUAL(len, (uint32_t)writer.putData(src, len)); + sum += len; + } + EXPECT_EQUAL(256u, sum); + } + { + FastOS_File file; + ASSERT_TRUE(file.OpenReadOnly("fileheader.tmp")); + + uint8_t buf[256]; + EXPECT_EQUAL(256, file.Read(buf, 256)); + for (uint32_t i = 0; i < 256; ++i) { + EXPECT_EQUAL(i, (uint32_t)buf[i]); + } + + file.Close(); + file.Delete(); + } +} + +void +Test::testFileHeader() +{ + uint32_t len = 0; + { + FileHeader header; + header.putTag(FileHeader::Tag("foo", 6.9)); + header.putTag(FileHeader::Tag("bar", 6699)); + header.putTag(FileHeader::Tag("baz", "666999")); + + FastOS_File file; + ASSERT_TRUE(file.OpenWriteOnlyTruncate("fileheader.tmp")); + len = header.writeFile(file); + EXPECT_EQUAL(len, header.getSize()); + file.Close(); + } + { + FastOS_File file; + ASSERT_TRUE(file.OpenReadWrite("fileheader.tmp")); + + FileHeader header; + EXPECT_EQUAL(len, header.readFile(file)); + EXPECT_EQUAL(len, header.getSize()); + + EXPECT_TRUE(header.hasTag("foo")); + EXPECT_EQUAL(6.9, header.getTag("foo").asFloat()); + EXPECT_TRUE(header.hasTag("bar")); + EXPECT_EQUAL(6699, header.getTag("bar").asInteger()); + EXPECT_TRUE(header.hasTag("baz")); + EXPECT_EQUAL("666999", header.getTag("baz").asString()); + + header.putTag(FileHeader::Tag("foo", 9.6)); + header.putTag(FileHeader::Tag("bar", 9966)); + header.putTag(FileHeader::Tag("baz", "999666")); + EXPECT_EQUAL(len, header.getSize()); + EXPECT_EQUAL(len, header.rewriteFile(file)); + + file.Close(); + } + { + FileHeader header; + + FastOS_File file; + ASSERT_TRUE(file.OpenReadOnly("fileheader.tmp")); + EXPECT_EQUAL(len, header.readFile(file)); + EXPECT_EQUAL(len, header.getSize()); + file.Close(); + file.Delete(); + + EXPECT_TRUE(header.hasTag("foo")); + EXPECT_EQUAL(9.6, header.getTag("foo").asFloat()); + EXPECT_TRUE(header.hasTag("bar")); + EXPECT_EQUAL(9966, header.getTag("bar").asInteger()); + EXPECT_TRUE(header.hasTag("baz")); + EXPECT_EQUAL("999666", header.getTag("baz").asString()); + } +} + +void +Test::testFileAlign() +{ + for (uint32_t alignTo = 1; alignTo < 16; ++alignTo) { + FileHeader header(alignTo); + header.putTag(FileHeader::Tag("foo", "bar")); + EXPECT_EQUAL(0u, header.getSize() % alignTo); + } +} + +void +Test::testFileSize() +{ + for (uint32_t minSize = 0; minSize < 512; ++minSize) { + FileHeader header(1u, minSize); + header.putTag(FileHeader::Tag("foo", "bar")); + EXPECT_TRUE(header.getSize() >= minSize); + } +} + +void +Test::testReadErrors() +{ + { + DataBuffer buf; + EXPECT_TRUE(testReadError(buf, "Failed to read header info.")); + } + { + DataBuffer buf; + buf.writeInt32(0xDEADBEAF); + buf.writeInt32(8); + EXPECT_TRUE(testReadError(buf, "Failed to verify magic bits.")); + } + { + DataBuffer buf; + buf.writeInt32(GenericHeader::MAGIC); + buf.writeInt32(8); + EXPECT_TRUE(testReadError(buf, "Failed to verify header size.")); + } + { + DataBuffer buf; + buf.writeInt32(GenericHeader::MAGIC); + buf.writeInt32(16); + buf.writeInt32(-1); + buf.writeInt32(0); + EXPECT_TRUE(testReadError(buf, "Failed to verify header version.")); + } + { + DataBuffer buf; + buf.writeInt32(GenericHeader::MAGIC); + buf.writeInt32(21); + buf.writeInt32(GenericHeader::VERSION); + buf.writeInt32(1); + buf.writeBytes("foo", 3); + buf.writeInt8(0); + buf.writeInt8((uint8_t)GenericHeader::Tag::TYPE_EMPTY); + EXPECT_TRUE(testReadError(buf, "Can not deserialize empty tag.")); + } +} + +bool +Test::testReadError(DataBuffer &buf, const std::string &expected) +{ + GenericHeader header; + header.putTag(GenericHeader::Tag("foo", "bar")); + try { + GenericHeader::BufferReader reader(buf); + header.read(reader); + EXPECT_TRUE(false); + return false; + } catch (IllegalHeaderException &e) { + if (!EXPECT_EQUAL(expected, e.getMessage())) { + return false; + } + } + if (!EXPECT_EQUAL(1u, header.getNumTags())) { + return false; + } + if (!EXPECT_EQUAL("bar", header.getTag("foo").asString())) { + return false; + } + return true; +} + +void +Test::testWriteErrors() +{ + GenericHeader header; + header.putTag(GenericHeader::Tag("foo", 69)); + + DataBuffer buf; + buf.ensureFree(4); + buf.moveFreeToData(buf.getFreeLen() - 4); + EXPECT_TRUE(header.getSize() > buf.getFreeLen()); + try { + GenericHeader::BufferWriter writer(buf); + header.write(writer); + EXPECT_TRUE(false); + } catch (IllegalHeaderException &e) { + EXPECT_EQUAL("Failed to write header.", e.getMessage()); + } + + EXPECT_TRUE(header.hasTag("foo")); + EXPECT_EQUAL(69, header.getTag("foo").asInteger()); +} + +void +Test::testRewriteErrors() +{ + FileHeader header; + header.putTag(FileHeader::Tag("foo", "bar")); + uint32_t len = header.getSize(); + + FastOS_File file; + ASSERT_TRUE(file.OpenWriteOnlyTruncate("fileheader.tmp")); + EXPECT_EQUAL(len, header.writeFile(file)); + file.Close(); + + ASSERT_TRUE(file.OpenReadWrite("fileheader.tmp")); + header.putTag(FileHeader::Tag("baz", "cox")); + EXPECT_TRUE(len != header.getSize()); + try { + header.rewriteFile(file); + EXPECT_TRUE(false); + } catch (IllegalHeaderException &e) { + EXPECT_EQUAL("Failed to rewrite resized header.", e.getMessage()); + } + file.Close(); +} + +void +Test::testLayout() +{ + FastOS_File file; + ASSERT_TRUE(file.OpenReadOnly("fileheader.dat")); + + FileHeader header; + uint32_t len = header.readFile(file); + EXPECT_EQUAL(len, header.getSize()); + file.Close(); + + EXPECT_TRUE(header.hasTag("foo")); + EXPECT_EQUAL(6.9, header.getTag("foo").asFloat()); + EXPECT_TRUE(header.hasTag("bar")); + EXPECT_EQUAL(6699, header.getTag("bar").asInteger()); + EXPECT_TRUE(header.hasTag("baz")); + EXPECT_EQUAL("666999", header.getTag("baz").asString()); +} + + +void +Test::testReadSize(bool mapped) +{ + DataBuffer buf; + buf.writeInt32(GenericHeader::MAGIC); + buf.writeInt32(21); + buf.writeInt32(GenericHeader::VERSION); + buf.writeInt32(1); + uint32_t headerLen = 0u; + if (mapped) { + GenericHeader::MMapReader reader(buf.getData(), buf.getDataLen()); + headerLen = FileHeader::readSize(reader); + } else { + GenericHeader::BufferReader reader(buf); + headerLen = FileHeader::readSize(reader); + } + EXPECT_EQUAL(21u, headerLen); +} + + +void +Test::testReadSizeErrors(bool mapped) +{ + { + DataBuffer buf; + EXPECT_TRUE(testReadSizeError(buf, "Failed to read header info.", + mapped)); + } + { + DataBuffer buf; + buf.writeInt32(0xDEADBEAF); + buf.writeInt32(8); + buf.writeInt32(0); + buf.writeInt32(0); + EXPECT_TRUE(testReadSizeError(buf, "Failed to verify magic bits.", + mapped)); + } + { + DataBuffer buf; + buf.writeInt32(GenericHeader::MAGIC); + buf.writeInt32(8); + buf.writeInt32(GenericHeader::VERSION); + buf.writeInt32(0); + EXPECT_TRUE(testReadSizeError(buf, "Failed to verify header size.", + mapped)); + } + { + DataBuffer buf; + buf.writeInt32(GenericHeader::MAGIC); + buf.writeInt32(16); + buf.writeInt32(-1); + buf.writeInt32(0); + EXPECT_TRUE(testReadSizeError(buf, + "Failed to verify header version.", + mapped)); + } +} + + +bool +Test::testReadSizeError(DataBuffer &buf, const std::string &expected, + bool mapped) +{ + uint32_t headerLen = 0u; + try { + if (mapped) { + GenericHeader::MMapReader reader(buf.getData(), buf.getDataLen()); + headerLen = FileHeader::readSize(reader); + } else { + GenericHeader::BufferReader reader(buf); + headerLen = FileHeader::readSize(reader); + } + EXPECT_TRUE(false); + return false; + } catch (IllegalHeaderException &e) { + if (!EXPECT_EQUAL(expected, e.getMessage())) { + return false; + } + } + EXPECT_EQUAL(headerLen, 0u); + return true; +} + diff --git a/staging_vespalib/src/tests/floatingpointtype/.gitignore b/staging_vespalib/src/tests/floatingpointtype/.gitignore new file mode 100644 index 00000000000..f99dd0fd707 --- /dev/null +++ b/staging_vespalib/src/tests/floatingpointtype/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +floatingpointtype_test +staging_vespalib_floatingpointtype_test_app diff --git a/staging_vespalib/src/tests/floatingpointtype/CMakeLists.txt b/staging_vespalib/src/tests/floatingpointtype/CMakeLists.txt new file mode 100644 index 00000000000..88c73073ed4 --- /dev/null +++ b/staging_vespalib/src/tests/floatingpointtype/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_floatingpointtype_test_app + SOURCES + floatingpointtypetest.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_floatingpointtype_test_app COMMAND staging_vespalib_floatingpointtype_test_app) diff --git a/staging_vespalib/src/tests/floatingpointtype/DESC b/staging_vespalib/src/tests/floatingpointtype/DESC new file mode 100644 index 00000000000..4bdf790ff44 --- /dev/null +++ b/staging_vespalib/src/tests/floatingpointtype/DESC @@ -0,0 +1 @@ +Floating point wrapper test. diff --git a/staging_vespalib/src/tests/floatingpointtype/FILES b/staging_vespalib/src/tests/floatingpointtype/FILES new file mode 100644 index 00000000000..dd496cb1e12 --- /dev/null +++ b/staging_vespalib/src/tests/floatingpointtype/FILES @@ -0,0 +1 @@ +floatingpointtypetest.cpp diff --git a/staging_vespalib/src/tests/floatingpointtype/floatingpointtypetest.cpp b/staging_vespalib/src/tests/floatingpointtype/floatingpointtypetest.cpp new file mode 100644 index 00000000000..41eed2f7dc9 --- /dev/null +++ b/staging_vespalib/src/tests/floatingpointtype/floatingpointtypetest.cpp @@ -0,0 +1,75 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + // Include first to make sure it includes all it need. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/objects/floatingpointtype.h> + +#include <vespa/vespalib/testkit/testapp.h> + +class Test : public vespalib::TestApp +{ +public: + void testFloatingPoint(); + int Main(); +}; + +void +Test::testFloatingPoint() +{ + vespalib::Double d1(1.0); + vespalib::Double d2(1.000000000000001); + vespalib::Double d3(-1.00000000000001); + vespalib::Double d4(4.0); + + EXPECT_TRUE(d1.getValue() != d2.getValue()); + + EXPECT_EQUAL(d1, d2); + EXPECT_EQUAL(d2, d1); + + EXPECT_NOT_EQUAL(d1, d3); + EXPECT_NOT_EQUAL(d1, d4); + + EXPECT_TRUE(d1 - d2 == 0); + EXPECT_TRUE(d2 - d1 == 0); + + EXPECT_TRUE(d1 - 1 == 0); + EXPECT_TRUE(d1 + 1 != 0); + + EXPECT_TRUE(d2 * d4 == 4.0); + EXPECT_TRUE(d2 / d4 == 0.25); + + EXPECT_TRUE(d1 >= 1); + EXPECT_TRUE(d1 <= 1); + EXPECT_TRUE(!(d1 < 1)); + EXPECT_TRUE(!(d1 > 1)); + + EXPECT_EQUAL(d2 * 4, d4); + + EXPECT_EQUAL(++d4, 5.0); + EXPECT_EQUAL(d4++, 5.0); + EXPECT_EQUAL(d4, 6.0); + + d4 /= 3; + EXPECT_EQUAL(d4, 2.00000000001); + d4 *= 2; + EXPECT_EQUAL(d4, 4.000000000001); + + EXPECT_EQUAL(--d4, 3.0); + EXPECT_EQUAL(d4--, 3.0); + EXPECT_EQUAL(d4, 2.0); + d4 /= 0.50000000001; + + EXPECT_EQUAL(d4, 4.0); + + EXPECT_TRUE(!(d3 + 1 > 0)); + EXPECT_TRUE(!(d3 + 1 < 0)); +} + +int +Test::Main() +{ + TEST_INIT("floatingpointtype_test"); + testFloatingPoint(); + TEST_DONE(); +} + +TEST_APPHOOK(Test) diff --git a/staging_vespalib/src/tests/growablebytebuffer/.gitignore b/staging_vespalib/src/tests/growablebytebuffer/.gitignore new file mode 100644 index 00000000000..8df98f682b8 --- /dev/null +++ b/staging_vespalib/src/tests/growablebytebuffer/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +growablebytebuffer_test +staging_vespalib_growablebytebuffer_test_app diff --git a/staging_vespalib/src/tests/growablebytebuffer/CMakeLists.txt b/staging_vespalib/src/tests/growablebytebuffer/CMakeLists.txt new file mode 100644 index 00000000000..cd19f0f1256 --- /dev/null +++ b/staging_vespalib/src/tests/growablebytebuffer/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_growablebytebuffer_test_app + SOURCES + growablebytebuffer_test.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_growablebytebuffer_test_app COMMAND staging_vespalib_growablebytebuffer_test_app) diff --git a/staging_vespalib/src/tests/growablebytebuffer/growablebytebuffer_test.cpp b/staging_vespalib/src/tests/growablebytebuffer/growablebytebuffer_test.cpp new file mode 100644 index 00000000000..121f8d2e011 --- /dev/null +++ b/staging_vespalib/src/tests/growablebytebuffer/growablebytebuffer_test.cpp @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("guard_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/growablebytebuffer.h> + +using namespace vespalib; + +class Test : public TestApp +{ +public: + void testGrowing(); + int Main(); +}; + +void +Test::testGrowing() +{ + GrowableByteBuffer buf(10); + + buf.putInt(3); + buf.putInt(7); + buf.putLong(1234); + buf.putDouble(1234); + buf.putString("hei der"); + + EXPECT_EQUAL(35u, buf.position()); +} + +int +Test::Main() +{ + TEST_INIT("guard_test"); + testGrowing(); + TEST_DONE(); +} + +TEST_APPHOOK(Test) diff --git a/staging_vespalib/src/tests/health_server/.gitignore b/staging_vespalib/src/tests/health_server/.gitignore new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/staging_vespalib/src/tests/health_server/.gitignore diff --git a/staging_vespalib/src/tests/json/.gitignore b/staging_vespalib/src/tests/json/.gitignore new file mode 100644 index 00000000000..963bafed038 --- /dev/null +++ b/staging_vespalib/src/tests/json/.gitignore @@ -0,0 +1 @@ +staging_vespalib_json_test_app diff --git a/staging_vespalib/src/tests/json/CMakeLists.txt b/staging_vespalib/src/tests/json/CMakeLists.txt new file mode 100644 index 00000000000..284e858e033 --- /dev/null +++ b/staging_vespalib/src/tests/json/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_json_test_app + SOURCES + json.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_json_test_app COMMAND staging_vespalib_json_test_app boost) diff --git a/staging_vespalib/src/tests/json/json.cpp b/staging_vespalib/src/tests/json/json.cpp new file mode 100644 index 00000000000..16d27643983 --- /dev/null +++ b/staging_vespalib/src/tests/json/json.cpp @@ -0,0 +1,473 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("json_test"); +#include <vespa/vespalib/testkit/testapp.h> + +#include <vespa/vespalib/util/exception.h> +#include <vespa/vespalib/util/jsonstream.h> +#include <cmath> + +using namespace vespalib; + +class JSONTest : public vespalib::TestApp +{ +private: + void testJSONWriterValues(); + void testJSONWriterObject(); + void testJSONWriterArray(); + void testJSONWriterComplex(); + void testJsonStream(); + void testJsonStreamErrors(); + void testJsonStreamStateReporting(); + +public: + int Main(); +}; + +void +JSONTest::testJSONWriterValues() +{ + JSONStringer js; + + { // bool + js.appendBool(true); + EXPECT_EQUAL(js.toString(), "true"); + js.clear().appendBool(false); + EXPECT_EQUAL(js.toString(), "false"); + } + { // double + js.clear().appendDouble(1234.5678); + EXPECT_EQUAL(js.toString(), "1234.5678"); + js.clear().appendDouble(-1234.5678); + EXPECT_EQUAL(js.toString(), "-1234.5678"); + js.clear().appendDouble(0.0); + EXPECT_EQUAL(js.toString(), "0.0"); + js.clear().appendDouble(0.00000000012345678912356789123456789); + EXPECT_EQUAL(js.toString(), "1.234567891235679e-10"); + js.clear().appendDouble(std::numeric_limits<double>::max()); + EXPECT_EQUAL(js.toString(), "1.797693134862316e+308"); + js.clear().appendDouble(std::numeric_limits<double>::min()); + EXPECT_EQUAL(js.toString(), "2.225073858507201e-308"); + js.clear().appendDouble(1.0 * (uint64_t(1) << 53)); + EXPECT_EQUAL(js.toString(), "9007199254740992.0"); + js.clear().appendDouble(1000); + EXPECT_EQUAL(js.toString(), "1000.0"); + } + { // float + js.clear().appendFloat(1234.5678f); + EXPECT_EQUAL(js.toString(), "1234.5677"); + js.clear().appendFloat(-1234.5678f); + EXPECT_EQUAL(js.toString(), "-1234.5677"); + js.clear().appendFloat(0.0f); + EXPECT_EQUAL(js.toString(), "0.0"); + js.clear().appendFloat(0.00000000012345678912356789123456789f); + EXPECT_EQUAL(js.toString(), "1.2345679e-10"); + js.clear().appendFloat(std::numeric_limits<float>::max()); + EXPECT_EQUAL(js.toString(), "3.4028235e+38"); + js.clear().appendFloat(std::numeric_limits<float>::min()); + EXPECT_EQUAL(js.toString(), "1.1754944e-38"); + js.clear().appendFloat(1.0 * (uint64_t(1) << 24)); + EXPECT_EQUAL(js.toString(), "16777216.0"); + js.clear().appendFloat(1000); + EXPECT_EQUAL(js.toString(), "1000.0"); + } + { // long + js.clear().appendInt64(4294967296ll); + EXPECT_EQUAL(js.toString(), "4294967296"); + js.clear().appendInt64(-4294967296ll); + EXPECT_EQUAL(js.toString(), "-4294967296"); + } + { // string + js.clear().appendString("string"); + EXPECT_EQUAL(js.toString(), "\"string\""); + } + { // NULL + js.clear().appendNull(); + EXPECT_EQUAL(js.toString(), "null"); + } + { // quote + js.clear().appendString("x\"y"); + EXPECT_EQUAL(js.toString(), "\"x\\\"y\""); + js.clear().appendString("x\\y"); + EXPECT_EQUAL(js.toString(), "\"x\\\\y\""); + js.clear().appendString("x/y"); + EXPECT_EQUAL(js.toString(), "\"x/y\""); + js.clear().appendString("x\by"); + EXPECT_EQUAL(js.toString(), "\"x\\by\""); + js.clear().appendString("x\fy"); + EXPECT_EQUAL(js.toString(), "\"x\\fy\""); + js.clear().appendString("x\ny"); + EXPECT_EQUAL(js.toString(), "\"x\\ny\""); + js.clear().appendString("x\ry"); + EXPECT_EQUAL(js.toString(), "\"x\\ry\""); + js.clear().appendString("x\ty"); + EXPECT_EQUAL(js.toString(), "\"x\\ty\""); + } +} + +void +JSONTest::testJSONWriterObject() +{ + JSONStringer js; + + { // single pair + js.beginObject().appendKey("k1").appendInt64(1l).endObject(); + EXPECT_EQUAL(js.toString(), "{\"k1\":1}"); + } + { // multiple pairs + js.clear().beginObject().appendKey("k1").appendInt64(1l).appendKey("k2").appendInt64(2l).endObject(); + EXPECT_EQUAL(js.toString(), "{\"k1\":1,\"k2\":2}"); + } + { // object in object + js.clear().beginObject().appendKey("k1").beginObject().appendKey("k1.1").appendInt64(11l).endObject().endObject(); + EXPECT_EQUAL(js.toString(), "{\"k1\":{\"k1.1\":11}}"); + } + { // object in object (multiple pairs) + js.clear().beginObject(). + appendKey("k1"). + beginObject(). + appendKey("k1.1").appendInt64(11l). + appendKey("k1.2").appendInt64(12l). + endObject(). + appendKey("k2"). + beginObject(). + appendKey("k2.1").appendInt64(21l). + appendKey("k2.2").appendInt64(22l). + endObject(). + endObject(); + EXPECT_EQUAL(js.toString(), "{\"k1\":{\"k1.1\":11,\"k1.2\":12},\"k2\":{\"k2.1\":21,\"k2.2\":22}}"); + } + { // array in object + js.clear().beginObject().appendKey("k1"). + beginArray().appendInt64(1l).appendInt64(2l).endArray().endObject(); + EXPECT_EQUAL(js.toString(), "{\"k1\":[1,2]}"); + } + { // array in object (multiple pairs) + js.clear().beginObject(). + appendKey("k1").beginArray().appendInt64(1l).appendInt64(2l).endArray(). + appendKey("k2").beginArray().appendInt64(3l).appendInt64(4l).endArray(). + endObject(); + EXPECT_EQUAL(js.toString(), "{\"k1\":[1,2],\"k2\":[3,4]}"); + } +} + + +void +JSONTest::testJSONWriterArray() +{ + JSONStringer js; + + { // single element + js.beginArray().appendInt64(1l).endArray(); + EXPECT_EQUAL(js.toString(), "[1]"); + } + { // multiple elements + js.clear().beginArray().appendInt64(1l).appendInt64(2l).endArray(); + EXPECT_EQUAL(js.toString(), "[1,2]"); + } + { // array in array + js.clear().beginArray().beginArray().appendInt64(1l).endArray().endArray(); + EXPECT_EQUAL(js.toString(), "[[1]]"); + } + { // array in array (multiple elements) + js.clear().beginArray(). + beginArray().appendInt64(1l).appendInt64(2l).endArray(). + beginArray().appendInt64(3l).appendInt64(4l).endArray(). + endArray(); + EXPECT_EQUAL(js.toString(), "[[1,2],[3,4]]"); + } + { // object in array + js.clear().beginArray(). + beginObject().appendKey("k1").appendInt64(1l).endObject(). + endArray(); + EXPECT_EQUAL(js.toString(), "[{\"k1\":1}]"); + } + { // object in array (multiple elements) + js.clear().beginArray(). + beginObject().appendKey("k1").appendInt64(1l).appendKey("k2").appendInt64(2l).endObject(). + beginObject().appendKey("k3").appendInt64(3l).appendKey("k4").appendInt64(4l).endObject(). + endArray(); + EXPECT_EQUAL(js.toString(), "[{\"k1\":1,\"k2\":2},{\"k3\":3,\"k4\":4}]"); + } +} + + +void +JSONTest::testJSONWriterComplex() +{ + JSONStringer js; + + js.beginObject(); + { // object + js.appendKey("k1"); + js.beginObject(); + { + js.appendKey("k1.1"); + js.appendInt64(1l); + } + { + js.appendKey("k1.2"); + js.beginArray(); + js.appendInt64(2l); + js.appendInt64(3l); + js.endArray(); + } + js.endObject(); + } + { // object of object + js.appendKey("k2"); + js.beginObject(); + { + js.appendKey("k2.1"); + js.beginObject(); + { + js.appendKey("k2.1.1"); + js.appendInt64(4l); + } + { + js.appendKey("k2.1.2"); + js.beginArray(); + js.appendInt64(5l); + js.appendInt64(6l); + js.endArray(); + } + js.endObject(); + } + js.endObject(); + } + { // array of object + js.appendKey("k3"); + js.beginArray(); + { + js.beginObject(); + { + js.appendKey("k3.1"); + js.appendInt64(7l); + } + { + js.appendKey("k3.2"); + js.beginArray(); + js.appendInt64(8l); + js.appendInt64(9l); + js.endArray(); + } + js.endObject(); + } + { + js.beginObject(); + { + js.appendKey("k3.1"); + js.appendInt64(10l); + } + { + js.appendKey("k3.2"); + js.beginArray(); + js.appendInt64(11l); + js.appendInt64(12l); + js.endArray(); + } + js.endObject(); + } + js.endArray(); + } + js.endObject(); + EXPECT_EQUAL(js.toString(), "{\"k1\":{\"k1.1\":1,\"k1.2\":[2,3]},\"k2\":{\"k2.1\":{\"k2.1.1\":4,\"k2.1.2\":[5,6]}},\"k3\":[{\"k3.1\":7,\"k3.2\":[8,9]},{\"k3.1\":10,\"k3.2\":[11,12]}]}"); +} + +namespace { + struct Builder : public vespalib::JsonStreamTypes { + void build(JsonStream& s) { + s << Object() << "k1" << Object() + << "k1.1" << 1l + << "k1.2" << Array() + << 2l << 3l << End() + << End() + << "k2" << Object() + << "k2.1" << Object() + << "k2.1.1" << 4l + << "k2.1.2" << Array() + << 5l << 6l << End() + << End() + << End() + << "k3" << Array() + << Object() + << "k3.1" << 7l + << "k3.2" << Array() + << 8l << 9l << End() + << End() + << Object() + << "k3.1" << 10l + << "k3.2" << Array() + << 11l << 12l << End() + << End() + << End() + << End(); + } + }; +} + +void +JSONTest::testJsonStream() +{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + Builder b; + b.build(stream); + stream.finalize(); + EXPECT_EQUAL(as.str(), "{\"k1\":{\"k1.1\":1,\"k1.2\":[2,3]},\"k2\":{\"k2.1\":{\"k2.1.1\":4,\"k2.1.2\":[5,6]}},\"k3\":[{\"k3.1\":7,\"k3.2\":[8,9]},{\"k3.1\":10,\"k3.2\":[11,12]}]}"); +} + +void +JSONTest::testJsonStreamErrors() +{ + using namespace vespalib::jsonstream; + // Unsupported object keys + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << Object(); + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: An object value cannot be an object key ({}(ObjectExpectingKey))", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << true; + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: A bool value cannot be an object key ({}(ObjectExpectingKey))", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << 13; + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: An int64_t value cannot be an object key ({}(ObjectExpectingKey))", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << uint64_t(13); + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: A uint64_t value cannot be an object key ({}(ObjectExpectingKey))", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << 0.5; + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: A double value cannot be an object key ({}(ObjectExpectingKey))", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << Array(); + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: An array value cannot be an object key ({}(ObjectExpectingKey))", e.getReason()); + } + // Invalid points to add End() + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << "foo" << End(); + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: Object got key but not value. Cannot end it now ({foo}(ObjectExpectingValue))", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << End(); + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: No tag to end. At root ((RootExpectingArrayOrObjectStart))", e.getReason()); + } + // Adding to finalized stream + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << End() << "foo"; + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: Stream already finalized. Can't add a string value. (Finalized)", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << End() << false; + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: Stream already finalized. Can't add a bool value. (Finalized)", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << End() << 13; + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: Stream already finalized. Can't add an int64_t value. (Finalized)", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << End() << uint64_t(13); + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: Stream already finalized. Can't add a uint64_t value. (Finalized)", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << End() << 0.2; + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: Stream already finalized. Can't add a double value. (Finalized)", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << End() << Object(); + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: Stream already finalized. Can't start a new object. (Finalized)", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << End() << Array(); + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: Stream already finalized. Can't start a new array. (Finalized)", e.getReason()); + } + try{ + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Object() << End() << End(); + } catch (vespalib::JsonStreamException& e) { + EXPECT_EQUAL("Invalid state on call: Stream already finalized. Can't end it. (Finalized)", e.getReason()); + } +} + +void +JSONTest::testJsonStreamStateReporting() +{ + using namespace vespalib::jsonstream; + vespalib::asciistream as; + vespalib::JsonStream stream(as); + stream << Array() << 13 + << "foo" + << Object() << "key" << "value" << End() + << false + << End(); + EXPECT_EQUAL("Current: Finalized", stream.getJsonStreamState()); +} + +int +JSONTest::Main() +{ + TEST_INIT("json_test"); + + testJSONWriterValues(); + testJSONWriterObject(); + testJSONWriterArray(); + testJSONWriterComplex(); + testJsonStream(); + testJsonStreamErrors(); + testJsonStreamStateReporting(); + + TEST_DONE(); +} + +TEST_APPHOOK(JSONTest); + diff --git a/staging_vespalib/src/tests/librarypool/.gitignore b/staging_vespalib/src/tests/librarypool/.gitignore new file mode 100644 index 00000000000..1a1aea2fda0 --- /dev/null +++ b/staging_vespalib/src/tests/librarypool/.gitignore @@ -0,0 +1 @@ +staging_vespalib_librarypool_test_app diff --git a/staging_vespalib/src/tests/librarypool/CMakeLists.txt b/staging_vespalib/src/tests/librarypool/CMakeLists.txt new file mode 100644 index 00000000000..0ef8d03ccda --- /dev/null +++ b/staging_vespalib/src/tests/librarypool/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_librarypool_test_app + SOURCES + librarypool_test.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_librarypool_test_app COMMAND staging_vespalib_librarypool_test_app) diff --git a/staging_vespalib/src/tests/librarypool/DESC b/staging_vespalib/src/tests/librarypool/DESC new file mode 100644 index 00000000000..2330c52fae7 --- /dev/null +++ b/staging_vespalib/src/tests/librarypool/DESC @@ -0,0 +1 @@ +Library pool test. Take a look at librarypool_test.cpp for details. diff --git a/staging_vespalib/src/tests/librarypool/FILES b/staging_vespalib/src/tests/librarypool/FILES new file mode 100644 index 00000000000..c8bb90cf093 --- /dev/null +++ b/staging_vespalib/src/tests/librarypool/FILES @@ -0,0 +1 @@ +librarypool_test.cpp diff --git a/staging_vespalib/src/tests/librarypool/librarypool_test.cpp b/staging_vespalib/src/tests/librarypool/librarypool_test.cpp new file mode 100644 index 00000000000..5808d1c91e5 --- /dev/null +++ b/staging_vespalib/src/tests/librarypool/librarypool_test.cpp @@ -0,0 +1,41 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/librarypool.h> +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/log/log.h> +LOG_SETUP("librarypool_test"); + +using namespace vespalib; + +class Test : public TestApp +{ +public: + int Main(); +}; + +int +Test::Main() +{ + TEST_INIT("librarypool_test"); + LibraryPool p; + ASSERT_TRUE(p.get("z") == NULL); + p.loadLibrary("z"); + ASSERT_TRUE(p.get("z") != NULL); + ASSERT_TRUE(p.get("z")->GetSymbol("some_symbol_that_is_not_there") == NULL); + ASSERT_TRUE(p.get("z")->GetSymbol("compress") != NULL); + try { + p.loadLibrary("not_found"); + ASSERT_TRUE(false); + } catch (const IllegalArgumentException & e) { + ASSERT_TRUE(p.get("not_found") == NULL); + } + { + const LibraryPool & c(p); + ASSERT_TRUE(c.get("z") != NULL); + ASSERT_TRUE(c.get("not_found") == NULL); + } + TEST_DONE(); +} + +TEST_APPHOOK(Test) diff --git a/staging_vespalib/src/tests/memorydatastore/.gitignore b/staging_vespalib/src/tests/memorydatastore/.gitignore new file mode 100644 index 00000000000..634cea1ae88 --- /dev/null +++ b/staging_vespalib/src/tests/memorydatastore/.gitignore @@ -0,0 +1 @@ +staging_vespalib_memorydatastore_test_app diff --git a/staging_vespalib/src/tests/memorydatastore/CMakeLists.txt b/staging_vespalib/src/tests/memorydatastore/CMakeLists.txt new file mode 100644 index 00000000000..386cf7d6865 --- /dev/null +++ b/staging_vespalib/src/tests/memorydatastore/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_memorydatastore_test_app + SOURCES + memorydatastore.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_memorydatastore_test_app COMMAND staging_vespalib_memorydatastore_test_app) diff --git a/staging_vespalib/src/tests/memorydatastore/memorydatastore.cpp b/staging_vespalib/src/tests/memorydatastore/memorydatastore.cpp new file mode 100644 index 00000000000..2bb5a272783 --- /dev/null +++ b/staging_vespalib/src/tests/memorydatastore/memorydatastore.cpp @@ -0,0 +1,75 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("data_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/data/memorydatastore.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vector> + +using namespace vespalib; + +class MemoryDataStoreTest : public vespalib::TestApp +{ +private: + void testMemoryDataStore(); + void testVariableSizeVector(); +public: + int Main(); +}; + +void +MemoryDataStoreTest::testMemoryDataStore() +{ + MemoryDataStore s(256); + std::vector<MemoryDataStore::Reference> v; + v.push_back(s.push_back("mumbo", 5)); + for (size_t i(0); i < 50; i++) { + v.push_back(s.push_back("mumbo", 5)); + EXPECT_EQUAL(static_cast<const char *>(v[i].data()) + 5, v[i+1].data()); + } + v.push_back(s.push_back("mumbo", 5)); + EXPECT_EQUAL(52ul, v.size()); + EXPECT_NOT_EQUAL(static_cast<const char *>(v[50].data()) + 5, v[51].data()); + for (size_t i(0); i < v.size(); i++) { + EXPECT_EQUAL(0, memcmp("mumbo", v[i].data(), 5)); + } +} + +void +MemoryDataStoreTest::testVariableSizeVector() +{ + VariableSizeVector v(256); + for (size_t i(0); i < 10000; i++) { + asciistream os; + os << i; + v.push_back(os.str().c_str(), os.str().size()); + } + for (size_t i(0); i < v.size(); i++) { + asciistream os; + os << i; + EXPECT_EQUAL(os.str().size(), v[i].size()); + EXPECT_EQUAL(0, memcmp(os.str().c_str(), v[i].data(), os.str().size())); + } + size_t i(0); + for (auto it(v.begin()), mt(v.end()); it != mt; it++, i++) { + asciistream os; + os << i; + EXPECT_EQUAL(os.str().size(), it->size()); + EXPECT_EQUAL(0, memcmp(os.str().c_str(), (*it).data(), os.str().size())); + } + +} + +int +MemoryDataStoreTest::Main() +{ + TEST_INIT("data_test"); + testMemoryDataStore(); + testVariableSizeVector(); + + TEST_DONE(); +} + +TEST_APPHOOK(MemoryDataStoreTest); + diff --git a/staging_vespalib/src/tests/objectdump/.gitignore b/staging_vespalib/src/tests/objectdump/.gitignore new file mode 100644 index 00000000000..d243567aea6 --- /dev/null +++ b/staging_vespalib/src/tests/objectdump/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +objectdump_test +staging_vespalib_objectdump_test_app diff --git a/staging_vespalib/src/tests/objectdump/CMakeLists.txt b/staging_vespalib/src/tests/objectdump/CMakeLists.txt new file mode 100644 index 00000000000..e8b5f76b934 --- /dev/null +++ b/staging_vespalib/src/tests/objectdump/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_objectdump_test_app + SOURCES + objectdump.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_objectdump_test_app COMMAND staging_vespalib_objectdump_test_app) diff --git a/staging_vespalib/src/tests/objectdump/DESC b/staging_vespalib/src/tests/objectdump/DESC new file mode 100644 index 00000000000..747ea8bb015 --- /dev/null +++ b/staging_vespalib/src/tests/objectdump/DESC @@ -0,0 +1 @@ +Test dumping of identifiable objects to human-readable form. diff --git a/staging_vespalib/src/tests/objectdump/FILES b/staging_vespalib/src/tests/objectdump/FILES new file mode 100644 index 00000000000..f44f8ce9c4c --- /dev/null +++ b/staging_vespalib/src/tests/objectdump/FILES @@ -0,0 +1 @@ +objectdump.cpp diff --git a/staging_vespalib/src/tests/objectdump/objectdump.cpp b/staging_vespalib/src/tests/objectdump/objectdump.cpp new file mode 100644 index 00000000000..3427bbbff3b --- /dev/null +++ b/staging_vespalib/src/tests/objectdump/objectdump.cpp @@ -0,0 +1,107 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("objectdump_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/objects/identifiable.h> +#include <vespa/vespalib/objects/visit.h> + +using namespace vespalib; + +#define CID_Base 10000000 +#define CID_Foo 10000001 +#define CID_Bar 10000002 +#define CID_Baz 10000003 + +struct Base : public vespalib::Identifiable +{ + DECLARE_IDENTIFIABLE(Base); + virtual Base *clone() const { return new Base(*this); } +}; +IMPLEMENT_IDENTIFIABLE(Base, vespalib::Identifiable); + +struct Baz : public Base +{ + DECLARE_IDENTIFIABLE(Baz); + virtual Baz *clone() const { return new Baz(*this); } +}; +IMPLEMENT_IDENTIFIABLE(Baz, Base); + +struct Bar : public Base +{ + DECLARE_IDENTIFIABLE(Bar); + bool _bool; + int8_t _int8; + uint8_t _uint8; + int16_t _int16; + uint16_t _uint16; + int32_t _int32; + uint32_t _uint32; + int64_t _int64; + uint64_t _uint64; + float _float; + double _double; + std::string _string; + Bar() : _bool(true), _int8(-1), _uint8(1), _int16(-2), _uint16(2), + _int32(-4), _uint32(4), _int64(-8), _uint64(8), + _float(2.5), _double(2.75), _string("bla bla") {} + + virtual Bar *clone() const { return new Bar(*this); } + + virtual void visitMembers(ObjectVisitor &v) const { + visit(v, "_bool", _bool); + visit(v, "_int8", _int8); + visit(v, "_uint8", _uint8); + visit(v, "_int16", _int16); + visit(v, "_uint16", _uint16); + visit(v, "_int32", _int32); + visit(v, "_uint32", _uint32); + visit(v, "_int64", _int64); + visit(v, "_uint64", _uint64); + visit(v, "_float", _float); + visit(v, "_double", _double); + visit(v, "_string", _string); + visit(v, "info", "a dummy string"); + visit(v, "(const char*)0", (const char*)0); + } +}; +IMPLEMENT_IDENTIFIABLE(Bar, Base); + +struct Foo : public Base +{ + DECLARE_IDENTIFIABLE(Foo); + Bar _objMember; + Baz _objMember2; + Baz *_objPtr; + std::vector<Bar> _list; + std::vector<IdentifiablePtr<Base> > _list2; + + Foo() : _objMember(), _objMember2(), _objPtr(0), _list(), _list2() { + _list.push_back(Bar()); + _list.push_back(Bar()); + _list.push_back(Bar()); + _list2.push_back(Bar()); + _list2.push_back(Baz()); + } + virtual Foo *clone() const { return new Foo(*this); } + + virtual void visitMembers(ObjectVisitor &v) const { + visit(v, "_objMember", _objMember); + visit(v, "_objMember2", _objMember2); + visit(v, "_objPtr", _objPtr); + visit(v, "_list", _list); + visit(v, "_list2", _list2); + } +}; +IMPLEMENT_IDENTIFIABLE(Foo, Base); + +TEST_SETUP(Test); + +int +Test::Main() +{ + TEST_INIT("objectdump_test"); + Foo foo; + fprintf(stderr, "%s", foo.asString().c_str()); + TEST_DONE(); +} diff --git a/staging_vespalib/src/tests/objects/.gitignore b/staging_vespalib/src/tests/objects/.gitignore new file mode 100644 index 00000000000..2b10aff029c --- /dev/null +++ b/staging_vespalib/src/tests/objects/.gitignore @@ -0,0 +1,5 @@ +.depend +Makefile +asciistream_test +identifiable_test +staging_vespalib_identifiable_test_app diff --git a/staging_vespalib/src/tests/objects/CMakeLists.txt b/staging_vespalib/src/tests/objects/CMakeLists.txt new file mode 100644 index 00000000000..1ef21d0aaa5 --- /dev/null +++ b/staging_vespalib/src/tests/objects/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_identifiable_test_app + SOURCES + identifiable_test.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_identifiable_test_app COMMAND staging_vespalib_identifiable_test_app) diff --git a/staging_vespalib/src/tests/objects/DESC b/staging_vespalib/src/tests/objects/DESC new file mode 100644 index 00000000000..fe2ee1fa6f4 --- /dev/null +++ b/staging_vespalib/src/tests/objects/DESC @@ -0,0 +1,2 @@ +identifiable test. Take a look at identifiable.cpp for details. +asciistream test. Take a look at asciistream.cpp for details. diff --git a/staging_vespalib/src/tests/objects/FILES b/staging_vespalib/src/tests/objects/FILES new file mode 100644 index 00000000000..f50b9f9ca5a --- /dev/null +++ b/staging_vespalib/src/tests/objects/FILES @@ -0,0 +1,2 @@ +identifiable.cpp +asciistream.cpp diff --git a/staging_vespalib/src/tests/objects/identifiable_test.cpp b/staging_vespalib/src/tests/objects/identifiable_test.cpp new file mode 100644 index 00000000000..2fe793dae10 --- /dev/null +++ b/staging_vespalib/src/tests/objects/identifiable_test.cpp @@ -0,0 +1,339 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("identifiable_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/objects/identifiable.h> +#include <vespa/vespalib/objects/namedobject.h> +#include <vespa/vespalib/util/exceptions.h> + +using namespace vespalib; + +class IdentifiableTest : public TestApp { + void requireThatIdentifiableCastCanCastPointers(); + void requireThatIdentifiableCastCanCastReferences(); + void testNamedObject(); + void testNboStream(); + template <typename T> + void testStream(const T & a); + void testNboSerializer(); + template <typename T> + void testSerializer(const T & a); +public: + int Main(); +}; + +#define CID_Abstract 0x700000 +#define CID_A 0x700001 +#define CID_B 0x700002 +#define CID_C 0x700003 + +class Abstract : public Identifiable +{ +public: + DECLARE_IDENTIFIABLE_ABSTRACT(Abstract); + virtual ~Abstract() { } + virtual void someAbstractVirtualMethod() = 0; +}; + +class A : public Abstract +{ +public: + DECLARE_IDENTIFIABLE(A); + A() { } + virtual void someAbstractVirtualMethod() { }; +}; + +class B : public A +{ +public: + DECLARE_IDENTIFIABLE(B); + B() { } +}; + +class C : public Identifiable +{ +private: + int _value; + +public: + DECLARE_IDENTIFIABLE(C); + C() : _value(0) {} + C(int value) : _value(value) {} + C *clone() const { return new C(*this); } + virtual int cmp(const Identifiable &rhs) const { + int result(cmpClassId(rhs)); + if (result == 0) { + result = _value - static_cast<const C &>(rhs)._value; + } + return result; + } +}; + +IMPLEMENT_IDENTIFIABLE_ABSTRACT(Abstract, Identifiable); +IMPLEMENT_IDENTIFIABLE(A, Abstract); +IMPLEMENT_IDENTIFIABLE(B, A); +IMPLEMENT_IDENTIFIABLE(C, Identifiable); + +void +IdentifiableTest::testNamedObject() +{ + NamedObject a("first"), b("second");; + nbostream os; + NBOSerializer nos(os); + nos << a << b; + EXPECT_EQUAL(27u,os.size()); + Identifiable::UP o1; + o1 = Identifiable::create(nos); + EXPECT_EQUAL(14u, os.size()); + ASSERT_TRUE(o1->inherits(NamedObject::classId)); + ASSERT_TRUE(o1->getClass().id() == NamedObject::classId); + EXPECT_TRUE(static_cast<const NamedObject &>(*o1).getName() == "first"); + o1 = Identifiable::create(nos); + EXPECT_EQUAL(0u, os.size()); + ASSERT_TRUE(o1->inherits(NamedObject::classId)); + ASSERT_TRUE(o1->getClass().id() == NamedObject::classId); + EXPECT_TRUE(static_cast<const NamedObject &>(*o1).getName() == "second"); +} + +template <typename T> +void IdentifiableTest::testStream(const T & a) +{ + nbostream s; + s << a; + T b; + s >> b; + EXPECT_TRUE(s.empty()); + EXPECT_EQUAL(a, b); + EXPECT_EQUAL(nbostream::ok, s.state()); + EXPECT_TRUE(s.good()); +} + +template <typename T> +void IdentifiableTest::testSerializer(const T & a) +{ + nbostream t; + NBOSerializer s(t); + s << a; + T b; + s >> b; + EXPECT_TRUE(s.getStream().empty()); + EXPECT_EQUAL(a, b); + EXPECT_EQUAL(nbostream::ok, s.getStream().state()); +} + +void IdentifiableTest::testNboSerializer() +{ + testSerializer(true); + testSerializer(false); + testSerializer(static_cast<int8_t>('a')); + testSerializer(static_cast<uint8_t>(156)); + testSerializer(static_cast<int16_t>(156)); + testSerializer(static_cast<int32_t>(156)); + testSerializer(static_cast<int64_t>(156)); + testSerializer(static_cast<uint16_t>(156)); + testSerializer(static_cast<uint32_t>(156)); + testSerializer(static_cast<uint64_t>(156)); + testSerializer(static_cast<float>(156)); + testSerializer(static_cast<double>(156)); + testSerializer(vespalib::string("abcdefgh")); +} + +void IdentifiableTest::testNboStream() +{ + testStream(true); + testStream(false); + testStream('a'); + testStream(static_cast<unsigned char>(156)); + testStream(static_cast<int16_t>(156)); + testStream(static_cast<int32_t>(156)); + testStream(static_cast<int64_t>(156)); + testStream(static_cast<uint16_t>(156)); + testStream(static_cast<uint32_t>(156)); + testStream(static_cast<uint64_t>(156)); + testStream(static_cast<float>(156)); + testStream(static_cast<double>(156)); + testStream(std::string("abcdefgh")); + testStream(vespalib::string("abcdefgh")); + { + nbostream s(4); + EXPECT_EQUAL(4u, s.capacity()); + s << "abcdef"; + EXPECT_EQUAL(nbostream::ok, s.state()); + EXPECT_EQUAL(10u, s.size()); + EXPECT_EQUAL(16u, s.capacity()); + EXPECT_EQUAL(0, strncmp(s.c_str()+4, "abcdef", 6)); + } + { + nbostream s(8); + EXPECT_EQUAL(0u, s.size()); + EXPECT_EQUAL(8u, s.capacity()); + const char * prev = s.c_str(); + s << "ABCD"; + EXPECT_EQUAL(8u, s.size()); + EXPECT_EQUAL(8u, s.capacity()); + EXPECT_EQUAL(prev, s.c_str()); + s << "A long string that will cause resizing"; + EXPECT_EQUAL(50u, s.size()); + EXPECT_EQUAL(64u, s.capacity()); + EXPECT_NOT_EQUAL(prev, s.c_str()); + } + { + nbostream s(8); + EXPECT_EQUAL(0u, s.size()); + EXPECT_EQUAL(8u, s.capacity()); + const char * prev = s.c_str(); + s << "ABCD"; + EXPECT_EQUAL(8u, s.size()); + EXPECT_EQUAL(8u, s.capacity()); + EXPECT_EQUAL(prev, s.c_str()); + s.reserve(50); + EXPECT_NOT_EQUAL(prev, s.c_str()); + EXPECT_EQUAL(8u, s.size()); + EXPECT_EQUAL(64u, s.capacity()); + prev = s.c_str(); + s << "A long string that will cause resizing"; + EXPECT_EQUAL(50u, s.size()); + EXPECT_EQUAL(64u, s.capacity()); + EXPECT_EQUAL(prev, s.c_str()); + } + { + nbostream s; + s << long(9); + EXPECT_EQUAL(8u, s.size()); + EXPECT_EQUAL(0u, s.rp()); + long a(7), b(1); + s >> a; + EXPECT_EQUAL(0u, s.size()); + EXPECT_EQUAL(8u, s.rp()); + EXPECT_TRUE(s.empty()); + EXPECT_TRUE(s.good()); + EXPECT_EQUAL(9, a); + try { + s >> b; + EXPECT_TRUE(false); + } catch (const IllegalStateException & e) { + EXPECT_EQUAL("Stream failed bufsize(1024), readp(8), writep(8)", e.getMessage()); + } + EXPECT_EQUAL(0u, s.size()); + EXPECT_EQUAL(8u, s.rp()); + EXPECT_TRUE(s.empty()); + EXPECT_FALSE(s.good()); + EXPECT_EQUAL(1, b); + EXPECT_EQUAL(nbostream::eof, s.state()); + } +} + +int +IdentifiableTest::Main() +{ + TEST_INIT("identifiable_test"); + + TEST_DO(requireThatIdentifiableCastCanCastPointers()); + TEST_DO(requireThatIdentifiableCastCanCastReferences()); + testNamedObject(); + testNboStream(); + testNboSerializer(); + + A a; + B b; + + const Identifiable::RuntimeClass & rtcA = a.getClass(); + EXPECT_EQUAL(rtcA.id(), static_cast<unsigned int>(A::classId)); + EXPECT_EQUAL(strcmp(rtcA.name(), "A"), 0); + + const Identifiable::RuntimeClass & rtcB = b.getClass(); + EXPECT_EQUAL(rtcB.id(), static_cast<unsigned int>(B::classId)); + EXPECT_EQUAL(strcmp(rtcB.name(), "B"), 0); + + const Identifiable::RuntimeClass * rt(Identifiable::classFromId(0x1ab76245)); + ASSERT_TRUE(rt == NULL); + rt = Identifiable::classFromId(Abstract::classId); + ASSERT_TRUE(rt != NULL); + Identifiable * u = rt->create(); + ASSERT_TRUE(u == NULL); + rt = Identifiable::classFromId(A::classId); + ASSERT_TRUE(rt != NULL); + rt = Identifiable::classFromId(B::classId); + ASSERT_TRUE(rt != NULL); + + Identifiable * o = rt->create(); + ASSERT_TRUE(o != NULL); + + const Identifiable::RuntimeClass & rtc = o->getClass(); + ASSERT_TRUE(rtc.id() == B::classId); + ASSERT_TRUE(strcmp(rtc.name(), "B") == 0); + ASSERT_TRUE(o->inherits(B::classId)); + ASSERT_TRUE(o->inherits(A::classId)); + ASSERT_TRUE(o->inherits(Abstract::classId)); + ASSERT_TRUE(o->inherits(Identifiable::classId)); + ASSERT_TRUE(o->getClass().id() == B::classId); + nbostream os; + NBOSerializer nos(os); + nos << *o; + EXPECT_EQUAL(os.size(), 4u); + Identifiable::UP o2 = Identifiable::create(nos); + EXPECT_TRUE(os.empty()); + ASSERT_TRUE(o->inherits(B::classId)); + ASSERT_TRUE(o->getClass().id() == B::classId); + delete o; + + rt = Identifiable::classFromName("NotBNorA"); + ASSERT_TRUE(rt == NULL); + rt = Identifiable::classFromName("B"); + ASSERT_TRUE(rt != NULL); + o = rt->create(); + ASSERT_TRUE(o != NULL); + const Identifiable::RuntimeClass & rtc2 = o->getClass(); + ASSERT_TRUE(rtc2.id() == B::classId); + ASSERT_TRUE(strcmp(rtc2.name(), "B") == 0); + ASSERT_TRUE(o->inherits(B::classId)); + ASSERT_TRUE(o->inherits(A::classId)); + ASSERT_TRUE(o->inherits(Abstract::classId)); + ASSERT_TRUE(o->inherits(Identifiable::classId)); + ASSERT_TRUE(o->getClass().id() == B::classId); + delete o; + + IdentifiablePtr<C> c0(NULL); + IdentifiablePtr<C> c1(new C(10)); + IdentifiablePtr<C> c2(new C(20)); + + EXPECT_LESS(c0.cmp(c1), 0); + EXPECT_EQUAL(c0.cmp(c0), 0); + EXPECT_GREATER(c1.cmp(c0), 0); + + EXPECT_LESS(c1.cmp(c2), 0); + EXPECT_EQUAL(c1.cmp(c1), 0); + EXPECT_GREATER(c2.cmp(c1), 0); + + TEST_DONE(); +} + +void IdentifiableTest::requireThatIdentifiableCastCanCastPointers() { + A a; + B b; + EXPECT_TRUE(Identifiable::cast<A *>(&a)); + EXPECT_TRUE(Identifiable::cast<A *>(&b)); + EXPECT_TRUE(!Identifiable::cast<B *>(&a)); + EXPECT_TRUE(Identifiable::cast<B *>(&b)); + EXPECT_TRUE(Identifiable::cast<Abstract *>(&a)); + EXPECT_TRUE(Identifiable::cast<Abstract *>(&b)); +} + +void IdentifiableTest::requireThatIdentifiableCastCanCastReferences() { + A a; + B b; + try { + // These should not throw. + Identifiable::cast<A &>(a); + Identifiable::cast<A &>(b); + Identifiable::cast<B &>(b); + Identifiable::cast<Abstract &>(a); + Identifiable::cast<Abstract &>(b); + } catch (std::bad_cast &e) { + TEST_FATAL(e.what()); + } + EXPECT_EXCEPTION(Identifiable::cast<B &>(a), std::bad_cast, "bad_cast"); +} + +TEST_APPHOOK(IdentifiableTest) diff --git a/staging_vespalib/src/tests/objectselection/.gitignore b/staging_vespalib/src/tests/objectselection/.gitignore new file mode 100644 index 00000000000..3dece3fbbfe --- /dev/null +++ b/staging_vespalib/src/tests/objectselection/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +objectselection_test +staging_vespalib_objectselection_test_app diff --git a/staging_vespalib/src/tests/objectselection/CMakeLists.txt b/staging_vespalib/src/tests/objectselection/CMakeLists.txt new file mode 100644 index 00000000000..53453035267 --- /dev/null +++ b/staging_vespalib/src/tests/objectselection/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_objectselection_test_app + SOURCES + objectselection.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_objectselection_test_app COMMAND staging_vespalib_objectselection_test_app) diff --git a/staging_vespalib/src/tests/objectselection/DESC b/staging_vespalib/src/tests/objectselection/DESC new file mode 100644 index 00000000000..929fb82d99a --- /dev/null +++ b/staging_vespalib/src/tests/objectselection/DESC @@ -0,0 +1 @@ +objectselection test. Take a look at objectselection.cpp for details. diff --git a/staging_vespalib/src/tests/objectselection/FILES b/staging_vespalib/src/tests/objectselection/FILES new file mode 100644 index 00000000000..ebc80a4c647 --- /dev/null +++ b/staging_vespalib/src/tests/objectselection/FILES @@ -0,0 +1 @@ +objectselection.cpp diff --git a/staging_vespalib/src/tests/objectselection/objectselection.cpp b/staging_vespalib/src/tests/objectselection/objectselection.cpp new file mode 100644 index 00000000000..bc09d17fce5 --- /dev/null +++ b/staging_vespalib/src/tests/objectselection/objectselection.cpp @@ -0,0 +1,93 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("objectselection_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/objects/identifiable.h> +#include <vespa/vespalib/objects/objectpredicate.h> +#include <vespa/vespalib/objects/objectoperation.h> + +using namespace vespalib; + +#define CID_Foo 60000005 +#define CID_Bar 60000010 + +struct Foo : public Identifiable +{ + typedef IdentifiablePtr<Foo> CP; + std::vector<CP> nodes; + + DECLARE_IDENTIFIABLE(Foo); + virtual Foo *clone() const { return new Foo(*this); } + virtual void selectMembers(const ObjectPredicate &p, ObjectOperation &o) { + for (uint32_t i = 0; i < nodes.size(); ++i) { + nodes[i]->select(p, o); + } + } +}; +IMPLEMENT_IDENTIFIABLE(Foo, Identifiable); + +struct Bar : public Foo +{ + int value; + + DECLARE_IDENTIFIABLE(Bar); + Bar() : value(0) {} + Bar(int v) { value = v; } + virtual Bar *clone() const { return new Bar(*this); } +}; +IMPLEMENT_IDENTIFIABLE(Bar, Identifiable); + +struct ObjectType : public ObjectPredicate +{ + uint32_t cid; + ObjectType(uint32_t id) : cid(id) {} + virtual bool check(const Identifiable &obj) const { + return (obj.getClass().id() == cid); + } +}; + +struct ObjectCollect : public ObjectOperation +{ + std::vector<Identifiable*> nodes; + virtual void execute(Identifiable &obj) { + nodes.push_back(&obj); + } +}; + +TEST_SETUP(Test); + +int +Test::Main() +{ + TEST_INIT("objectselection_test"); + { + Foo f1; + Foo f2; + Foo f3; + Bar b1(1); + Bar b2(2); + Bar b3(3); + Bar b4(4); + f2.nodes.push_back(b1); + f2.nodes.push_back(b2); + f3.nodes.push_back(b3); + f3.nodes.push_back(b4); + f1.nodes.push_back(f2); + f1.nodes.push_back(f3); + + ObjectType predicate(Bar::classId); + ObjectCollect operation; + f1.select(predicate, operation); + ASSERT_TRUE(operation.nodes.size() == 4); + ASSERT_TRUE(operation.nodes[0]->getClass().id() == Bar::classId); + ASSERT_TRUE(operation.nodes[1]->getClass().id() == Bar::classId); + ASSERT_TRUE(operation.nodes[2]->getClass().id() == Bar::classId); + ASSERT_TRUE(operation.nodes[3]->getClass().id() == Bar::classId); + ASSERT_TRUE(((Bar*)operation.nodes[0])->value == 1); + ASSERT_TRUE(((Bar*)operation.nodes[1])->value == 2); + ASSERT_TRUE(((Bar*)operation.nodes[2])->value == 3); + ASSERT_TRUE(((Bar*)operation.nodes[3])->value == 4); + } + TEST_DONE(); +} diff --git a/staging_vespalib/src/tests/polymorphicarray/.gitignore b/staging_vespalib/src/tests/polymorphicarray/.gitignore new file mode 100644 index 00000000000..b3a74390312 --- /dev/null +++ b/staging_vespalib/src/tests/polymorphicarray/.gitignore @@ -0,0 +1 @@ +staging_vespalib_polymorphicarray_test_app diff --git a/staging_vespalib/src/tests/polymorphicarray/CMakeLists.txt b/staging_vespalib/src/tests/polymorphicarray/CMakeLists.txt new file mode 100644 index 00000000000..929dd258cfe --- /dev/null +++ b/staging_vespalib/src/tests/polymorphicarray/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_polymorphicarray_test_app + SOURCES + polymorphicarray_test.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_polymorphicarray_test_app COMMAND staging_vespalib_polymorphicarray_test_app) diff --git a/staging_vespalib/src/tests/polymorphicarray/DESC b/staging_vespalib/src/tests/polymorphicarray/DESC new file mode 100644 index 00000000000..dd68c799d64 --- /dev/null +++ b/staging_vespalib/src/tests/polymorphicarray/DESC @@ -0,0 +1 @@ +Array for polymorphic types test. Take a look at polymorphicarray_test.cpp for details. diff --git a/staging_vespalib/src/tests/polymorphicarray/FILES b/staging_vespalib/src/tests/polymorphicarray/FILES new file mode 100644 index 00000000000..d61af365757 --- /dev/null +++ b/staging_vespalib/src/tests/polymorphicarray/FILES @@ -0,0 +1 @@ +polymorphicarray_test.cpp diff --git a/staging_vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp b/staging_vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp new file mode 100644 index 00000000000..9d549d3cc34 --- /dev/null +++ b/staging_vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp @@ -0,0 +1,123 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/util/polymorphicarrays.h> + +using namespace vespalib; + +class A { +public: + virtual ~A() { } + virtual void assign(const A & rhs) { (void) rhs; assert(false); } // Required by the primitive array. + virtual A * clone() const { assert(false); return nullptr; } // Required for the complex array. + + // For testing + virtual bool operator == (const A & rhs) const = 0; + virtual void print(std::ostream & os) const = 0; +}; + +class Primitive : public A +{ +public: + Primitive(size_t v=11) : _v(v) { } + size_t value() const { return _v; } + bool operator == (const A & rhs) const override { + return dynamic_cast<const Primitive &>(rhs).value() == value(); + } + void assign(const A & rhs) override { + _v = dynamic_cast<const Primitive &>(rhs).value(); + } + void print(std::ostream & os) const override { + os << _v; + } +private: + size_t _v; +}; + + +class Complex : public A +{ +public: + Complex(size_t v=11) : _v(v) { } + size_t value() const { return _v; } + bool operator == (const A & rhs) const override { + return dynamic_cast<const Complex &>(rhs).value() == value(); + } + Complex * clone() const override { + return new Complex(_v); + } + void print(std::ostream & os) const override { + os << _v; + } +private: + size_t _v; +}; + +std::ostream & operator << (std::ostream & os, const A & v) { + v.print(os); + return os; +} + + +template <typename T> +void +verifyArray(IArrayT<A> & array) +{ + EXPECT_EQUAL(0u, array.size()); + for (size_t i(0); i < 10; i++) { + array.push_back(T(i)); + } + EXPECT_EQUAL(10u, array.size()); + for (size_t i(0); i < 10; i++) { + EXPECT_EQUAL(T(i), array[i]); + } + IArrayT<A>::UP copy(array.clone()); + array.clear(); + EXPECT_EQUAL(0u, array.size()); + + for (size_t i(0); i < copy->size(); i++) { + array.push_back((*copy)[i]); + } + + array.resize(19); + EXPECT_EQUAL(19u, array.size()); + for (size_t i(0); i < 10; i++) { + EXPECT_EQUAL(T(i), array[i]); + } + for (size_t i(10); i < array.size(); i++) { + EXPECT_EQUAL(T(11), array[i]); + } + array.resize(13); + EXPECT_EQUAL(13u, array.size()); + for (size_t i(0); i < 10; i++) { + EXPECT_EQUAL(T(i), array[i]); + } + for (size_t i(10); i < array.size(); i++) { + EXPECT_EQUAL(T(11), array[i]); + } + dynamic_cast<T &>(array[1]) = T(17); + EXPECT_EQUAL(T(0), array[0]); + EXPECT_EQUAL(T(17), array[1]); + EXPECT_EQUAL(T(2), array[2]); +} + + +TEST("require that primitive arrays conforms") { + PrimitiveArrayT<Primitive, A> a; + verifyArray<Primitive>(a); + EXPECT_EQUAL(7u, a[7].value()); +} + +class Factory : public ComplexArrayT<A>::Factory +{ +public: + A * create() { return new Complex(); } + virtual Factory * clone() const { return new Factory(*this); } +}; + +TEST("require that complex arrays conforms") { + ComplexArrayT<A> a(Factory::UP(new Factory())); + verifyArray<Complex>(a); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/staging_vespalib/src/tests/programoptions/.gitignore b/staging_vespalib/src/tests/programoptions/.gitignore new file mode 100644 index 00000000000..31f3aa61556 --- /dev/null +++ b/staging_vespalib/src/tests/programoptions/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +programoptions_test +staging_vespalib_programoptions_test_app diff --git a/staging_vespalib/src/tests/programoptions/CMakeLists.txt b/staging_vespalib/src/tests/programoptions/CMakeLists.txt new file mode 100644 index 00000000000..7e436e7cee7 --- /dev/null +++ b/staging_vespalib/src/tests/programoptions/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_programoptions_test_app + SOURCES + programoptions_test.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_programoptions_test_app COMMAND staging_vespalib_programoptions_test_app) diff --git a/staging_vespalib/src/tests/programoptions/DESC b/staging_vespalib/src/tests/programoptions/DESC new file mode 100644 index 00000000000..6ec9a059c7d --- /dev/null +++ b/staging_vespalib/src/tests/programoptions/DESC @@ -0,0 +1 @@ +programoptions test. Take a look at programoptions.cpp for details. diff --git a/staging_vespalib/src/tests/programoptions/FILES b/staging_vespalib/src/tests/programoptions/FILES new file mode 100644 index 00000000000..468a6bfb10a --- /dev/null +++ b/staging_vespalib/src/tests/programoptions/FILES @@ -0,0 +1 @@ +programoptions.cpp diff --git a/staging_vespalib/src/tests/programoptions/programoptions_test.cpp b/staging_vespalib/src/tests/programoptions/programoptions_test.cpp new file mode 100644 index 00000000000..5dd27c1ce38 --- /dev/null +++ b/staging_vespalib/src/tests/programoptions/programoptions_test.cpp @@ -0,0 +1,362 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/util/programoptions.h> +#include <vespa/vespalib/util/programoptions_testutils.h> + +#include <iostream> +#include <vespa/log/log.h> +#include <vespa/vespalib/testkit/testapp.h> + +LOG_SETUP("programoptions_test"); + +namespace vespalib { + +class Test : public vespalib::TestApp +{ +public: + void testSyntaxPage(); + void testNormalUsage(); + void testFailures(); + void testVectorArgument(); + void testAllHiddenOption(); + void testOptionsAfterArguments(); + int Main(); +}; + +int +Test::Main() +{ + TEST_INIT("programoptions_test"); + srandom(1); + testSyntaxPage(); + testNormalUsage(); + testFailures(); + testVectorArgument(); + testAllHiddenOption(); + // Currently not supported + // testOptionsAfterArguments(); + TEST_DONE(); +} + +struct MyOptions : public ProgramOptions { + bool boolOpt; + bool boolWithDefOpt; + int intOpt; + uint32_t uintOpt; + float floatOpt; + std::string stringOpt; + std::string argString; + int argInt; + std::string argOptionalString; + std::map<std::string, std::string> properties; + int anotherOptionalArg; + + MyOptions(int argc, const char* const* argv) + : ProgramOptions(argc, argv) + { + // Required options + addOption("uintopt u", uintOpt, "Sets an unsigned int"); + // Optional options + addOption("b bool", boolOpt, "Enables a flag"); + addOption("boolwithdef", boolWithDefOpt, true, "If set turns to false"); + + addOption("intopt i", intOpt, 5, "Sets a signed int"); + addOption("floatopt", floatOpt, 4.0f, "Sets a float\nMultiline baby"); + addOption("string s", stringOpt, std::string("ballalaika"), + "Sets a string value. This is a very long description that " + "should be broken down into multiple lines in some sensible " + "way."); + addOptionHeader("Advanced options"); + addOption("p properties", properties, "Property map"); + addHiddenIdentifiers("prop"); + setArgumentTypeName("key"); + setArgumentTypeName("value", 1); + + addArgument("argString", argString, "Required string argument."); + addArgument("argInt", argInt, "Required int argument."); + addArgument("argOptionalString", argOptionalString, std::string("foo"), + "Optional string argument with a long description so we " + "can see that it will be broken correctly."); + addArgument("argSecondOptional", anotherOptionalArg, 3, + "Yet another optional argument"); + + setSyntaxMessage("A test program to see if this utility works."); + setSyntaxPageMaxLeftColumnSize(25); + } + +}; + +void Test::testSyntaxPage() { + AppOptions opts("myapp"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + std::ostringstream actual; + options.writeSyntaxPage(actual); + + std::string expected( +"\nA test program to see if this utility works.\n\n" +"Usage: myapp [options] <argString> <argInt> [argOptionalString] [argSecondOptional]\n\n" +"Arguments:\n" +" argString (string) : Required string argument.\n" +" argInt (int) : Required int argument.\n" +" argOptionalString (string)\n" +" : Optional string argument with a long description so\n" +" we can see that it will be broken correctly.\n" +" (optional)\n" +" argSecondOptional (int) : Yet another optional argument (optional)\n\n" +"Options:\n" +" --uintopt -u <uint> : Sets an unsigned int (required)\n" +" -b --bool : Enables a flag\n" +" --boolwithdef : If set turns to false\n" +" --intopt -i <int> : Sets a signed int (default 5)\n" +" --floatopt <float> : Sets a float\n" +" Multiline baby (default 4)\n" +" --string -s <string> : Sets a string value. This is a very long description\n" +" that should be broken down into multiple lines in some\n" +" sensible way. (default \"ballalaika\")\n\n" +"Advanced options:\n" +" -p --properties <key> <value> : Property map (default empty)\n" + ); + EXPECT_EQUAL(expected, actual.str()); +} + +void Test::testNormalUsage() { + { + AppOptions opts("myapp -b --uintopt 4 -s foo tit 1 tei 6"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + options.parse(); + EXPECT_EQUAL(true, options.boolOpt); + EXPECT_EQUAL(true, options.boolWithDefOpt); + EXPECT_EQUAL(5, options.intOpt); + EXPECT_EQUAL(4u, options.uintOpt); + EXPECT_APPROX(4, options.floatOpt, 0.00001); + EXPECT_EQUAL("foo", options.stringOpt); + EXPECT_EQUAL("tit", options.argString); + EXPECT_EQUAL(1, options.argInt); + EXPECT_EQUAL("tei", options.argOptionalString); + EXPECT_EQUAL(0u, options.properties.size()); + EXPECT_EQUAL(6, options.anotherOptionalArg); + } + { + AppOptions opts("myapp --uintopt 6 tit 1"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + options.parse(); + EXPECT_EQUAL(false, options.boolOpt); + EXPECT_EQUAL(true, options.boolWithDefOpt); + EXPECT_EQUAL(5, options.intOpt); + EXPECT_EQUAL(6u, options.uintOpt); + EXPECT_APPROX(4, options.floatOpt, 0.00001); + EXPECT_EQUAL("ballalaika", options.stringOpt); + EXPECT_EQUAL("tit", options.argString); + EXPECT_EQUAL(1, options.argInt); + EXPECT_EQUAL("foo", options.argOptionalString); + EXPECT_EQUAL(0u, options.properties.size()); + EXPECT_EQUAL(3, options.anotherOptionalArg); + } + // Arguments coming after options. + // (Required for nesting of short options) + { + AppOptions opts("myapp --uintopt --intopt 6 -8 tit 1 tei"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + options.parse(); + EXPECT_EQUAL(false, options.boolOpt); + EXPECT_EQUAL(true, options.boolWithDefOpt); + EXPECT_EQUAL(-8, options.intOpt); + EXPECT_EQUAL(6u, options.uintOpt); + EXPECT_APPROX(4, options.floatOpt, 0.00001); + EXPECT_EQUAL("ballalaika", options.stringOpt); + EXPECT_EQUAL("tit", options.argString); + EXPECT_EQUAL(1, options.argInt); + EXPECT_EQUAL("tei", options.argOptionalString); + EXPECT_EQUAL(0u, options.properties.size()); + } + { + AppOptions opts( "myapp -uib 6 -8 --boolwithdef tit 1 tei"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + options.parse(); + EXPECT_EQUAL(true, options.boolOpt); + EXPECT_EQUAL(false, options.boolWithDefOpt); + EXPECT_EQUAL(-8, options.intOpt); + EXPECT_EQUAL(6u, options.uintOpt); + EXPECT_APPROX(4, options.floatOpt, 0.00001); + EXPECT_EQUAL("ballalaika", options.stringOpt); + EXPECT_EQUAL("tit", options.argString); + EXPECT_EQUAL(1, options.argInt); + EXPECT_EQUAL("tei", options.argOptionalString); + EXPECT_EQUAL(0u, options.properties.size()); + } + // Properties + { + AppOptions opts("myapp -u 6 -p foo bar --prop hmm brr tit 1 tei"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + options.parse(); + EXPECT_EQUAL(false, options.boolOpt); + EXPECT_EQUAL(true, options.boolWithDefOpt); + EXPECT_EQUAL(5, options.intOpt); + EXPECT_EQUAL(6u, options.uintOpt); + EXPECT_APPROX(4, options.floatOpt, 0.00001); + EXPECT_EQUAL("ballalaika", options.stringOpt); + EXPECT_EQUAL("tit", options.argString); + EXPECT_EQUAL(1, options.argInt); + EXPECT_EQUAL("tei", options.argOptionalString); + EXPECT_EQUAL(2u, options.properties.size()); + EXPECT_EQUAL("bar", options.properties["foo"]); + EXPECT_EQUAL("brr", options.properties["hmm"]); + } +} + +void Test::testFailures() { + // Non-existing long option + { + AppOptions opts("myapp -b --uintopt 4 -s foo --none"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + try{ + options.parse(); + TEST_FATAL("Expected exception"); + } catch (InvalidCommandLineArgumentsException& e) { + EXPECT_EQUAL("Invalid option 'none'.", e.getMessage()); + } + } + // Non-existing short option + { + AppOptions opts("myapp -b --uintopt 4 -s foo -q"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + try{ + options.parse(); + TEST_FATAL("Expected exception"); + } catch (InvalidCommandLineArgumentsException& e) { + EXPECT_EQUAL("Invalid option 'q'.", e.getMessage()); + } + } + // Lacking option argument + { + AppOptions opts("myapp -b --uintopt 4 -s"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + try{ + options.parse(); + TEST_FATAL("Expected exception"); + } catch (InvalidCommandLineArgumentsException& e) { + EXPECT_EQUAL("Option 's' needs 1 arguments. Only 0 available.", + e.getMessage()); + } + } + // Out of signed ranged + { + AppOptions opts("myapp -b --uintopt 4 -intopt 3000000000"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + try{ + options.parse(); + TEST_FATAL("Expected exception"); + } catch (InvalidCommandLineArgumentsException& e) { + EXPECT_EQUAL("The argument '3000000000' can not be interpreted as a " + "number of type int.", e.getMessage()); + } + } + // Negative value to unsigned var (Currently doesnt fail) +/* + { + AppOptions opts("myapp -b --uintopt -1 foo 0"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + try{ + options.parse(); + TEST_FATAL("Expected exception"); + } catch (InvalidCommandLineArgumentsException& e) { + EXPECT_EQUAL("The argument '-1' can not be interpreted as a " + "number of type uint.", e.getMessage()); + } + } + */ + // Lacking required option + { + AppOptions opts("myapp -b"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + try{ + options.parse(); + TEST_FATAL("Expected exception"); + } catch (InvalidCommandLineArgumentsException& e) { + EXPECT_EQUAL("Option 'uintopt' has no default and must be set.", + e.getMessage()); + } + } + // Lacking required argument + { + AppOptions opts("myapp --uintopt 1 tit"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + try{ + options.parse(); + TEST_FATAL("Expected exception"); + } catch (InvalidCommandLineArgumentsException& e) { + EXPECT_EQUAL("Insufficient data is given to set required argument " + "'argInt'.", + e.getMessage()); + } + } + // Argument of wrong type + { + AppOptions opts("myapp --uintopt 1 tit en"); + MyOptions options(opts.getArgCount(), opts.getArguments()); + try{ + options.parse(); + TEST_FATAL("Expected exception"); + } catch (InvalidCommandLineArgumentsException& e) { + EXPECT_EQUAL("The argument 'en' can not be interpreted as a number " + "of type int.", + e.getMessage()); + } + } +} + +void Test::testVectorArgument() +{ + AppOptions opts("myapp foo bar baz"); + std::vector<std::string> args; + ProgramOptions options(opts.getArgCount(), opts.getArguments()); + options.addListArgument("ids", args, "Vector element"); + std::ostringstream actual; + options.writeSyntaxPage(actual); + std::string expected( +"\nUsage: myapp [ids...]\n\n" +"Arguments:\n" +" ids (string[]) : Vector element\n" + ); + EXPECT_EQUAL(expected, actual.str()); + + options.parse(); + EXPECT_EQUAL(3u, args.size()); + EXPECT_EQUAL("foo", args[0]); + EXPECT_EQUAL("bar", args[1]); + EXPECT_EQUAL("baz", args[2]); +} + +void Test::testAllHiddenOption() +{ + AppOptions opts("myapp --foo bar"); + std::string option; + ProgramOptions options(opts.getArgCount(), opts.getArguments()); + options.addOption("", option, "Description"); + options.addHiddenIdentifiers("foo"); + std::ostringstream actual; + options.writeSyntaxPage(actual); + std::string expected("\nUsage: myapp\n"); + EXPECT_EQUAL(expected, actual.str()); + + options.parse(); + EXPECT_EQUAL("bar", option); +} + +void Test::testOptionsAfterArguments() +{ + AppOptions opts("myapp bar --foo baz"); + std::string option; + std::string argument; + ProgramOptions options(opts.getArgCount(), opts.getArguments()); + options.addOption("foo", option, "Description"); + options.addArgument("arg", argument, "Description"); + options.parse(); + EXPECT_EQUAL("baz", option); + EXPECT_EQUAL("bar", argument); +} + +} // vespalib + +TEST_APPHOOK(vespalib::Test) diff --git a/staging_vespalib/src/tests/rusage/.gitignore b/staging_vespalib/src/tests/rusage/.gitignore new file mode 100644 index 00000000000..195922a9ced --- /dev/null +++ b/staging_vespalib/src/tests/rusage/.gitignore @@ -0,0 +1 @@ +staging_vespalib_rusage_test_app diff --git a/staging_vespalib/src/tests/rusage/CMakeLists.txt b/staging_vespalib/src/tests/rusage/CMakeLists.txt new file mode 100644 index 00000000000..b927110179e --- /dev/null +++ b/staging_vespalib/src/tests/rusage/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_rusage_test_app + SOURCES + rusage_test.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_rusage_test_app COMMAND staging_vespalib_rusage_test_app) diff --git a/staging_vespalib/src/tests/rusage/DESC b/staging_vespalib/src/tests/rusage/DESC new file mode 100644 index 00000000000..957ed92a119 --- /dev/null +++ b/staging_vespalib/src/tests/rusage/DESC @@ -0,0 +1 @@ +rusage test. Take a look at rusage_test.cpp for details. diff --git a/staging_vespalib/src/tests/rusage/FILES b/staging_vespalib/src/tests/rusage/FILES new file mode 100644 index 00000000000..bea76c6f532 --- /dev/null +++ b/staging_vespalib/src/tests/rusage/FILES @@ -0,0 +1 @@ +rusage_test.cpp diff --git a/staging_vespalib/src/tests/rusage/rusage_test.cpp b/staging_vespalib/src/tests/rusage/rusage_test.cpp new file mode 100644 index 00000000000..3da7f1d0b54 --- /dev/null +++ b/staging_vespalib/src/tests/rusage/rusage_test.cpp @@ -0,0 +1,76 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <stddef.h> +#include <vespa/log/log.h> +LOG_SETUP("alloc_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/rusage.h> + +using namespace vespalib; + +class Test : public TestApp +{ +public: + int Main(); + void testRUsage(); +}; + +int +Test::Main() +{ + TEST_INIT("rusage_test"); + testRUsage(); + TEST_DONE(); +} + +void +Test::testRUsage() +{ + RUsage r1; + EXPECT_EQUAL("", r1.toString()); + RUsage r2; + EXPECT_EQUAL(r2.toString(), r1.toString()); + RUsage diff = r2-r1; + EXPECT_EQUAL(diff.toString(), r2.toString()); + { + RUsage then = RUsage::createSelf(17765895674); + RUsage now = RUsage::createSelf(); + EXPECT_NOT_EQUAL(now.toString(), then.toString()); + } + { + RUsage then = RUsage::createChildren(1337583); + RUsage now = RUsage::createChildren(); + EXPECT_NOT_EQUAL(now.toString(), then.toString()); + } + { + timeval a, b, c, d, r; + a.tv_usec = 7; + a.tv_sec = 7; + b.tv_usec = 7; + b.tv_sec = 7; + c.tv_usec = 1; + c.tv_sec = 8; + d.tv_usec = 9; + d.tv_sec = 4; + r = a - b; + EXPECT_EQUAL(0, r.tv_sec); + EXPECT_EQUAL(0, r.tv_usec); + r = b - a; + EXPECT_EQUAL(0, r.tv_sec); + EXPECT_EQUAL(0, r.tv_usec); + r = a - c; + EXPECT_EQUAL(-1, r.tv_sec); + EXPECT_EQUAL( 6, r.tv_usec); + r = c - a; + EXPECT_EQUAL(0, r.tv_sec); + EXPECT_EQUAL(999994, r.tv_usec); + r = a - d; + EXPECT_EQUAL(2, r.tv_sec); + EXPECT_EQUAL(999998, r.tv_usec); + r = d - a; + EXPECT_EQUAL(-3, r.tv_sec); + EXPECT_EQUAL( 2, r.tv_usec); + } +} + +TEST_APPHOOK(Test) diff --git a/staging_vespalib/src/tests/shutdownguard/.gitignore b/staging_vespalib/src/tests/shutdownguard/.gitignore new file mode 100644 index 00000000000..a596164ac7b --- /dev/null +++ b/staging_vespalib/src/tests/shutdownguard/.gitignore @@ -0,0 +1 @@ +staging_vespalib_shutdownguard_test_app diff --git a/staging_vespalib/src/tests/shutdownguard/CMakeLists.txt b/staging_vespalib/src/tests/shutdownguard/CMakeLists.txt new file mode 100644 index 00000000000..75093312cd3 --- /dev/null +++ b/staging_vespalib/src/tests/shutdownguard/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_shutdownguard_test_app + SOURCES + shutdownguard_test.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_shutdownguard_test_app NO_VALGRIND COMMAND staging_vespalib_shutdownguard_test_app) diff --git a/staging_vespalib/src/tests/shutdownguard/DESC b/staging_vespalib/src/tests/shutdownguard/DESC new file mode 100644 index 00000000000..869fc61d80f --- /dev/null +++ b/staging_vespalib/src/tests/shutdownguard/DESC @@ -0,0 +1 @@ +shutdown guard test. Take a look at shutdownguard_test.cpp for details. diff --git a/staging_vespalib/src/tests/shutdownguard/FILES b/staging_vespalib/src/tests/shutdownguard/FILES new file mode 100644 index 00000000000..0c040a71f20 --- /dev/null +++ b/staging_vespalib/src/tests/shutdownguard/FILES @@ -0,0 +1 @@ +suicide_test.cpp diff --git a/staging_vespalib/src/tests/shutdownguard/shutdownguard_test.cpp b/staging_vespalib/src/tests/shutdownguard/shutdownguard_test.cpp new file mode 100644 index 00000000000..c31e34afe76 --- /dev/null +++ b/staging_vespalib/src/tests/shutdownguard/shutdownguard_test.cpp @@ -0,0 +1,40 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("shutdownguard_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/shutdownguard.h> + +using namespace vespalib; + +TEST_SETUP(Test); + +int +Test::Main() +{ + TEST_INIT("shutdownguard_test"); + { + ShutdownGuard farFuture(123456789); + FastOS_Thread::Sleep(20); + } + EXPECT_TRUE(true); + pid_t child = fork(); + if (child == 0) { + ShutdownGuard soon(30); + for (int i = 0; i < 1000; ++i) { + FastOS_Thread::Sleep(20); + } + exit(0); + } + for (int i = 0; i < 1000; ++i) { + FastOS_Thread::Sleep(20); + int stat = 0; + if (waitpid(child, &stat, WNOHANG) == child) { + EXPECT_TRUE(WIFEXITED(stat)); + EXPECT_EQUAL(1, WEXITSTATUS(stat)); + break; + } + EXPECT_TRUE(i < 800); + } + TEST_DONE(); +} diff --git a/staging_vespalib/src/tests/state_server/.gitignore b/staging_vespalib/src/tests/state_server/.gitignore new file mode 100644 index 00000000000..b1884d4e060 --- /dev/null +++ b/staging_vespalib/src/tests/state_server/.gitignore @@ -0,0 +1 @@ +staging_vespalib_state_server_test_app diff --git a/staging_vespalib/src/tests/state_server/CMakeLists.txt b/staging_vespalib/src/tests/state_server/CMakeLists.txt new file mode 100644 index 00000000000..a62549a563b --- /dev/null +++ b/staging_vespalib/src/tests/state_server/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_state_server_test_app + SOURCES + state_server_test.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_state_server_test_app NO_VALGRIND NO_VALGRIND COMMAND staging_vespalib_state_server_test_app) diff --git a/staging_vespalib/src/tests/state_server/FILES b/staging_vespalib/src/tests/state_server/FILES new file mode 100644 index 00000000000..2328b597e56 --- /dev/null +++ b/staging_vespalib/src/tests/state_server/FILES @@ -0,0 +1 @@ +state_server_test.cpp diff --git a/staging_vespalib/src/tests/state_server/state_server_test.cpp b/staging_vespalib/src/tests/state_server/state_server_test.cpp new file mode 100644 index 00000000000..79236523694 --- /dev/null +++ b/staging_vespalib/src/tests/state_server/state_server_test.cpp @@ -0,0 +1,444 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/net/state_server.h> +#include <vespa/vespalib/net/simple_health_producer.h> +#include <vespa/vespalib/net/simple_metrics_producer.h> +#include <vespa/vespalib/net/simple_component_config_producer.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/slaveproc.h> +#include <vespa/vespalib/net/state_explorer.h> +#include <vespa/vespalib/net/slime_explorer.h> +#include <vespa/vespalib/net/generic_state_handler.h> + +using namespace vespalib; + +//----------------------------------------------------------------------------- + +vespalib::string root_path = "/state/v1/"; +vespalib::string short_root_path = "/state/v1"; +vespalib::string metrics_path = "/state/v1/metrics"; +vespalib::string health_path = "/state/v1/health"; +vespalib::string config_path = "/state/v1/config"; + +vespalib::string total_metrics_path = "/metrics/total"; + +vespalib::string unknown_path = "/this/path/is/not/known"; +vespalib::string unknown_state_path = "/state/v1/this/path/is/not/known"; +vespalib::string my_path = "/my/path"; + +vespalib::string host_tag = "HOST"; +std::map<vespalib::string,vespalib::string> empty_params; + +//----------------------------------------------------------------------------- + +vespalib::string run_cmd(const vespalib::string &cmd) { + std::string out; + ASSERT_TRUE(SlaveProc::run(cmd.c_str(), out)); + return out; +} + +vespalib::string getPage(int port, const vespalib::string &path, const vespalib::string &extra_params = "") { + vespalib::string result = run_cmd(make_string("curl -s %s http://localhost:%d%s", extra_params.c_str(), port, path.c_str())); + vespalib::string chunked_result = run_cmd(make_string("curl -H transfer-encoding:chunked -s %s http://localhost:%d%s", extra_params.c_str(), port, path.c_str())); + ASSERT_EQUAL(result, chunked_result); + return result; +} + +vespalib::string getFull(int port, const vespalib::string &path) { return getPage(port, path, "-D -"); } + +//----------------------------------------------------------------------------- + +struct DummyHandler : JsonGetHandler { + vespalib::string result; + DummyHandler(const vespalib::string &result_in) : result(result_in) {} + virtual vespalib::string get(const vespalib::string &, + const vespalib::string &, + const std::map<vespalib::string,vespalib::string> &) const + { + return result; + } +}; + +//----------------------------------------------------------------------------- + +TEST_F("require that unknown url returns 404 response", HttpServer(0)) { + f1.start(); + std::string expect("HTTP/1.1 404 Not Found\r\n" + "Connection: close\r\n" + "\r\n"); + std::string actual = getFull(f1.port(), unknown_path); + EXPECT_EQUAL(expect, actual); +} + +TEST_FF("require that empty known url returns 404 response", DummyHandler(""), HttpServer(0)) { + auto token = f2.repo().bind(my_path, f1); + f2.start(); + std::string expect("HTTP/1.1 404 Not Found\r\n" + "Connection: close\r\n" + "\r\n"); + std::string actual = getFull(f2.port(), my_path); + EXPECT_EQUAL(expect, actual); +} + +TEST_FF("require that non-empty known url returns expected headers", DummyHandler("[123]"), HttpServer(0)) { + auto token = f2.repo().bind(my_path, f1); + f2.start(); + vespalib::string expect("HTTP/1.1 200 OK\r\n" + "Connection: close\r\n" + "Content-Type: application/json\r\n" + "\r\n" + "[123]"); + std::string actual = getFull(f2.port(), my_path); + EXPECT_EQUAL(expect, actual); +} + +TEST_FFFF("require that handler is selected based on longest matching url prefix", + DummyHandler("[1]"), DummyHandler("[2]"), DummyHandler("[3]"), + HttpServer(0)) +{ + auto token2 = f4.repo().bind("/foo/bar", f2); + auto token1 = f4.repo().bind("/foo", f1); + auto token3 = f4.repo().bind("/foo/bar/baz", f3); + f4.start(); + int port = f4.port(); + EXPECT_EQUAL("", getPage(port, "/fox")); + EXPECT_EQUAL("[1]", getPage(port, "/foo")); + EXPECT_EQUAL("[1]", getPage(port, "/foo/fox")); + EXPECT_EQUAL("[2]", getPage(port, "/foo/bar")); + EXPECT_EQUAL("[2]", getPage(port, "/foo/bar/fox")); + EXPECT_EQUAL("[3]", getPage(port, "/foo/bar/baz")); + EXPECT_EQUAL("[3]", getPage(port, "/foo/bar/baz/fox")); +} + +struct EchoHost : JsonGetHandler { + virtual vespalib::string get(const vespalib::string &host, + const vespalib::string &, + const std::map<vespalib::string,vespalib::string> &) const + { + return "[\"" + host + "\"]"; + } +}; + +TEST_FF("require that host is passed correctly", EchoHost(), HttpServer(0)) { + auto token = f2.repo().bind(my_path, f1); + f2.start(); + EXPECT_EQUAL(make_string("%s:%d", run_cmd("hostname").c_str(), f2.port()), f2.host()); + vespalib::string default_result = make_string("[\"%s\"]", f2.host().c_str()); + vespalib::string localhost_result = make_string("[\"%s:%d\"]", "localhost", f2.port()); + vespalib::string silly_result = "[\"sillyserver\"]"; + EXPECT_EQUAL(localhost_result, run_cmd(make_string("curl -s http://localhost:%d/my/path", f2.port()))); + EXPECT_EQUAL(silly_result, run_cmd(make_string("curl -s http://localhost:%d/my/path -H \"Host: sillyserver\"", f2.port()))); + EXPECT_EQUAL(default_result, run_cmd(make_string("curl -s http://localhost:%d/my/path -H \"Host:\"", f2.port()))); +} + +//----------------------------------------------------------------------------- + +TEST_FFFF("require that the state server wires the appropriate url prefixes", + SimpleHealthProducer(), SimpleMetricsProducer(), SimpleComponentConfigProducer(), + StateServer(0, f1, f2, f3)) +{ + f2.setTotalMetrics("{}"); // avoid empty result + int port = f4.getListenPort(); + EXPECT_TRUE(getFull(port, short_root_path).find("HTTP/1.1 200 OK") == 0); + EXPECT_TRUE(getFull(port, total_metrics_path).find("HTTP/1.1 200 OK") == 0); + EXPECT_TRUE(getFull(port, unknown_path).find("HTTP/1.1 404 Not Found") == 0); +} + +TEST_FFFF("require that the state server exposes the state api handler repo", + SimpleHealthProducer(), SimpleMetricsProducer(), SimpleComponentConfigProducer(), + StateServer(0, f1, f2, f3)) +{ + int port = f4.getListenPort(); + vespalib::string page1 = getPage(port, root_path); + auto token = f4.repo().add_root_resource("state/v1/custom"); + vespalib::string page2 = getPage(port, root_path); + EXPECT_NOT_EQUAL(page1, page2); + token.reset(); + vespalib::string page3 = getPage(port, root_path); + EXPECT_EQUAL(page3, page1); +} + +//----------------------------------------------------------------------------- + +TEST_FFFF("require that json handlers can be removed from repo", + DummyHandler("[1]"), DummyHandler("[2]"), DummyHandler("[3]"), + JsonHandlerRepo()) +{ + auto token1 = f4.bind("/foo", f1); + auto token2 = f4.bind("/foo/bar", f2); + auto token3 = f4.bind("/foo/bar/baz", f3); + std::map<vespalib::string,vespalib::string> params; + EXPECT_EQUAL("[1]", f4.get("", "/foo", params)); + EXPECT_EQUAL("[2]", f4.get("", "/foo/bar", params)); + EXPECT_EQUAL("[3]", f4.get("", "/foo/bar/baz", params)); + token2.reset(); + EXPECT_EQUAL("[1]", f4.get("", "/foo", params)); + EXPECT_EQUAL("[1]", f4.get("", "/foo/bar", params)); + EXPECT_EQUAL("[3]", f4.get("", "/foo/bar/baz", params)); +} + +TEST_FFFF("require that json handlers can be shadowed", + DummyHandler("[1]"), DummyHandler("[2]"), DummyHandler("[3]"), + JsonHandlerRepo()) +{ + auto token1 = f4.bind("/foo", f1); + auto token2 = f4.bind("/foo/bar", f2); + std::map<vespalib::string,vespalib::string> params; + EXPECT_EQUAL("[1]", f4.get("", "/foo", params)); + EXPECT_EQUAL("[2]", f4.get("", "/foo/bar", params)); + auto token3 = f4.bind("/foo/bar", f3); + EXPECT_EQUAL("[3]", f4.get("", "/foo/bar", params)); + token3.reset(); + EXPECT_EQUAL("[2]", f4.get("", "/foo/bar", params)); +} + +TEST_F("require that root resources can be tracked", JsonHandlerRepo()) +{ + EXPECT_TRUE(std::vector<vespalib::string>({}) == f1.get_root_resources()); + auto token1 = f1.add_root_resource("/health"); + EXPECT_TRUE(std::vector<vespalib::string>({"/health"}) == f1.get_root_resources()); + auto token2 = f1.add_root_resource("/config"); + EXPECT_TRUE(std::vector<vespalib::string>({"/health", "/config"}) == f1.get_root_resources()); + auto token3 = f1.add_root_resource("/custom/foo"); + EXPECT_TRUE(std::vector<vespalib::string>({"/health", "/config", "/custom/foo"}) == f1.get_root_resources()); + token2.reset(); + EXPECT_TRUE(std::vector<vespalib::string>({"/health", "/custom/foo"}) == f1.get_root_resources()); +} + +//----------------------------------------------------------------------------- + +TEST_FFFF("require that state api responds to the expected paths", + SimpleHealthProducer(), SimpleMetricsProducer(), SimpleComponentConfigProducer(), + StateApi(f1, f2, f3)) +{ + f2.setTotalMetrics("{}"); // avoid empty result + EXPECT_TRUE(!f4.get(host_tag, short_root_path, empty_params).empty()); + EXPECT_TRUE(!f4.get(host_tag, root_path, empty_params).empty()); + EXPECT_TRUE(!f4.get(host_tag, health_path, empty_params).empty()); + EXPECT_TRUE(!f4.get(host_tag, metrics_path, empty_params).empty()); + EXPECT_TRUE(!f4.get(host_tag, config_path, empty_params).empty()); + EXPECT_TRUE(!f4.get(host_tag, total_metrics_path, empty_params).empty()); + EXPECT_TRUE(f4.get(host_tag, unknown_path, empty_params).empty()); + EXPECT_TRUE(f4.get(host_tag, unknown_state_path, empty_params).empty()); +} + +TEST_FFFF("require that top-level urls are generated correctly", + SimpleHealthProducer(), SimpleMetricsProducer(), SimpleComponentConfigProducer(), + StateApi(f1, f2, f3)) +{ + EXPECT_EQUAL("{\"resources\":[" + "{\"url\":\"http://HOST/state/v1/health\"}," + "{\"url\":\"http://HOST/state/v1/metrics\"}," + "{\"url\":\"http://HOST/state/v1/config\"}]}", + f4.get(host_tag, root_path, empty_params)); + EXPECT_EQUAL(f4.get(host_tag, root_path, empty_params), + f4.get(host_tag, short_root_path, empty_params)); +} + +TEST_FFFF("require that top-level resource list can be extended", + SimpleHealthProducer(), SimpleMetricsProducer(), SimpleComponentConfigProducer(), + StateApi(f1, f2, f3)) +{ + auto token = f4.repo().add_root_resource("/state/v1/custom"); + EXPECT_EQUAL("{\"resources\":[" + "{\"url\":\"http://HOST/state/v1/health\"}," + "{\"url\":\"http://HOST/state/v1/metrics\"}," + "{\"url\":\"http://HOST/state/v1/config\"}," + "{\"url\":\"http://HOST/state/v1/custom\"}]}", + f4.get(host_tag, root_path, empty_params)); +} + +TEST_FFFF("require that health resource works as expected", + SimpleHealthProducer(), SimpleMetricsProducer(), SimpleComponentConfigProducer(), + StateApi(f1, f2, f3)) +{ + EXPECT_EQUAL("{\"status\":{\"code\":\"up\"}}", + f4.get(host_tag, health_path, empty_params)); + f1.setFailed("FAIL MSG"); + EXPECT_EQUAL("{\"status\":{\"code\":\"down\",\"message\":\"FAIL MSG\"}}", + f4.get(host_tag, health_path, empty_params)); +} + +TEST_FFFF("require that metrics resource works as expected", + SimpleHealthProducer(), SimpleMetricsProducer(), SimpleComponentConfigProducer(), + StateApi(f1, f2, f3)) +{ + EXPECT_EQUAL("{\"status\":{\"code\":\"up\"}}", + f4.get(host_tag, metrics_path, empty_params)); + f1.setFailed("FAIL MSG"); + EXPECT_EQUAL("{\"status\":{\"code\":\"down\",\"message\":\"FAIL MSG\"}}", + f4.get(host_tag, metrics_path, empty_params)); + f1.setOk(); + f2.setMetrics("{\"foo\":\"bar\"}"); + EXPECT_EQUAL("{\"status\":{\"code\":\"up\"},\"metrics\":{\"foo\":\"bar\"}}", + f4.get(host_tag, metrics_path, empty_params)); +} + +TEST_FFFF("require that config resource works as expected", + SimpleHealthProducer(), SimpleMetricsProducer(), SimpleComponentConfigProducer(), + StateApi(f1, f2, f3)) +{ + EXPECT_EQUAL("{\"config\":{}}", + f4.get(host_tag, config_path, empty_params)); + f3.addConfig(SimpleComponentConfigProducer::Config("foo", 3)); + EXPECT_EQUAL("{\"config\":{\"generation\":3,\"foo\":{\"generation\":3}}}", + f4.get(host_tag, config_path, empty_params)); + f3.addConfig(SimpleComponentConfigProducer::Config("foo", 4)); + f3.addConfig(SimpleComponentConfigProducer::Config("bar", 4, "error")); + EXPECT_EQUAL("{\"config\":{\"generation\":4,\"bar\":{\"generation\":4,\"message\":\"error\"},\"foo\":{\"generation\":4}}}", + f4.get(host_tag, config_path, empty_params)); + f3.removeConfig("bar"); + EXPECT_EQUAL("{\"config\":{\"generation\":4,\"foo\":{\"generation\":4}}}", + f4.get(host_tag, config_path, empty_params)); +} + +TEST_FFFF("require that state api also can return total metric", + SimpleHealthProducer(), SimpleMetricsProducer(), SimpleComponentConfigProducer(), + StateApi(f1, f2, f3)) +{ + f2.setTotalMetrics("{\"foo\":\"bar\"}"); + EXPECT_EQUAL("{\"foo\":\"bar\"}", + f4.get(host_tag, total_metrics_path, empty_params)); +} + +TEST_FFFFF("require that custom handlers can be added to the state server", + SimpleHealthProducer(), SimpleMetricsProducer(), SimpleComponentConfigProducer(), + StateApi(f1, f2, f3), DummyHandler("[123]")) +{ + EXPECT_EQUAL("", f4.get(host_tag, my_path, empty_params)); + auto token = f4.repo().bind(my_path, f5); + EXPECT_EQUAL("[123]", f4.get(host_tag, my_path, empty_params)); + token.reset(); + EXPECT_EQUAL("", f4.get(host_tag, my_path, empty_params)); +} + +struct EchoConsumer : MetricsProducer { + virtual vespalib::string getMetrics(const vespalib::string &consumer) { + return "[\"" + consumer + "\"]"; + } + virtual vespalib::string getTotalMetrics(const vespalib::string &consumer) { + return "[\"" + consumer + "\"]"; + } +}; + +TEST_FFFF("require that metrics consumer is passed correctly", + SimpleHealthProducer(), EchoConsumer(), SimpleComponentConfigProducer(), + StateApi(f1, f2, f3)) +{ + std::map<vespalib::string,vespalib::string> my_params; + my_params["consumer"] = "ME"; + EXPECT_EQUAL("{\"status\":{\"code\":\"up\"},\"metrics\":[\"\"]}", f4.get(host_tag, metrics_path, empty_params)); + EXPECT_EQUAL("{\"status\":{\"code\":\"up\"},\"metrics\":[\"ME\"]}", f4.get(host_tag, metrics_path, my_params)); + EXPECT_EQUAL("[\"\"]", f4.get(host_tag, total_metrics_path, empty_params)); + EXPECT_EQUAL("[\"ME\"]", f4.get(host_tag, total_metrics_path, my_params)); +} + +void check_json(const vespalib::string &expect_json, const vespalib::string &actual_json) { + Slime expect_slime; + Slime actual_slime; + EXPECT_TRUE(slime::JsonFormat::decode(expect_json, expect_slime) > 0); + EXPECT_TRUE(slime::JsonFormat::decode(actual_json, actual_slime) > 0); + EXPECT_EQUAL(expect_slime, actual_slime); +} + +TEST("require that generic state can be explored") { + vespalib::string json_model = + "{" + " foo: 'bar'," + " cnt: 123," + " engine: {" + " up: 'yes'," + " stats: {" + " latency: 5," + " qps: 100" + " }" + " }," + " list: {" + " one: {" + " size: {" + " value: 1" + " }" + " }," + " two: {" + " size: 2" + " }" + " }" + "}"; + vespalib::string json_root = + "{" + " full: true," + " foo: 'bar'," + " cnt: 123," + " engine: {" + " up: 'yes'," + " url: 'http://HOST/state/v1/engine'" + " }," + " list: {" + " one: {" + " size: {" + " value: 1," + " url: 'http://HOST/state/v1/list/one/size'" + " }" + " }," + " two: {" + " size: 2," + " url: 'http://HOST/state/v1/list/two'" + " }" + " }" + "}"; + vespalib::string json_engine = + "{" + " full: true," + " up: 'yes'," + " stats: {" + " latency: 5," + " qps: 100," + " url: 'http://HOST/state/v1/engine/stats'" + " }" + "}"; + vespalib::string json_engine_stats = + "{" + " full: true," + " latency: 5," + " qps: 100" + "}"; + vespalib::string json_list = + "{" + " one: {" + " size: {" + " value: 1," + " url: 'http://HOST/state/v1/list/one/size'" + " }" + " }," + " two: {" + " size: 2," + " url: 'http://HOST/state/v1/list/two'" + " }" + "}"; + vespalib::string json_list_one = + "{" + " size: {" + " value: 1," + " url: 'http://HOST/state/v1/list/one/size'" + " }" + "}"; + vespalib::string json_list_one_size = "{ full: true, value: 1 }"; + vespalib::string json_list_two = "{ full: true, size: 2 }"; + //------------------------------------------------------------------------- + Slime slime_state; + EXPECT_TRUE(slime::JsonFormat::decode(json_model, slime_state) > 0); + SlimeExplorer slime_explorer(slime_state.get()); + GenericStateHandler state_handler(short_root_path, slime_explorer); + EXPECT_EQUAL("", state_handler.get(host_tag, unknown_path, empty_params)); + EXPECT_EQUAL("", state_handler.get(host_tag, unknown_state_path, empty_params)); + check_json(json_root, state_handler.get(host_tag, root_path, empty_params)); + check_json(json_engine, state_handler.get(host_tag, root_path + "engine", empty_params)); + check_json(json_engine_stats, state_handler.get(host_tag, root_path + "engine/stats", empty_params)); + check_json(json_list, state_handler.get(host_tag, root_path + "list", empty_params)); + check_json(json_list_one, state_handler.get(host_tag, root_path + "list/one", empty_params)); + check_json(json_list_one_size, state_handler.get(host_tag, root_path + "list/one/size", empty_params)); + check_json(json_list_two, state_handler.get(host_tag, root_path + "list/two", empty_params)); +} + +TEST_MAIN_WITH_PROCESS_PROXY() { TEST_RUN_ALL(); } diff --git a/staging_vespalib/src/tests/stllike/.gitignore b/staging_vespalib/src/tests/stllike/.gitignore new file mode 100644 index 00000000000..268b9e88393 --- /dev/null +++ b/staging_vespalib/src/tests/stllike/.gitignore @@ -0,0 +1,6 @@ +.depend +Makefile +lrucache_test +cache_test +staging_vespalib_cache_test_app +staging_vespalib_lrucache_test_app diff --git a/staging_vespalib/src/tests/stllike/CMakeLists.txt b/staging_vespalib/src/tests/stllike/CMakeLists.txt new file mode 100644 index 00000000000..690582676ad --- /dev/null +++ b/staging_vespalib/src/tests/stllike/CMakeLists.txt @@ -0,0 +1,15 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_lrucache_test_app + SOURCES + lrucache.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_lrucache_test_app COMMAND staging_vespalib_lrucache_test_app) +vespa_add_executable(staging_vespalib_cache_test_app + SOURCES + cache_test.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_cache_test_app COMMAND staging_vespalib_cache_test_app) diff --git a/staging_vespalib/src/tests/stllike/DESC b/staging_vespalib/src/tests/stllike/DESC new file mode 100644 index 00000000000..ec13a79916b --- /dev/null +++ b/staging_vespalib/src/tests/stllike/DESC @@ -0,0 +1 @@ +hash test. Take a look at hash.cpp for details. diff --git a/staging_vespalib/src/tests/stllike/FILES b/staging_vespalib/src/tests/stllike/FILES new file mode 100644 index 00000000000..eb3794b4a28 --- /dev/null +++ b/staging_vespalib/src/tests/stllike/FILES @@ -0,0 +1,3 @@ +hash.cpp +uniq_by_sort_map_hash.cpp +lrucache.cpp diff --git a/staging_vespalib/src/tests/stllike/avl.cpp b/staging_vespalib/src/tests/stllike/avl.cpp new file mode 100644 index 00000000000..53df03319d7 --- /dev/null +++ b/staging_vespalib/src/tests/stllike/avl.cpp @@ -0,0 +1,265 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("memory_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespalib/stllike/avl_set.h> +#include <vespa/vespalib/stllike/avl_map.h> + +using namespace vespalib; +using std::make_pair; + +class Test : public TestApp +{ +public: + int Main(); + void testAvlTreeSet(); + void testAvlTreeSet2(); + void testAvlTreeMap(); + void testAvlTreeFind(); +}; + +int +Test::Main() +{ + TEST_INIT("avl_test"); + testAvlTreeSet(); + testAvlTreeSet2(); + testAvlTreeMap(); + testAvlTreeFind(); + TEST_DONE(); +} + +namespace { + struct Foo { + int i; + + Foo() : i(0) {} + Foo(int i_) : i(i_) {} + + bool operator<(const Foo& f) const + { return (i < f.i); } + friend std::ostream & operator << (std::ostream & os, const Foo & f) { return os << f.i; } + }; +} + +void Test::testAvlTreeSet2() +{ + const size_t testSize(2000); + avl_set<Foo> set(100); + // Verfify start conditions. + EXPECT_TRUE(set.size() == 0); + EXPECT_TRUE(set.begin() == set.end()); + EXPECT_TRUE(set.find(7) == set.end()); + // Insert one element + set.insert(Foo(7)); + EXPECT_TRUE(set.size() == 1); + EXPECT_TRUE(set.begin() != set.end()); + EXPECT_TRUE(set.find(Foo(7)) != set.end()); + EXPECT_TRUE(*set.find(Foo(7)) == Foo(7)); + EXPECT_TRUE(set.find(Foo(8)) == set.end()); + // erase non existing + set.erase(Foo(8)); + EXPECT_TRUE(set.size() == 1); + EXPECT_TRUE(set.begin() != set.end()); + EXPECT_TRUE(set.find(Foo(7)) != set.end()); + EXPECT_TRUE(*set.find(Foo(7)) == Foo(7)); + EXPECT_TRUE(set.find(Foo(8)) == set.end()); + // erase existing + set.erase(Foo(7)); + EXPECT_TRUE(set.size() == 0); + EXPECT_TRUE(set.begin() == set.end()); + EXPECT_TRUE(set.find(Foo(7)) == set.end()); + for (size_t i(0); i < testSize; i++) { + set.insert(Foo(i)); + avl_set<Foo, Foo::avl>::iterator it = set.find(Foo(i)); + ASSERT_TRUE(it != set.end()); + for (size_t j=0; j < i; j++) { + it = set.find(Foo(j)); + ASSERT_TRUE(it != set.end()); + } + } + EXPECT_TRUE(set.size() == testSize); + avl_set<Foo, Foo::avl>::iterator it = set.find(Foo((testSize/2)-1)); + ASSERT_TRUE(it != set.end()); + EXPECT_EQUAL(*it, Foo((testSize/2)-1)); + for (size_t i(0); i < testSize/2; i++) { + set.erase(Foo(i*2)); + } + ASSERT_TRUE(it != set.end()); + EXPECT_EQUAL(*it, Foo((testSize/2)-1)); + EXPECT_TRUE(set.find(Foo(testSize/2)) == set.end()); + EXPECT_TRUE(set.size() == testSize/2); + for (size_t i(0); i < testSize; i++) { + set.insert(Foo(i)); + } + EXPECT_EQUAL(set.size(), testSize); + EXPECT_TRUE(*set.find(Foo(7)) == Foo(7)); + EXPECT_TRUE(*set.find(Foo(0)) == Foo(0)); + EXPECT_TRUE(*set.find(Foo(1)) == Foo(1)); + EXPECT_TRUE(*set.find(Foo(testSize-1)) == Foo(testSize-1)); + EXPECT_TRUE(set.find(Foo(testSize)) == set.end()); + + set.clear(); + + EXPECT_EQUAL(set.size(), 0u); + EXPECT_TRUE(set.find(Foo(7)) == set.end()); +} + +void Test::testAvlTreeSet() +{ + avl_set<int> set(1000); + // Verfify start conditions. + EXPECT_TRUE(set.size() == 0); + EXPECT_TRUE(set.begin() == set.end()); + EXPECT_TRUE(set.find(7) == set.end()); + // Insert one element + set.insert(7); + EXPECT_TRUE(set.size() == 1); + EXPECT_TRUE(set.begin() != set.end()); + EXPECT_TRUE(set.find(7) != set.end()); + EXPECT_TRUE(*set.find(7) == 7); + EXPECT_TRUE(set.find(8) == set.end()); + // erase non existing + set.erase(8); + EXPECT_TRUE(set.size() == 1); + EXPECT_TRUE(set.begin() != set.end()); + EXPECT_TRUE(set.find(7) != set.end()); + EXPECT_TRUE(*set.find(7) == 7); + EXPECT_TRUE(set.find(8) == set.end()); + // erase existing + set.erase(7); + EXPECT_TRUE(set.size() == 0); + EXPECT_TRUE(set.begin() == set.end()); + EXPECT_TRUE(set.find(7) == set.end()); + for (size_t i(0); i < 10000; i++) { + set.insert(i); + } + EXPECT_TRUE(set.size() == 10000); + for (size_t i(0); i < 5000; i++) { + set.erase(i*2); + } + EXPECT_TRUE(*set.find(4999) == 4999); + EXPECT_TRUE(set.find(5000) == set.end()); + EXPECT_TRUE(set.size() == 5000); + for (size_t i(0); i < 10000; i++) { + set.insert(i); + } + EXPECT_EQUAL(set.size(), 10000u); + EXPECT_TRUE(*set.find(7) == 7); + EXPECT_TRUE(*set.find(0) == 0); + EXPECT_TRUE(*set.find(1) == 1); + EXPECT_TRUE(*set.find(9999) == 9999); + EXPECT_TRUE(set.find(10000) == set.end()); + + set.clear(); + + EXPECT_EQUAL(set.size(), 0u); + EXPECT_TRUE(set.find(7) == set.end()); +} + +void Test::testAvlTreeMap() +{ + avl_map<int, int> set(1000); + // Verfify start conditions. + EXPECT_TRUE(set.size() == 0); + EXPECT_TRUE(set.begin() == set.end()); + EXPECT_TRUE(set.find(7) == set.end()); + // Insert one element + set.insert(make_pair(7, 70)); + EXPECT_TRUE(set.size() == 1); + EXPECT_TRUE(set.begin() != set.end()); + EXPECT_TRUE(set.find(7) != set.end()); + EXPECT_TRUE(set.find(7)->first == 7); + EXPECT_TRUE(set.find(7)->second == 70); + EXPECT_TRUE(set.find(8) == set.end()); + // erase non existing + set.erase(8); + EXPECT_TRUE(set.size() == 1); + EXPECT_TRUE(set.begin() != set.end()); + EXPECT_TRUE(set.find(7) != set.end()); + EXPECT_TRUE(set.find(7)->first == 7); + EXPECT_TRUE(set.find(7)->second == 70); + EXPECT_TRUE(set.find(8) == set.end()); + // erase existing + set.erase(7); + EXPECT_TRUE(set.size() == 0); + EXPECT_TRUE(set.begin() == set.end()); + EXPECT_TRUE(set.find(7) == set.end()); + for (size_t i(0); i < 10000; i++) { + set.insert(make_pair(i,i*10)); + } + EXPECT_TRUE(set.size() == 10000); + for (size_t i(0); i < 5000; i++) { + set.erase(i*2); + } + EXPECT_TRUE(set.find(4999)->first == 4999); + EXPECT_TRUE(set.find(4999)->second == 49990); + EXPECT_TRUE(set.find(5000) == set.end()); + EXPECT_TRUE(set.size() == 5000); + for (size_t i(0); i < 10000; i++) { + set.insert(make_pair(i,i*10)); + } + EXPECT_EQUAL(set.size(), 10000u); + EXPECT_TRUE(set.find(7)->first == 7); + EXPECT_TRUE(set.find(7)->second == 70); + EXPECT_TRUE(set.find(0)->first == 0); + EXPECT_TRUE(set.find(0)->second == 0); + EXPECT_TRUE(set.find(1)->first == 1); + EXPECT_TRUE(set.find(1)->second == 10); + EXPECT_TRUE(set.find(9999)->first == 9999); + EXPECT_TRUE(set.find(9999)->second == 99990); + EXPECT_TRUE(set.find(10000) == set.end()); + + avl_map<int, int> set2(7); + set.swap(set2); + EXPECT_EQUAL(set2.size(), 10000u); + EXPECT_TRUE(set2.find(7)->first == 7); + EXPECT_TRUE(set2.find(7)->second == 70); + + EXPECT_EQUAL(set.size(), 0u); + EXPECT_TRUE(set.find(7) == set.end()); + for (int i=0; i < 100; i++) { + set.insert(make_pair(i,i*10)); + } + for (int i=0; i < 100; i++) { + EXPECT_TRUE(set.find(i)->second == i*10); + } + + avl_map<int, int> set3; + set3.insert(set.begin(), set.end()); + for (int i=0; i < 100; i++) { + EXPECT_EQUAL(i*10, set.find(i)->second); + } +} + +class S { +public: + explicit S(uint64_t l=0) : _a(l&0xfffffffful), _b(l>>32) { } + uint32_t avl() const { return _a; } + uint32_t a() const { return _a; } + friend bool operator == (const S & a, const S & b) { return a._a == b._a && a._b == b._b; } +private: + uint32_t _a, _b; +}; + +struct myavl { + size_t operator() (const S & arg) const { return arg.avl(); } +}; + +struct myextract { + uint32_t operator() (const S & arg) const { return arg.a(); } +}; + +void Test::testAvlTreeFind() +{ + avl_set<S, myavl> set(1000); + for (size_t i(0); i < 10000; i++) { + set.insert(S(i)); + } + EXPECT_TRUE(*set.find(S(1)) == S(1)); + avl_set<S, myavl>::iterator cit = set.find<uint32_t, myextract, vespalib::avl<uint32_t>, std::equal_to<uint32_t> >(7); + EXPECT_TRUE(*cit == S(7)); +} + +TEST_APPHOOK(Test) diff --git a/staging_vespalib/src/tests/stllike/cache_test.cpp b/staging_vespalib/src/tests/stllike/cache_test.cpp new file mode 100644 index 00000000000..288ef367be4 --- /dev/null +++ b/staging_vespalib/src/tests/stllike/cache_test.cpp @@ -0,0 +1,165 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("cache_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/stllike/cache.h> +#include <map> + +using namespace vespalib; + +template<typename K, typename V> +class Map : public std::map<K, V> { + typedef typename std::map<K, V>::const_iterator const_iterator; + typedef std::map<K, V> M; +public: + bool read(const K & k, V & v) const { + const_iterator found = M::find(k); + bool ok(found != this->end()); + if (ok) { + v = found->second; + } + return ok; + } + void write(const K & k, const V & v) { + (*this)[k] = v; + } + void erase(const K & k) { + M::erase(k); + } +}; + +class Test : public TestApp +{ +public: + int Main(); +private: + typedef LruParam<uint32_t, string> P; + typedef Map<uint32_t, string> B; + void testCache(); + void testCacheSize(); + void testCacheSizeDeep(); + void testCacheEntriesHonoured(); + void testCacheMaxSizeHonoured(); + void testThatMultipleRemoveOnOverflowIsFine(); +}; + +int +Test::Main() +{ + TEST_INIT("cache_test"); + testCache(); + testCacheSize(); + testCacheSizeDeep(); + testCacheEntriesHonoured(); + testCacheMaxSizeHonoured(); + testThatMultipleRemoveOnOverflowIsFine(); + TEST_DONE(); +} + +void Test::testCache() +{ + B m; + cache< CacheParam<P, B> > cache(m, -1); + // Verfify start conditions. + EXPECT_TRUE(cache.size() == 0); + EXPECT_TRUE( ! cache.hasKey(1) ); + cache.write(1, "First inserted string"); + EXPECT_TRUE( cache.hasKey(1) ); + m[2] = "String inserted beneath"; + EXPECT_TRUE( ! cache.hasKey(2) ); + EXPECT_EQUAL( cache.read(2), "String inserted beneath"); + EXPECT_TRUE( cache.hasKey(2) ); + cache.erase(1); + EXPECT_TRUE( ! cache.hasKey(1) ); + EXPECT_TRUE(cache.size() == 1); +} + +void Test::testCacheSize() +{ + B m; + cache< CacheParam<P, B> > cache(m, -1); + cache.write(1, "10 bytes string"); + EXPECT_EQUAL(80u, cache.sizeBytes()); +} + +void Test::testCacheSizeDeep() +{ + B m; + cache< CacheParam<P, B, zero<uint32_t>, size<string> > > cache(m, -1); + cache.write(1, "15 bytes string"); + EXPECT_EQUAL(95u, cache.sizeBytes()); +} + +void Test::testCacheEntriesHonoured() +{ + B m; + cache< CacheParam<P, B, zero<uint32_t>, size<string> > > cache(m, -1); + cache.maxElements(1); + cache.write(1, "15 bytes string"); + EXPECT_EQUAL(1u, cache.size()); + EXPECT_EQUAL(95u, cache.sizeBytes()); + cache.write(2, "16 bytes stringg"); + EXPECT_EQUAL(1u, cache.size()); + EXPECT_TRUE( cache.hasKey(2) ); + EXPECT_FALSE( cache.hasKey(1) ); + EXPECT_EQUAL(96u, cache.sizeBytes()); +} + +void Test::testCacheMaxSizeHonoured() +{ + B m; + cache< CacheParam<P, B, zero<uint32_t>, size<string> > > cache(m, 200); + cache.write(1, "15 bytes string"); + EXPECT_EQUAL(1u, cache.size()); + EXPECT_EQUAL(95u, cache.sizeBytes()); + cache.write(2, "16 bytes stringg"); + EXPECT_EQUAL(2u, cache.size()); + EXPECT_EQUAL(191u, cache.sizeBytes()); + cache.write(3, "17 bytes stringgg"); + EXPECT_EQUAL(3u, cache.size()); + EXPECT_EQUAL(288u, cache.sizeBytes()); + cache.write(4, "18 bytes stringggg"); + EXPECT_EQUAL(3u, cache.size()); + EXPECT_EQUAL(291u, cache.sizeBytes()); +} + +void Test::testThatMultipleRemoveOnOverflowIsFine() +{ + B m; + cache< CacheParam<P, B, zero<uint32_t>, size<string> > > cache(m, 2000); + + for (size_t j(0); j < 5; j++) { + for (size_t i(0); cache.size() == i; i++) { + cache.write(j*53+i, "a"); + } + } + EXPECT_EQUAL(25u, cache.size()); + EXPECT_EQUAL(2025u, cache.sizeBytes()); + EXPECT_FALSE( cache.hasKey(0) ); + string ls("long string aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + string vls=ls+ls+ls+ls+ls+ls; + cache.write(53+5, ls); + EXPECT_EQUAL(25u, cache.size()); + EXPECT_EQUAL(2498u, cache.sizeBytes()); + EXPECT_FALSE( cache.hasKey(1) ); + cache.write(53*7+5, ls); + EXPECT_EQUAL(19u, cache.size()); + EXPECT_EQUAL(2485u, cache.sizeBytes()); + EXPECT_FALSE( cache.hasKey(2) ); + cache.write(53*8+5, vls); + EXPECT_EQUAL(14u, cache.size()); + EXPECT_EQUAL(4923u, cache.sizeBytes()); + cache.write(53*9+6, vls); + EXPECT_EQUAL(1u, cache.size()); + EXPECT_EQUAL(2924u, cache.sizeBytes()); +} + + +TEST_APPHOOK(Test) diff --git a/staging_vespalib/src/tests/stllike/lrucache.cpp b/staging_vespalib/src/tests/stllike/lrucache.cpp new file mode 100644 index 00000000000..de5bcb8a365 --- /dev/null +++ b/staging_vespalib/src/tests/stllike/lrucache.cpp @@ -0,0 +1,192 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("lrucache_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/stllike/lrucache_map.h> + +using namespace vespalib; + +TEST("testCache") { + lrucache_map< LruParam<int, string> > cache(7); + // Verfify start conditions. + EXPECT_TRUE(cache.size() == 0); + cache.insert(1, "First inserted string"); + EXPECT_TRUE(cache.verifyInternals()); + EXPECT_TRUE(cache.size() == 1); + EXPECT_TRUE(cache.hasKey(1)); + cache.insert(2, "Second inserted string"); + EXPECT_TRUE(cache.verifyInternals()); + EXPECT_TRUE(cache.size() == 2); + EXPECT_TRUE(cache.hasKey(1)); + EXPECT_TRUE(cache.hasKey(2)); + cache.insert(3, "Third inserted string"); + EXPECT_TRUE(cache.verifyInternals()); + EXPECT_TRUE(cache.size() == 3); + EXPECT_TRUE(cache.hasKey(1)); + EXPECT_TRUE(cache.hasKey(2)); + EXPECT_TRUE(cache.hasKey(3)); + cache.insert(4, "Fourth inserted string"); + EXPECT_TRUE(cache.verifyInternals()); + EXPECT_TRUE(cache.size() == 4); + EXPECT_TRUE(cache.hasKey(1)); + EXPECT_TRUE(cache.hasKey(2)); + EXPECT_TRUE(cache.hasKey(3)); + EXPECT_TRUE(cache.hasKey(4)); + cache.insert(5, "Fifth inserted string"); + EXPECT_TRUE(cache.verifyInternals()); + EXPECT_TRUE(cache.size() == 5); + EXPECT_TRUE(cache.hasKey(1)); + EXPECT_TRUE(cache.hasKey(2)); + EXPECT_TRUE(cache.hasKey(3)); + EXPECT_TRUE(cache.hasKey(4)); + EXPECT_TRUE(cache.hasKey(5)); + cache.insert(6, "Sixt inserted string"); + EXPECT_TRUE(cache.verifyInternals()); + EXPECT_TRUE(cache.size() == 6); + EXPECT_TRUE(cache.hasKey(1)); + EXPECT_TRUE(cache.hasKey(2)); + EXPECT_TRUE(cache.hasKey(3)); + EXPECT_TRUE(cache.hasKey(4)); + EXPECT_TRUE(cache.hasKey(5)); + EXPECT_TRUE(cache.hasKey(6)); + cache.insert(7, "Seventh inserted string"); + EXPECT_TRUE(cache.verifyInternals()); + EXPECT_EQUAL(cache.size(), 7u); + EXPECT_TRUE(cache.hasKey(1)); + EXPECT_TRUE(cache.hasKey(2)); + EXPECT_TRUE(cache.hasKey(3)); + EXPECT_TRUE(cache.hasKey(4)); + EXPECT_TRUE(cache.hasKey(5)); + EXPECT_TRUE(cache.hasKey(6)); + EXPECT_TRUE(cache.hasKey(7)); + cache.insert(8, "Eighth inserted string"); + EXPECT_TRUE(cache.verifyInternals()); + EXPECT_EQUAL(cache.size(), 7u); + EXPECT_TRUE(cache.hasKey(2)); + EXPECT_TRUE(cache.hasKey(3)); + EXPECT_TRUE(cache.hasKey(4)); + EXPECT_TRUE(cache.hasKey(5)); + EXPECT_TRUE(cache.hasKey(6)); + EXPECT_TRUE(cache.hasKey(7)); + EXPECT_TRUE(cache.hasKey(8)); + cache.insert(15, "Eighth inserted string"); + EXPECT_TRUE(cache.verifyInternals()); + EXPECT_EQUAL(cache.size(), 7u); + EXPECT_TRUE(cache.hasKey(3)); + EXPECT_TRUE(cache.hasKey(4)); + EXPECT_TRUE(cache.hasKey(5)); + EXPECT_TRUE(cache.hasKey(6)); + EXPECT_TRUE(cache.hasKey(7)); + EXPECT_TRUE(cache.hasKey(8)); + EXPECT_TRUE(cache.hasKey(15)); + // Test get and erase + cache.get(3); + EXPECT_TRUE(cache.verifyInternals()); + cache.erase(3); + EXPECT_TRUE(cache.verifyInternals()); + EXPECT_TRUE(!cache.hasKey(3)); +} + +typedef std::shared_ptr<std::string> MyKey; +typedef std::shared_ptr<std::string> MyData; + +struct SharedEqual : public std::binary_function<MyKey, MyKey, bool> { + bool operator()(const MyKey & a, const MyKey & b) { + return ((*a) == (*b)); + } +}; + +struct SharedHash { + size_t operator() (const MyKey & arg) const { return arg->size(); } +}; + + +TEST("testCacheInsertOverResize") { + typedef vespalib::LinkedPtr<std::string> LS; + typedef lrucache_map< LruParam<int, LS> > Cache; + + Cache cache(100); + size_t sum(0); + for (size_t i(0); i < cache.capacity()*10; i++) { + LS s(new std::string("abc")); + cache[random()] = s; + sum += strlen(s->c_str()); + EXPECT_EQUAL(strlen(s->c_str()), s->size()); + } + EXPECT_EQUAL(sum, cache.capacity()*10*3); +} + +TEST("testCacheErase") { + lrucache_map< LruParam<MyKey, MyData, SharedHash, SharedEqual> > cache(4); + + MyData d(new std::string("foo")); + MyKey k(new std::string("barlol")); + // Verfify start conditions. + EXPECT_TRUE(cache.size() == 0); + EXPECT_TRUE(d.use_count() == 1); + EXPECT_TRUE(k.use_count() == 1); + cache.insert(k, d); + EXPECT_TRUE(d.use_count() == 2); + EXPECT_TRUE(k.use_count() == 2); + cache.erase(k); + EXPECT_TRUE(d.use_count() == 1); + EXPECT_TRUE(k.use_count() == 1); +} + +TEST("testCacheIterator") { + typedef lrucache_map< LruParam<int, string> > Cache; + Cache cache(3); + cache.insert(1, "first"); + cache.insert(2, "second"); + cache.insert(3, "third"); + Cache::iterator it(cache.begin()); + Cache::iterator mt(cache.end()); + ASSERT_TRUE(it != mt); + ASSERT_EQUAL("third", *it); + ASSERT_TRUE(it != mt); + ASSERT_EQUAL("second", *(++it)); + ASSERT_TRUE(it != mt); + ASSERT_EQUAL("second", *it++); + ASSERT_TRUE(it != mt); + ASSERT_EQUAL("first", *it); + ASSERT_TRUE(it != mt); + it++; + ASSERT_TRUE(it == mt); + cache.insert(4, "fourth"); + Cache::iterator it2(cache.begin()); + Cache::iterator it3(cache.begin()); + ASSERT_EQUAL("fourth", *it2); + ASSERT_TRUE(it2 == it3); + it2++; + ASSERT_TRUE(it2 != it3); + it2++; + it2++; + ASSERT_TRUE(it2 == mt); + Cache::iterator it4 = cache.erase(it3); + ASSERT_EQUAL("third", *it4); + ASSERT_EQUAL("third", *cache.begin()); + Cache::iterator it5(cache.erase(cache.end())); + ASSERT_TRUE(it5 == cache.end()); +} + +TEST("testCacheIteratorErase") { + typedef lrucache_map< LruParam<int, string> > Cache; + Cache cache(3); + cache.insert(1, "first"); + cache.insert(8, "second"); + cache.insert(15, "third"); + cache.insert(15, "third"); + cache.insert(8, "second"); + cache.insert(1, "first"); + Cache::iterator it(cache.begin()); + ASSERT_EQUAL("first", *it); + it++; + ASSERT_EQUAL("second", *it); + it = cache.erase(it); + ASSERT_EQUAL("third", *it); + cache.erase(it); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/staging_vespalib/src/tests/timer/.gitignore b/staging_vespalib/src/tests/timer/.gitignore new file mode 100644 index 00000000000..9031e40152a --- /dev/null +++ b/staging_vespalib/src/tests/timer/.gitignore @@ -0,0 +1 @@ +staging_vespalib_timer_test_app diff --git a/staging_vespalib/src/tests/timer/CMakeLists.txt b/staging_vespalib/src/tests/timer/CMakeLists.txt new file mode 100644 index 00000000000..ab16cc236c4 --- /dev/null +++ b/staging_vespalib/src/tests/timer/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_timer_test_app + SOURCES + timer_test.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_timer_test_app COMMAND staging_vespalib_timer_test_app) diff --git a/staging_vespalib/src/tests/timer/timer_test.cpp b/staging_vespalib/src/tests/timer/timer_test.cpp new file mode 100644 index 00000000000..de2f081f48a --- /dev/null +++ b/staging_vespalib/src/tests/timer/timer_test.cpp @@ -0,0 +1,60 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("timer_test"); +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/timer.h> +#include <vespa/vespalib/util/executor.h> + +using namespace vespalib; +using vespalib::Executor; +typedef Executor::Task Task; + +class Test : public TestApp +{ +public: + int Main(); + void testScheduling(); + void testReset(); +}; + +class TestTask : public Task { +private: + vespalib::CountDownLatch &_latch; +public: + TestTask(vespalib::CountDownLatch & latch) : _latch(latch) { } + void run() { _latch.countDown(); } +}; + +int +Test::Main() +{ + TEST_INIT("timer_test"); + testScheduling(); + testReset(); + TEST_DONE(); +} + +void Test::testScheduling() +{ + vespalib::CountDownLatch latch1(3); + vespalib::CountDownLatch latch2(2); + Timer timer; + timer.scheduleAtFixedRate(Task::UP(new TestTask(latch1)), 0.1, 0.2); + timer.scheduleAtFixedRate(Task::UP(new TestTask(latch2)), 0.5, 0.5); + EXPECT_TRUE(latch1.await(60000)); + EXPECT_TRUE(latch2.await(60000)); +} + +void Test::testReset() +{ + vespalib::CountDownLatch latch1(2); + Timer timer; + timer.scheduleAtFixedRate(Task::UP(new TestTask(latch1)), 2.0, 3.0); + timer.reset(); + EXPECT_TRUE(!latch1.await(3000)); + timer.scheduleAtFixedRate(Task::UP(new TestTask(latch1)), 0.2, 0.3); + EXPECT_TRUE(latch1.await(60000)); +} + +TEST_APPHOOK(Test) diff --git a/staging_vespalib/src/tests/trace/.gitignore b/staging_vespalib/src/tests/trace/.gitignore new file mode 100644 index 00000000000..14ed55a124e --- /dev/null +++ b/staging_vespalib/src/tests/trace/.gitignore @@ -0,0 +1 @@ +staging_vespalib_trace_test_app diff --git a/staging_vespalib/src/tests/trace/CMakeLists.txt b/staging_vespalib/src/tests/trace/CMakeLists.txt new file mode 100644 index 00000000000..cb381733bff --- /dev/null +++ b/staging_vespalib/src/tests/trace/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_trace_test_app + SOURCES + trace.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_trace_test_app COMMAND staging_vespalib_trace_test_app) diff --git a/staging_vespalib/src/tests/trace/trace.cpp b/staging_vespalib/src/tests/trace/trace.cpp new file mode 100644 index 00000000000..a7896d43338 --- /dev/null +++ b/staging_vespalib/src/tests/trace/trace.cpp @@ -0,0 +1,110 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP("trace_test"); +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/trace/tracenode.h> +#include <vespa/vespalib/trace/slime_trace_serializer.h> +#include <vespa/vespalib/trace/slime_trace_deserializer.h> + +using namespace vespalib; +using namespace vespalib::slime; + +TEST("that a single trace node is serialized") { + TraceNode node; + Slime slime; + SlimeTraceSerializer serializer(slime.setObject()); + node.accept(serializer); + Inspector & i(slime.get()); + EXPECT_TRUE(i["timestamp"].valid()); + EXPECT_EQUAL(0l, i["timestamp"].asLong()); + EXPECT_FALSE(i["payload"].valid()); +} + +TEST("that a trace node with children is serialized") { + TraceNode node; + node.addChild("foo", 1234); + node.addChild("bar", 1235); + Slime slime; + SlimeTraceSerializer serializer(slime.setObject()); + node.accept(serializer); + Inspector & i(slime.get()); + EXPECT_TRUE(i["timestamp"].valid()); + EXPECT_EQUAL(0l, i["timestamp"].asLong()); + EXPECT_TRUE(i["children"].valid()); + Inspector & iBar(i["children"][0]); + Inspector & iFoo(i["children"][1]); + EXPECT_TRUE(iFoo.valid()); + EXPECT_TRUE(iBar.valid()); + EXPECT_EQUAL(1234, iFoo["timestamp"].asLong()); + EXPECT_EQUAL("foo", iFoo["payload"].asString().make_string()); + EXPECT_EQUAL(1235, iBar["timestamp"].asLong()); + EXPECT_EQUAL("bar", iBar["payload"].asString().make_string()); +} + +TEST("that an empty root trace node can be deserialized") { + Slime slime; + Cursor & root(slime.setObject()); + SlimeTraceDeserializer deserializer(root); + TraceNode node(deserializer.deserialize()); + EXPECT_FALSE(node.hasNote()); + EXPECT_EQUAL(0, node.getTimestamp()); +} + + +TEST("that a single trace node can be deserialized") { + Slime slime; + Cursor & root(slime.setObject()); + root.setLong("timestamp", 1234); + root.setString("payload", "hello"); + SlimeTraceDeserializer deserializer(root); + TraceNode node(deserializer.deserialize()); + EXPECT_EQUAL(1234, node.getTimestamp()); + EXPECT_TRUE(node.hasNote()); + EXPECT_EQUAL("hello", node.getNote()); +} + +TEST("that a trace node with children can be deserialized") { + Slime slime; + Cursor & root(slime.setObject()); + Cursor & rootChildren(root.setArray("children")); + Cursor & foo(rootChildren.addObject()); + foo.setLong("timestamp", 123); + Cursor &fooArray(foo.setArray("children")); + Cursor &foobar(fooArray.addObject()); + foobar.setLong("timestamp", 45); + foobar.setString("payload", "world"); + Cursor & bar(rootChildren.addObject()); + bar.setLong("timestamp", 67); + bar.setString("payload", "!"); + + vespalib::slime::SimpleBuffer buf; + vespalib::slime::JsonFormat::encode(slime, buf, false); + + SlimeTraceDeserializer deserializer(root); + TraceNode node(deserializer.deserialize()); + EXPECT_FALSE(node.hasNote()); + ASSERT_EQUAL(2u, node.getNumChildren()); + TraceNode fooNode(node.getChild(0)); + ASSERT_EQUAL(1u, fooNode.getNumChildren()); + TraceNode fooBarNode(fooNode.getChild(0)); + EXPECT_EQUAL("world", fooBarNode.getNote()); + TraceNode barNode(node.getChild(1)); + EXPECT_EQUAL("!", barNode.getNote()); + ASSERT_EQUAL(0u, barNode.getNumChildren()); +} + +TEST("test serialization and deserialization") { + TraceNode root; + root.addChild("foo", 45); + root.addChild("bar"); + root.addChild(TraceNode()); + Slime slime; + SlimeTraceSerializer s(slime.setObject()); + root.accept(s); + SlimeTraceDeserializer d(slime.get()); + TraceNode root2(d.deserialize()); + ASSERT_EQUAL(3u, root2.getNumChildren()); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/staging_vespalib/src/tests/util/process_memory_stats/.gitignore b/staging_vespalib/src/tests/util/process_memory_stats/.gitignore new file mode 100644 index 00000000000..e32e9f22dd3 --- /dev/null +++ b/staging_vespalib/src/tests/util/process_memory_stats/.gitignore @@ -0,0 +1,2 @@ +mapfile +staging_vespalib_process_memory_stats_test_app diff --git a/staging_vespalib/src/tests/util/process_memory_stats/CMakeLists.txt b/staging_vespalib/src/tests/util/process_memory_stats/CMakeLists.txt new file mode 100644 index 00000000000..a9c23944575 --- /dev/null +++ b/staging_vespalib/src/tests/util/process_memory_stats/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_process_memory_stats_test_app + SOURCES + process_memory_stats_test.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_process_memory_stats_test_app COMMAND sh process_memory_stats_test.sh) diff --git a/staging_vespalib/src/tests/util/process_memory_stats/DESC b/staging_vespalib/src/tests/util/process_memory_stats/DESC new file mode 100644 index 00000000000..3f2b5fb571e --- /dev/null +++ b/staging_vespalib/src/tests/util/process_memory_stats/DESC @@ -0,0 +1 @@ +ProcessMemoryStats test. Take a look at process_memory_stats_test.cpp for details. diff --git a/staging_vespalib/src/tests/util/process_memory_stats/FILES b/staging_vespalib/src/tests/util/process_memory_stats/FILES new file mode 100644 index 00000000000..18ccce970a4 --- /dev/null +++ b/staging_vespalib/src/tests/util/process_memory_stats/FILES @@ -0,0 +1 @@ +process_memory_stats_test.cpp diff --git a/staging_vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.cpp b/staging_vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.cpp new file mode 100644 index 00000000000..43288d630cf --- /dev/null +++ b/staging_vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.cpp @@ -0,0 +1,79 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/util/process_memory_stats.h> +#include <iostream> +#include <fstream> + + +using namespace vespalib; + +namespace { + +std::string toString(const ProcessMemoryStats &stats) +{ + std::ostringstream os; + os << "Mapped(" + << stats.getMappedVirt() << "," << stats.getMappedRss() << + "), Anonymous(" + << stats.getAnonymousVirt() << "," << stats.getAnonymousRss() << ")"; + return os.str(); +} + +} + +TEST("Simple stats") +{ + ProcessMemoryStats stats(ProcessMemoryStats::create()); + std::cout << toString(stats) << std::endl; + EXPECT_LESS(0u, stats.getMappedVirt()); + EXPECT_LESS(0u, stats.getMappedRss()); + EXPECT_LESS(0u, stats.getAnonymousVirt()); + EXPECT_LESS(0u, stats.getAnonymousRss()); +} + +TEST("grow anonymous memory") +{ + ProcessMemoryStats stats1(ProcessMemoryStats::create()); + std::cout << toString(stats1) << std::endl; + size_t mapLen = 64 * 1024; + void *mapAddr = mmap(nullptr, mapLen, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + EXPECT_NOT_EQUAL(reinterpret_cast<void *>(-1), mapAddr); + ProcessMemoryStats stats2(ProcessMemoryStats::create()); + std::cout << toString(stats2) << std::endl; + EXPECT_LESS_EQUAL(stats1.getAnonymousVirt() + mapLen, + stats2.getAnonymousVirt()); + memset(mapAddr, 1, mapLen); + ProcessMemoryStats stats3(ProcessMemoryStats::create()); + std::cout << toString(stats3) << std::endl; + // Cannot check that resident grows if swap is enabled and system loaded + munmap(mapAddr, mapLen); +} + +TEST("grow mapped memory") +{ + std::ofstream of("mapfile"); + size_t mapLen = 64 * 1024; + std::vector<char> buf(mapLen, 4); + of.write(&buf[0], buf.size()); + of.close(); + int mapfileFileDescriptor = open("mapfile", O_RDONLY, 0666); + EXPECT_LESS_EQUAL(0, mapfileFileDescriptor); + ProcessMemoryStats stats1(ProcessMemoryStats::create()); + std::cout << toString(stats1) << std::endl; + void *mapAddr = mmap(nullptr, mapLen, PROT_READ, MAP_SHARED, + mapfileFileDescriptor, 0); + EXPECT_NOT_EQUAL(reinterpret_cast<void *>(-1), mapAddr); + ProcessMemoryStats stats2(ProcessMemoryStats::create()); + std::cout << toString(stats2) << std::endl; + EXPECT_LESS_EQUAL(stats1.getMappedVirt() + mapLen, stats2.getMappedVirt()); + EXPECT_EQUAL(0, memcmp(mapAddr, &buf[0], mapLen)); + ProcessMemoryStats stats3(ProcessMemoryStats::create()); + std::cout << toString(stats3) << std::endl; + // Cannot check that resident grows if swap is enabled and system loaded + munmap(mapAddr, mapLen); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/staging_vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.sh b/staging_vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.sh new file mode 100755 index 00000000000..57b6c8cb613 --- /dev/null +++ b/staging_vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.sh @@ -0,0 +1,4 @@ +#!/bin/sh +rm -f mapfile +$VALGRIND ./staging_vespalib_process_memory_stats_test_app +rm -f mapfile diff --git a/staging_vespalib/src/tests/xmlserializable/.gitignore b/staging_vespalib/src/tests/xmlserializable/.gitignore new file mode 100644 index 00000000000..d2910cb7407 --- /dev/null +++ b/staging_vespalib/src/tests/xmlserializable/.gitignore @@ -0,0 +1,4 @@ +*_test +.depend +Makefile +staging_vespalib_xmlserializable_test_app diff --git a/staging_vespalib/src/tests/xmlserializable/CMakeLists.txt b/staging_vespalib/src/tests/xmlserializable/CMakeLists.txt new file mode 100644 index 00000000000..9d351cec4d9 --- /dev/null +++ b/staging_vespalib/src/tests/xmlserializable/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(staging_vespalib_xmlserializable_test_app + SOURCES + xmlserializabletest.cpp + DEPENDS + staging_vespalib +) +vespa_add_test(NAME staging_vespalib_xmlserializable_test_app COMMAND staging_vespalib_xmlserializable_test_app) diff --git a/staging_vespalib/src/tests/xmlserializable/DESC b/staging_vespalib/src/tests/xmlserializable/DESC new file mode 100644 index 00000000000..7768b3e7b6d --- /dev/null +++ b/staging_vespalib/src/tests/xmlserializable/DESC @@ -0,0 +1 @@ +xmlserializable test. Take a look at xmlserializabletest.cpp for details. diff --git a/staging_vespalib/src/tests/xmlserializable/FILES b/staging_vespalib/src/tests/xmlserializable/FILES new file mode 100644 index 00000000000..e7c7d2397b7 --- /dev/null +++ b/staging_vespalib/src/tests/xmlserializable/FILES @@ -0,0 +1 @@ +xmlserializabletest.cpp diff --git a/staging_vespalib/src/tests/xmlserializable/xmlserializabletest.cpp b/staging_vespalib/src/tests/xmlserializable/xmlserializabletest.cpp new file mode 100644 index 00000000000..a44802735eb --- /dev/null +++ b/staging_vespalib/src/tests/xmlserializable/xmlserializabletest.cpp @@ -0,0 +1,165 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/util/xmlserializable.h> + +#include <vespa/vespalib/testkit/testapp.h> + +namespace vespalib { + +class Test : public vespalib::TestApp +{ +public: + void testNormalUsage(); + void testEscaping(); + void testNesting(); + void testIndent(); + + int Main(); +}; + +int +Test::Main() +{ + TEST_INIT("xmlserializables_test"); + srandom(1); + testNormalUsage(); + testEscaping(); + testNesting(); + testIndent(); + TEST_DONE(); +} + +void +Test::testNormalUsage() +{ + std::ostringstream ost; + XmlOutputStream xos(ost); + using namespace vespalib::xml; + xos << XmlTag("car") + << XmlTag("door") + << XmlAttribute("windowstate", "up") + << XmlEndTag() + << XmlTag("description") + << "This is a car description used to test" + << XmlEndTag() + << XmlEndTag(); + std::string expected = + "\n<car>\n" + "<door windowstate=\"up\"/>\n" + "<description>This is a car description used to test</description>\n" + "</car>"; + EXPECT_EQUAL(expected, "\n" + ost.str()); +} + +void +Test::testEscaping() +{ + std::ostringstream ost; + XmlOutputStream xos(ost); + using namespace vespalib::xml; + xos << XmlTag("!#trash%-", CONVERT_ILLEGAL_CHARACTERS) + << XmlTag("foo") + << XmlAttribute("bar", "<100%\" &\n>") + << XmlEndTag() + << XmlTag("escaped") + << XmlEscapedContent() + << XmlContentWrapper("<>&\"'% \r\n\t\f\0", 12) + << XmlEndTag() + << XmlTag("encoded") + << XmlBase64Content() + << XmlContentWrapper("<>&\"'% \t\f\0", 10) + << XmlEndTag() + << XmlTag("auto1") + << XmlContentWrapper("<>&\t\f\r\nfoo", 10) + << XmlEndTag() + << XmlTag("auto2") + << XmlContentWrapper("<>&\t\0\r\nfoo", 10) + << XmlEndTag() + << XmlEndTag(); + std::string expected = + "\n<__trash_->\n" + "<foo bar=\"<100%" & >\"/>\n" + "<escaped><>&\"'% \n	�</escaped>\n" + "<encoded binaryencoding=\"base64\">PD4mIiclIAkMAA==</encoded>\n" + "<auto1><>&	 \nfoo</auto1>\n" + "<auto2 binaryencoding=\"base64\">PD4mCQANCmZvbw==</auto2>\n" + "</__trash_->"; + EXPECT_EQUAL(expected, "\n" + ost.str()); +} + +namespace { + struct LookAndFeel : public XmlSerializable { + + LookAndFeel() {} + + void printXml(XmlOutputStream& out) const { + using namespace vespalib::xml; + out << XmlAttribute("color", "blue") + << XmlTag("other") + << XmlAttribute("count", 5) + << XmlTag("something") << "foo" << XmlEndTag() + << XmlTag("else") << "bar" << XmlEndTag() + << XmlEndTag(); + } + }; +} + +void +Test::testNesting() +{ + std::ostringstream ost; + XmlOutputStream xos(ost); + using namespace vespalib::xml; + xos << XmlTag("car") + << XmlTag("door") + << LookAndFeel() + << XmlEndTag() + << XmlTag("description") + << "This is a car description used to test" + << XmlEndTag() + << XmlEndTag(); + std::string expected = + "\n<car>\n" + "<door color=\"blue\">\n" + "<other count=\"5\">\n" + "<something>foo</something>\n" + "<else>bar</else>\n" + "</other>\n" + "</door>\n" + "<description>This is a car description used to test</description>\n" + "</car>"; + EXPECT_EQUAL(expected, "\n" + ost.str()); +} + +void +Test::testIndent() +{ + std::ostringstream ost; + XmlOutputStream xos(ost, " "); + using namespace vespalib::xml; + xos << XmlTag("foo") + << XmlTag("bar") << 2.14 << XmlEndTag() + << "Litt innhold" + << XmlTag("nytag") + << "Mer innhold" + << XmlTag("base") + << XmlBase64Content() << "foobar" + << XmlEndTag() + << XmlEndTag() + << XmlEndTag(); + std::string expected = "\n" + "<foo>\n" + " <bar>2.14</bar>\n" + " Litt innhold\n" + " <nytag>\n" + " Mer innhold\n" + " <base binaryencoding=\"base64\">Zm9vYmFy</base>\n" + " </nytag>\n" + "</foo>"; + EXPECT_EQUAL(expected, "\n" + ost.str()); +} + +} // vespalib + +TEST_APPHOOK(vespalib::Test) diff --git a/staging_vespalib/src/vespa/vespalib/.gitignore b/staging_vespalib/src/vespa/vespalib/.gitignore new file mode 100644 index 00000000000..fae8ae72fc9 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/.gitignore @@ -0,0 +1,3 @@ +*.so.* +.depend +Makefile diff --git a/staging_vespalib/src/vespa/vespalib/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/CMakeLists.txt new file mode 100644 index 00000000000..7bdcc9bbf64 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/CMakeLists.txt @@ -0,0 +1,14 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(staging_vespalib + SOURCES + $<TARGET_OBJECTS:staging_vespalib_vespalib_encoding> + $<TARGET_OBJECTS:staging_vespalib_vespalib_util> + $<TARGET_OBJECTS:staging_vespalib_vespalib_data> + $<TARGET_OBJECTS:staging_vespalib_vespalib_objects> + $<TARGET_OBJECTS:staging_vespalib_vespalib_stllike> + $<TARGET_OBJECTS:staging_vespalib_vespalib_net> + $<TARGET_OBJECTS:staging_vespalib_vespalib_trace> + $<TARGET_OBJECTS:staging_vespalib_vespalib_hwaccelrated> + INSTALL lib64 + DEPENDS +) diff --git a/staging_vespalib/src/vespa/vespalib/data/.gitignore b/staging_vespalib/src/vespa/vespalib/data/.gitignore new file mode 100644 index 00000000000..0b3af54ee50 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/data/.gitignore @@ -0,0 +1,3 @@ +*.So +.depend* +Makefile diff --git a/staging_vespalib/src/vespa/vespalib/data/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/data/CMakeLists.txt new file mode 100644 index 00000000000..f6950b2e2b5 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/data/CMakeLists.txt @@ -0,0 +1,7 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(staging_vespalib_vespalib_data OBJECT + SOURCES + databuffer.cpp + fileheader.cpp + DEPENDS +) diff --git a/staging_vespalib/src/vespa/vespalib/data/databuffer.cpp b/staging_vespalib/src/vespa/vespalib/data/databuffer.cpp new file mode 100644 index 00000000000..43b82aaf62b --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/data/databuffer.cpp @@ -0,0 +1,200 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "databuffer.h" +#include <vespa/log/log.h> +LOG_SETUP(".databuffer"); + +namespace vespalib { + +namespace { +size_t padbefore(size_t alignment, const char *buf) { + return (alignment - (size_t(buf) % alignment)) % alignment; +} +} + +template <typename T> +DataBufferT<T>::DataBufferT(size_t len, size_t alignment) + : _alignment(alignment), + _externalBuf(NULL), + _bufstart(NULL), + _bufend(NULL), + _datapt(NULL), + _freept(NULL), + _buffer() +{ + assert(_alignment > 0); + if (len > 0) { + // avoid very small buffers for performance reasons: + size_t bufsize = 256; + while (bufsize < len + (_alignment - 1)) { + bufsize *= 2; + } + T newBuf(bufsize); + _bufstart = static_cast<char *>(newBuf.get()); + _buffer.swap(newBuf); + + _datapt = _bufstart + padbefore(alignment, _bufstart); + _freept = _datapt; + _bufend = _bufstart + bufsize; + assert(_bufstart != NULL); + } +} + + +template <typename T> +void +DataBufferT<T>::moveFreeToData(size_t len) +{ + assert(getFreeLen() >= len); + _freept += len; +} + + +template <typename T> +void +DataBufferT<T>::moveDeadToData(size_t len) +{ + assert(getDeadLen() >= len); + _datapt -= len; + if (_bufstart != _externalBuf) { + assert(getDeadLen() >= padbefore(_alignment, _bufstart)); // Do not move ahead of alignment. + } +} + + +template <typename T> +void +DataBufferT<T>::moveDataToFree(size_t len) +{ + assert(getDataLen() >= len); + _freept -= len; +} + + +template <typename T> +bool +DataBufferT<T>::shrink(size_t newsize) +{ + if (getBufSize() <= newsize || getDataLen() > newsize) { + return false; + } + char *newbuf = NULL; + char *newdata = NULL; + newsize += (_alignment - 1); + T newBuf(newsize); + if (newsize != 0) { + newbuf = static_cast<char *>(newBuf.get()); + assert(newbuf != NULL); + newdata = newbuf + padbefore(_alignment, newbuf); + memcpy(newdata, _datapt, getDataLen()); + } + _buffer.swap(newBuf); + _bufstart = newbuf; + _freept = newdata + getDataLen(); + _datapt = newdata; + _bufend = newbuf + newsize; + return true; +} + + +template <typename T> +void +DataBufferT<T>::pack(size_t needbytes) +{ + needbytes += (_alignment - 1); + size_t dataLen = getDataLen(); + + if ((getDeadLen() + getFreeLen()) < needbytes || + (getDeadLen() + getFreeLen()) * 4 < dataLen) + { + size_t bufsize = getBufSize() * 2; + if (bufsize < 256) { + bufsize = 256; + } + while (bufsize - dataLen < needbytes) + bufsize *= 2; + T newBuf(bufsize); + char *newbuf = static_cast<char *>(newBuf.get()); + assert(newbuf != NULL); + char *newdata = newbuf + padbefore(_alignment, newbuf); + memcpy(newdata, _datapt, dataLen); + _bufstart = newbuf; + _datapt = newdata; + _freept = newdata + dataLen; + _bufend = newbuf + bufsize; + _buffer.swap(newBuf); + } else { + char *datapt = _bufstart + padbefore(_alignment, _bufstart); + memmove(datapt, _datapt, dataLen); + _datapt = datapt; + _freept = _datapt + dataLen; + } +} + + +template <typename T> +bool +DataBufferT<T>::equals(DataBufferT *other) +{ + if (getDataLen() != other->getDataLen()) + return false; + return memcmp(getData(), other->getData(), getDataLen()) == 0; +} + + +template <typename T> +void +DataBufferT<T>::hexDump() +{ + char *pt = _datapt; + printf("*** DataBuffer HexDump BEGIN ***\n"); + uint32_t i = 0; + while (pt < _freept) { + printf("%x ", (unsigned char) *pt++); + if ((++i % 16) == 0) + printf("\n"); + } + if ((i % 16) != 0) + printf("\n"); + printf("*** DataBuffer HexDump END ***\n"); +} + + +template <typename T> +void +DataBufferT<T>::swap(DataBufferT &other) +{ + _buffer.swap(other._buffer); + std::swap(_alignment, other._alignment); + std::swap(_externalBuf, other._externalBuf); + std::swap(_bufstart, other._bufstart); + std::swap(_bufend, other._bufend); + std::swap(_datapt, other._datapt); + std::swap(_freept, other._freept); +} + +template <typename T> +T +DataBufferT<T>::stealBuffer() +{ + assert( ! referencesExternalData() ); + _externalBuf = nullptr; + _bufstart = nullptr; + _bufend = nullptr; + _datapt = nullptr; + _freept = nullptr; + return std::move(_buffer); +} + +template <typename T> +bool +DataBufferT<T>::referencesExternalData() const { + return (_externalBuf == _bufstart) && (getBufSize() > 0); +} + +template class DataBufferT<HeapAlloc>; +template class DataBufferT<MMapAlloc>; +template class DataBufferT<DefaultAlloc>; + + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/data/databuffer.h b/staging_vespalib/src/vespa/vespalib/data/databuffer.h new file mode 100644 index 00000000000..7db17808eae --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/data/databuffer.h @@ -0,0 +1,613 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <string.h> +#include <stdint.h> +#include <stdlib.h> +#include <assert.h> +#include <vespa/vespalib/util/alloc.h> + +namespace vespalib { + +/** + * This is a buffer that may hold the stream representation of + * packets. It has helper methods in order to simplify and standardize + * packet encoding and decoding. The default byte order for + * reading/writing integers is internet order (big endian). The + * methods with a 'Reverse' suffix assume that the data in the buffer + * is stored in reverse internet order (little endian). + * + * A databuffer covers a continuous chunk of memory that is split into + * 3 parts; 'dead', 'data' and 'free'. 'dead' denotes the space at the + * beginning of the buffer that may not currently be utilized, 'data' + * denotes the part that contains the actual data and 'free' denotes + * the free space at the end of the buffer. Initially, the 'dead' and + * 'data' parts are empty, and the 'free' part spans the entire + * buffer. When writing to the buffer, bytes are transfered from the + * 'free' part to the 'data' part of the buffer. When reading from the + * buffer, bytes are transferred from the 'data' part to the 'dead' + * part of the buffer. If the 'free' part of the buffer becomes empty, + * the data will be relocated within the buffer and/or a bigger buffer + * will be allocated. + **/ +template <typename T> +class DataBufferT +{ +private: + size_t _alignment; + char *_externalBuf; + char *_bufstart; + char *_bufend; + char *_datapt; + char *_freept; + T _buffer; + + DataBufferT(const DataBufferT &); + DataBufferT &operator=(const DataBufferT &); + +public: + typedef std::unique_ptr<DataBufferT<T>> UP; + + /** + * Construct a databuffer. + * + * @param len the initial size of the buffer. + * @param alignment required memory alignment for data start + **/ + DataBufferT(size_t len = 1024, size_t alignment = 1); + + /** + * Construct a databuffer using externally allocated memory. Note + * that the externally allocated memory will not be freed by the + * databuffer. + * + * @param buf pointer to preallocated memory + * @param len length of preallocated memory + **/ + DataBufferT(char *buf, size_t len) : + _alignment(1), + _externalBuf(buf), + _bufstart(buf), + _bufend(buf + len), + _datapt(_bufstart), + _freept(_bufstart) + { } + + DataBufferT(const char *buf, size_t len) : + _alignment(1), + _externalBuf(const_cast<char *>(buf)), + _bufstart(_externalBuf), + _bufend(_bufstart + len), + _datapt(_bufstart), + _freept(_bufend) + { } + + /** + * @return a pointer to the dead part of this buffer. + **/ + char *getDead() const { return _bufstart; } + + /** + * @return a pointer to the data part of this buffer. + **/ + char *getData() const { return _datapt; } + + /** + * @return a pointer to the free part of this buffer. + **/ + char *getFree() const { return _freept; } + + /** + * @return the length of the dead part of this buffer. + **/ + size_t getDeadLen() const { return _datapt - _bufstart; } + + /** + * @return the length of the data part of this buffer. + **/ + size_t getDataLen() const { return _freept - _datapt; } + + /** + * @return the length of the free part of this buffer. + **/ + size_t getFreeLen() const { return _bufend - _freept; } + + /** + * @return the length of the entire buffer. + **/ + size_t getBufSize() const { return _bufend - _bufstart; } + + + /** + * 'Move' bytes from the free part to the data part of this buffer. + * This will have the same effect as if the data already located in + * the free part of this buffer was written to the buffer. + * + * @param len number of bytes to 'move'. + **/ + void moveFreeToData(size_t len); + + /** + * 'Move' bytes from the data part to the dead part of this buffer. + * This will have the effect of discarding data without having to + * read it. + * + * @param len number of bytes to 'move'. + **/ + void moveDataToDead(size_t len) { _datapt += len; } + + + /** + * 'Move' bytes from the dead part to the data part of this buffer. + * This may be used to undo a read operation (un-discarding + * data). Note that writing to the buffer may result in + * reorganization making the data part of the buffer disappear. + * + * @param len number of bytes to 'move'. + **/ + void moveDeadToData(size_t len); + + /** + * 'Move' bytes from the data part to the free part of this buffer. + * This may be used to undo a write operation; discarding the data + * most recently written. + * + * @param len number of bytes to 'move'. + **/ + void moveDataToFree(size_t len); + + + /** + * Clear this buffer. + **/ + void clear() { _datapt = _freept = _bufstart; } + + + /** + * Shrink this buffer. The given value is the new wanted size of + * this buffer. If the buffer is already smaller or equal in size + * compared to the given value, no resizing is performed and false + * is returned (Use the @ref ensureFree method to ensure free + * space). If the buffer currently contains more data than can be + * held in a buffer of the wanted size, no resizing is performed and + * false is returned. + * + * @param newsize the wanted new size of this buffer (in bytes). + * @return true if the buffer was shrunk, false otherwise. + **/ + bool shrink(size_t newsize); + + /** + * Reorganize this buffer such that the dead part becomes empty and + * the free part contains at least the given number of + * bytes. Allocate a bigger buffer if needed. + * + * @param needbytes required size of free part. + **/ + void pack(size_t needbytes); + + /** + * Ensure that the free part contains at least the given number of + * bytes. This method invokes the @ref Pack method if the free part + * of the buffer is too small. + * + * @param needbytes required size of free part. + **/ + void ensureFree(size_t needbytes) + { + if (needbytes > getFreeLen()) + pack(needbytes); + } + + + /** + * Write an 8-bit unsigned integer to this buffer. + * + * @param n the integer to write. + **/ + void writeInt8(uint8_t n) + { + ensureFree(1); + *_freept++ = (char)n; + } + + /** + * Write a 16-bit unsigned integer to this buffer. + * + * @param n the integer to write. + **/ + void writeInt16(uint16_t n) + { + ensureFree(2); + _freept[1] = (char)n; + n >>= 8; + _freept[0] = (char)n; + _freept += 2; + } + + /** + * Write a 32-bit unsigned integer to this buffer. + * + * @param n the integer to write. + **/ + void writeInt32(uint32_t n) + { + ensureFree(4); + _freept[3] = (char)n; + n >>= 8; + _freept[2] = (char)n; + n >>= 8; + _freept[1] = (char)n; + n >>= 8; + _freept[0] = (char)n; + _freept += 4; + } + + /** + * Write a 64-bit unsigned integer to this buffer. + * + * @param n the integer to write. + **/ + void writeInt64(uint64_t n) + { + ensureFree(8); + _freept[7] = (char)n; + n >>= 8; + _freept[6] = (char)n; + n >>= 8; + _freept[5] = (char)n; + n >>= 8; + _freept[4] = (char)n; + n >>= 8; + _freept[3] = (char)n; + n >>= 8; + _freept[2] = (char)n; + n >>= 8; + _freept[1] = (char)n; + n >>= 8; + _freept[0] = (char)n; + _freept += 8; + } + + + + /** + * Read an 8-bit unsigned integer from this buffer. + * + * @return the integer that has been read. + **/ + uint8_t readInt8() + { + return (unsigned char)(*_datapt++); + } + + /** + * Read a 16-bit unsigned integer from this buffer. + * + * @return the integer that has been read. + **/ + uint16_t readInt16() + { + unsigned char *tmp = (unsigned char *)(_datapt); + _datapt += 2; + return ((*tmp << 8) + *(tmp + 1)); + } + + /** + * Read a 16-bit unsigned integer stored in reverse internet order + * from this buffer. + * + * @return the integer that has been read. + **/ + uint16_t readInt16Reverse() + { + unsigned char *tmp = (unsigned char *)(_datapt); + _datapt += 2; + return ((*(tmp + 1) << 8) + *tmp); + } + + /** + * Read a 32-bit unsigned integer from this buffer. + * + * @return the integer that has been read. + **/ + uint32_t readInt32() + { + unsigned char *tmp = (unsigned char *)(_datapt); + _datapt += 4; + return + ((((((uint32_t)(*tmp << 8) + *(tmp + 1)) << 8) + + *(tmp + 2)) << 8) + *(tmp + 3)); + } + + /** + * Read a 32-bit unsigned integer stored in reverse internet order + * from this buffer. + * + * @return the integer that has been read. + **/ + uint32_t readInt32Reverse() + { + unsigned char *tmp = (unsigned char *)(_datapt); + _datapt += 4; + return + ((((((uint32_t)(*(tmp + 3) << 8) + *(tmp + 2)) << 8) + + *(tmp + 1)) << 8) + *tmp); + } + + /** + * Read a 64-bit unsigned integer from this buffer. + * + * @return the integer that has been read. + **/ + uint64_t readInt64() + { + unsigned char *tmp = (unsigned char *)(_datapt); + _datapt += 8; + return + ((((((((((((((uint64_t)(*tmp << 8) + *(tmp + 1)) << 8) + + *(tmp + 2)) << 8) + *(tmp + 3)) << 8) + + *(tmp + 4)) << 8) + *(tmp + 5)) << 8) + + *(tmp + 6)) << 8) + *(tmp + 7)); + } + + /** + * Read a 64-bit unsigned integer stored in reverse internet order + * from this buffer. + * + * @return the integer that has been read. + **/ + uint64_t readInt64Reverse() + { + unsigned char *tmp = (unsigned char *)(_datapt); + _datapt += 8; + return + ((((((((((((((uint64_t)(*(tmp + 7) << 8) + *(tmp + 6)) << 8) + + *(tmp + 5)) << 8) + *(tmp + 4)) << 8) + + *(tmp + 3)) << 8) + *(tmp + 2)) << 8) + + *(tmp + 1)) << 8) + *tmp); + } + + float readFloat() + { + float f; + uint32_t i = readInt32(); + memcpy(&f, &i, sizeof(f)); + return f; + } + + double readDouble() + { + double f; + uint64_t i = readInt64(); + memcpy(&f, &i, sizeof(f)); + return f; + } + + void writeFloat(float f) + { + uint32_t i; + memcpy(&i, &f, sizeof(f)); + writeInt32(i); + } + + void writeDouble(double f) + { + uint64_t i; + memcpy(&i, &f, sizeof(f)); + writeInt64(i); + } + + + /** + * Peek at an 8-bit unsigned integer in this buffer. Unlike a read + * operation, this will not modify the buffer. + * + * @param offset offset of the integer to access. + * @return value of the accessed integer. + **/ + uint8_t peekInt8(size_t offset) + { + assert(getDataLen() >= offset + 1); + return (uint8_t) *(_datapt + offset); + } + + /** + * Peek at a 16-bit unsigned integer in this buffer. Unlike a read + * operation, this will not modify the buffer. + * + * @param offset offset of the integer to access. + * @return value of the accessed integer. + **/ + uint16_t peekInt16(size_t offset) + { + assert(getDataLen() >= offset + 2); + unsigned char *tmp = (unsigned char *)(_datapt + offset); + return (uint16_t) ((*tmp << 8) + *(tmp + 1)); + } + + /** + * Peek at a 16-bit unsigned integer stored in reverse internet + * order in this buffer. Unlike a read operation, this will not + * modify the buffer. + * + * @param offset offset of the integer to access. + * @return value of the accessed integer. + **/ + uint16_t peekInt16Reverse(size_t offset) + { + assert(getDataLen() >= offset + 2); + unsigned char *tmp = (unsigned char *)(_datapt + offset); + return (uint16_t) ((*(tmp + 1) << 8) + *tmp); + } + + /** + * Peek at a 32-bit unsigned integer in this buffer. Unlike a read + * operation, this will not modify the buffer. + * + * @param offset offset of the integer to access. + * @return value of the accessed integer. + **/ + uint32_t peekInt32(size_t offset) + { + assert(getDataLen() >= offset + 4); + unsigned char *tmp = (unsigned char *)(_datapt + offset); + return + ((((((uint32_t)(*tmp << 8) + *(tmp + 1)) << 8) + + *(tmp + 2)) << 8) + *(tmp + 3)); + } + + /** + * Peek at a 32-bit unsigned integer stored in reverse internet + * order in this buffer. Unlike a read operation, this will not + * modify the buffer. + * + * @param offset offset of the integer to access. + * @return value of the accessed integer. + **/ + uint32_t peekInt32Reverse(size_t offset) + { + assert(getDataLen() >= offset + 4); + unsigned char *tmp = (unsigned char *)(_datapt + offset); + return + ((((((uint32_t)(*(tmp + 3) << 8) + *(tmp + 2)) << 8) + + *(tmp + 1)) << 8) + *tmp); + } + + /** + * Peek at a 64-bit unsigned integer in this buffer. Unlike a read + * operation, this will not modify the buffer. + * + * @param offset offset of the integer to access. + * @return value of the accessed integer. + **/ + uint64_t peekInt64(size_t offset) + { + assert(getDataLen() >= offset + 8); + unsigned char *tmp = (unsigned char *)(_datapt + offset); + return + ((((((((((((((uint64_t)(*tmp << 8) + *(tmp + 1)) << 8) + + *(tmp + 2)) << 8) + *(tmp + 3)) << 8) + + *(tmp + 4)) << 8) + *(tmp + 5)) << 8) + + *(tmp + 6)) << 8) + *(tmp + 7)); + } + + /** + * Peek at a 64-bit unsigned integer stored in reverse internet + * order in this buffer. Unlike a read operation, this will not + * modify the buffer. + * + * @param offset offset of the integer to access. + * @return value of the accessed integer. + **/ + uint64_t peekInt64Reverse(size_t offset) + { + assert(getDataLen() >= offset + 8); + unsigned char *tmp = (unsigned char *)(_datapt + offset); + return + ((((((((((((((uint64_t)(*(tmp + 7) << 8) + *(tmp + 6)) << 8) + + *(tmp + 5)) << 8) + *(tmp + 4)) << 8) + + *(tmp + 3)) << 8) + *(tmp + 2)) << 8) + + *(tmp + 1)) << 8) + *tmp); + } + + + /** + * Write bytes to this buffer. + * + * @param src source byte buffer. + * @param len number of bytes to write. + **/ + void writeBytes(const void *src, size_t len) + { + ensureFree(len); + memcpy(_freept, src, len); + _freept += len; + } + + /** + * Fill buffer with zero-bytes. + * + * @param len number of zero-bytes to write. + **/ + void zeroFill(size_t len) + { + ensureFree(len); + memset(_freept, 0, len); + _freept += len; + } + + /** + * Read bytes from this buffer. + * + * @param dst destination byte buffer. + * @param len number of bytes to read. + **/ + void readBytes(void *dst, size_t len) + { + memcpy(dst, _datapt, len); + _datapt += len; + } + + /** + * Peek at bytes in this buffer. Unlike a read operation, this will + * not modify the buffer. + * + * @param dst destination byte buffer. + * @param len number of bytes to extract. + * @param offset byte offset into the buffer. + **/ + void peekBytes(void *dst, size_t len, size_t offset) + { + assert(_freept >= _datapt + offset + len); + memcpy(dst, _datapt + offset, len); + } + + /** + * Check if the data stored in this buffer equals the data stored in + * another buffer. + * + * @return true(equal)/false(not equal) + * @param other the other buffer. + **/ + bool equals(DataBufferT *other); + + /** + * Print a human-readable representation of this buffer to + * stdout. This method may be used for debugging purposes. + **/ + void hexDump(); + + /** + * Run some asserts to verify that this databuffer is in a legal + * state. + **/ + void assertValid() + { + assert(_bufstart <= _datapt); + assert(_datapt <= _freept); + assert(_freept <= _bufend); + } + + /** + * @return true if this buffer is referencing external data. + **/ + bool referencesExternalData() const; + + /** + * Swap the data stored in this buffer with the data stored in + * another buffer. Neither buffer may use externally allocated + * memory when swap is called. + * + * @param other the other buffer. + **/ + void swap(DataBufferT &other); + + T stealBuffer(); +}; + +typedef DataBufferT<DefaultAlloc> DataBuffer; +typedef DataBufferT<MMapAlloc> MMapDataBuffer; + +} // namespace vespalib + diff --git a/staging_vespalib/src/vespa/vespalib/data/fileheader.cpp b/staging_vespalib/src/vespa/vespalib/data/fileheader.cpp new file mode 100644 index 00000000000..2cbde3da04c --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/data/fileheader.cpp @@ -0,0 +1,566 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".fileheader"); + +#include "fileheader.h" + +namespace vespalib { + +VESPA_IMPLEMENT_EXCEPTION(IllegalHeaderException, vespalib::Exception); + +const uint32_t GenericHeader::MAGIC(0x5ca1ab1e); +const uint32_t GenericHeader::VERSION(1); +const GenericHeader::Tag GenericHeader::EMPTY; +const size_t ALIGNMENT=0x1000; + +GenericHeader::Tag::Tag() : + _type(TYPE_EMPTY), + _name(""), + _fVal(0), + _iVal(0), + _sVal("") +{ + // empty +} + +GenericHeader::Tag::Tag(const vespalib::string &name, float val) : + _type(TYPE_FLOAT), + _name(name), + _fVal(val), + _iVal(0), + _sVal("") +{ + // empty +} + +GenericHeader::Tag::Tag(const vespalib::string &name, double val) : + _type(TYPE_FLOAT), + _name(name), + _fVal(val), + _iVal(0), + _sVal("") +{ + // empty +} + +GenericHeader::Tag::Tag(const vespalib::string &name, int8_t val) : + _type(TYPE_INTEGER), + _name(name), + _fVal(0), + _iVal(val), + _sVal("") +{ + // empty +} + +GenericHeader::Tag::Tag(const vespalib::string &name, uint8_t val) : + _type(TYPE_INTEGER), + _name(name), + _fVal(0), + _iVal(val), + _sVal("") +{ + // empty +} + +GenericHeader::Tag::Tag(const vespalib::string &name, int16_t val) : + _type(TYPE_INTEGER), + _name(name), + _fVal(0), + _iVal(val), + _sVal("") +{ + // empty +} + +GenericHeader::Tag::Tag(const vespalib::string &name, uint16_t val) : + _type(TYPE_INTEGER), + _name(name), + _fVal(0), + _iVal(val), + _sVal("") +{ + // empty +} + +GenericHeader::Tag::Tag(const vespalib::string &name, int32_t val) : + _type(TYPE_INTEGER), + _name(name), + _fVal(0), + _iVal(val), + _sVal("") +{ + // empty +} + +GenericHeader::Tag::Tag(const vespalib::string &name, uint32_t val) : + _type(TYPE_INTEGER), + _name(name), + _fVal(0), + _iVal(val), + _sVal("") +{ + // empty +} + +GenericHeader::Tag::Tag(const vespalib::string &name, int64_t val) : + _type(TYPE_INTEGER), + _name(name), + _fVal(0), + _iVal(val), + _sVal("") +{ + // empty +} + +GenericHeader::Tag::Tag(const vespalib::string &name, uint64_t val) : + _type(TYPE_INTEGER), + _name(name), + _fVal(0), + _iVal(val), + _sVal("") +{ + // empty +} + +GenericHeader::Tag::Tag(const vespalib::string &name, const vespalib::string &val) : + _type(TYPE_STRING), + _name(name), + _fVal(0), + _iVal(0), + _sVal(val) +{ + // empty +} + +size_t +GenericHeader::Tag::getSize() const +{ + size_t ret = _name.size() + 2; + switch (_type) { + case TYPE_FLOAT: + case TYPE_INTEGER: + ret += 8; + break; + case TYPE_STRING: + ret += _sVal.size() + 1; + break; + default: + LOG_ASSERT(false); + } + return ret; +} + +size_t +GenericHeader::Tag::read(DataBuffer &buf) +{ + char *pos = buf.getData(); + vespalib::string name(pos); + buf.moveDataToDead(name.size() + 1); + uint8_t type = buf.readInt8(); + switch (type) { + case TYPE_FLOAT: + _fVal = buf.readDouble(); + break; + case TYPE_INTEGER: + _iVal = buf.readInt64(); + break; + case TYPE_STRING: + _sVal = vespalib::string(buf.getData()); + buf.moveDataToDead(_sVal.size() + 1); + break; + default: + throw IllegalHeaderException("Can not deserialize empty tag."); + } + _name = name; // assign here for exception safety + _type = (Type)type; + return buf.getData() - pos; +} + +size_t +GenericHeader::Tag::write(DataBuffer &buf) const +{ + int pos = buf.getDataLen(); + buf.writeBytes(_name.c_str(), _name.size() + 1); + buf.writeInt8(_type); + switch (_type) { + case TYPE_FLOAT: + buf.writeDouble(_fVal); + break; + case TYPE_INTEGER: + buf.writeInt64(_iVal); + break; + case TYPE_STRING: + buf.writeBytes(_sVal.c_str(), _sVal.size() + 1); + break; + default: + LOG_ASSERT(false); + } + return buf.getDataLen() - pos; +} + +GenericHeader::BufferReader::BufferReader(DataBuffer &buf) : + _buf(buf) +{ + // empty +} + +size_t +GenericHeader::BufferReader::getData(char *buf, size_t len) +{ + if (len > _buf.getDataLen()) { + len = _buf.getDataLen(); + } + _buf.readBytes(buf, len); + return len; +} + +GenericHeader::BufferWriter::BufferWriter(DataBuffer &buf) : + _buf(buf) +{ + // empty +} + +size_t +GenericHeader::BufferWriter::putData(const char *buf, size_t len) +{ + if (len > _buf.getFreeLen()) { + len = _buf.getFreeLen(); + } + _buf.writeBytes(buf, len); + return len; +} + + +GenericHeader::MMapReader::MMapReader(const char *buf, size_t sz) + : _buf(buf), + _sz(sz) +{ +} + + +size_t +GenericHeader::MMapReader::getData(char *buf, size_t len) +{ + size_t clen = std::min(len, _sz); + memcpy(buf, _buf, clen); + _buf += clen; + _sz -= clen; + return clen; +} + + +GenericHeader::GenericHeader() : + _tags() +{ + // empty +} + +const GenericHeader::Tag & +GenericHeader::getTag(size_t idx) const +{ + if (idx >= _tags.size()) { + return EMPTY; + } + TagMap::const_iterator it = _tags.begin(); + std::advance(it, idx); + return it->second; +} + +const GenericHeader::Tag & +GenericHeader::getTag(const vespalib::string &key) const +{ + TagMap::const_iterator it = _tags.find(key); + if (it == _tags.end()) { + return EMPTY; + } + return it->second; +} + +bool +GenericHeader::hasTag(const vespalib::string &key) const +{ + return _tags.find(key) != _tags.end(); +} + +bool +GenericHeader::putTag(const GenericHeader::Tag &tag) +{ + const vespalib::string &key = tag.getName(); + TagMap::iterator it = _tags.find(key); + if (it != _tags.end()) { + it->second = tag; + return false; + } + _tags.insert(TagMap::value_type(key, tag)); + return true; +} +bool +GenericHeader::removeTag(const vespalib::string &key) +{ + TagMap::iterator it = _tags.find(key); + if (it == _tags.end()) { + return false; + } + _tags.erase(it); + return true; +} + + +size_t +GenericHeader::getMinSize(void) +{ + return 4 /* magic */ + 4 /* size */ + 4 /* version */ + 4 /* num tags */; +} + + +size_t +GenericHeader::getSize() const +{ + size_t ret = getMinSize(); + for (TagMap::const_iterator it = _tags.begin(); + it != _tags.end(); ++it) + { + ret += it->second.getSize(); + } + return ret; +} + + +size_t +GenericHeader::readSize(IDataReader &reader) +{ + size_t hhSize = getMinSize(); + DataBuffer buf(hhSize, ALIGNMENT); + size_t numBytesRead = reader.getData(buf.getFree(), hhSize); + buf.moveFreeToData(numBytesRead); + + if (numBytesRead < hhSize) { + throw IllegalHeaderException("Failed to read header info."); + } + uint32_t magic = buf.readInt32(); + if (magic != MAGIC) { + throw IllegalHeaderException("Failed to verify magic bits."); + } + uint32_t numBytesTotal = buf.readInt32(); + if (numBytesTotal == 0) { + throw IllegalHeaderException("Failed to read header size."); + } + if (numBytesTotal < getMinSize()) { + throw IllegalHeaderException("Failed to verify header size."); + } + uint32_t version = buf.readInt32(); + if (version != VERSION) { + throw IllegalHeaderException("Failed to verify header version."); + } + return numBytesTotal; +} + + +size_t +GenericHeader::read(IDataReader &reader) +{ + size_t bufLen = 1024 * 32; + DataBuffer buf(bufLen, ALIGNMENT); + size_t numBytesRead = reader.getData(buf.getFree(), bufLen); + buf.moveFreeToData(numBytesRead); + + if (numBytesRead < 4 /* magic */ + 4 /* size */) { + throw IllegalHeaderException("Failed to read header info."); + } + uint32_t magic = buf.readInt32(); + if (magic != MAGIC) { + throw IllegalHeaderException("Failed to verify magic bits."); + } + uint32_t numBytesTotal = buf.readInt32(); + if (numBytesTotal == 0) { + throw IllegalHeaderException("Failed to read header size."); + } + if (numBytesTotal < getMinSize()) { + throw IllegalHeaderException("Failed to verify header size."); + } + if (numBytesRead < numBytesTotal) { + LOG(debug, "Read %d of %d header bytes, performing backfill.", + (uint32_t)numBytesRead, numBytesTotal); + uint32_t numBytesRemain = numBytesTotal - numBytesRead; + buf.ensureFree(numBytesRemain); + LOG(debug, "Reading remaining %d bytes of header.", numBytesRemain); + numBytesRead += reader.getData(buf.getFree(), numBytesRemain); + if (numBytesRead != numBytesTotal) { + throw IllegalHeaderException("Failed to read full header."); + } + buf.moveFreeToData(numBytesRemain); + } else { + buf.moveDataToFree(numBytesRead - numBytesTotal); + } + + uint32_t version = buf.readInt32(); + if (version != VERSION) { + throw IllegalHeaderException("Failed to verify header version."); + } + uint32_t numTags = buf.readInt32(); + TagMap tags; + for (uint32_t i = 0; i < numTags; ++i) { + Tag tag; + tag.read(buf); + tags.insert(TagMap::value_type(tag.getName(), tag)); + } + _tags.swap(tags); + return numBytesTotal; +} + +size_t +GenericHeader::write(IDataWriter &writer) const +{ + size_t numBytesTotal = getSize(); + DataBuffer buf(numBytesTotal, ALIGNMENT); + buf.writeInt32(MAGIC); + buf.writeInt32((uint32_t)numBytesTotal); + buf.writeInt32(VERSION); + buf.writeInt32((uint32_t)_tags.size()); + uint32_t numBytesInBuf = 16; + for (TagMap::const_iterator it = _tags.begin(); + it != _tags.end(); ++it) + { + numBytesInBuf += it->second.write(buf); + } + if (numBytesInBuf < numBytesTotal) { + buf.zeroFill(numBytesTotal - numBytesInBuf); + } + size_t numBytesWritten = writer.putData(buf.getData(), numBytesTotal); + if (numBytesWritten != numBytesTotal) { + throw IllegalHeaderException("Failed to write header."); + } + return numBytesWritten; +} + +FileHeader::FileReader::FileReader(FastOS_FileInterface &file) : + _file(file) +{ + // empty +} + +size_t +FileHeader::FileReader::getData(char *buf, size_t len) +{ + LOG_ASSERT(_file.IsOpened()); + LOG_ASSERT(_file.IsReadMode()); + + return _file.Read(buf, len); +} + +FileHeader::FileWriter::FileWriter(FastOS_FileInterface &file) : + _file(file) +{ + // empty +} + +size_t +FileHeader::FileWriter::putData(const char *buf, size_t len) +{ + LOG_ASSERT(_file.IsOpened()); + LOG_ASSERT(_file.IsWriteMode()); + + return _file.Write2(buf, len); +} + +FileHeader::FileHeader(size_t alignTo, size_t minSize) : + _alignTo(alignTo), + _minSize(minSize), + _fileSize(0) +{ + // empty +} + +size_t +FileHeader::getSize() const +{ + size_t ret = GenericHeader::getSize(); + if (_fileSize > ret) { + return _fileSize; + } + if (_minSize > ret) { + return _minSize; + } + size_t pad = ret % _alignTo; + return ret + (pad > 0 ? _alignTo - pad : 0); +} + +size_t +FileHeader::readFile(FastOS_FileInterface &file) +{ + FileReader reader(file); + return GenericHeader::read(reader); +} + +size_t +FileHeader::writeFile(FastOS_FileInterface &file) const +{ + FileWriter writer(file); + return GenericHeader::write(writer); +} + +size_t +FileHeader::rewriteFile(FastOS_FileInterface &file) +{ + LOG_ASSERT(file.IsOpened()); + LOG_ASSERT(file.IsReadMode()); + LOG_ASSERT(file.IsWriteMode()); + + // Store current position in file. + int64_t pos = file.GetPosition(); + if (pos != 0) { + file.SetPosition(0); + } + + // Assert that header size agrees with file content. + FileReader reader(file); + size_t wantSize = 4 /* magic */ + 4 /* size */; + DataBuffer buf(wantSize, ALIGNMENT); + size_t numBytesRead = reader.getData(buf.getFree(), wantSize); + if (numBytesRead < wantSize) { + throw IllegalHeaderException("Failed to read header info."); + } + uint32_t magic = buf.readInt32(); + if (magic != MAGIC) { + throw IllegalHeaderException("Failed to verify magic bits."); + } + uint32_t size = buf.readInt32(); + if (size == 0) { + throw IllegalHeaderException("Failed to read header size."); + } + if (size < GenericHeader::getSize()) { + throw IllegalHeaderException("Failed to rewrite resized header."); + } + _fileSize = size; + + // Write new header and reset file position. + file.SetPosition(0); + size_t ret = writeFile(file); + if (file.GetPosition() != pos) { + file.SetPosition(pos); + } + return ret; +} + +vespalib::asciistream & +operator<<(vespalib::asciistream &out, const GenericHeader::Tag &tag) +{ + switch (tag.getType()) { + case GenericHeader::Tag::TYPE_FLOAT: + out << tag.asFloat(); + break; + case GenericHeader::Tag::TYPE_INTEGER: + out << tag.asInteger(); + break; + case GenericHeader::Tag::TYPE_STRING: + out << tag.asString(); + break; + default: + LOG_ASSERT(false); + } + return out; +} + +} // namespace diff --git a/staging_vespalib/src/vespa/vespalib/data/fileheader.h b/staging_vespalib/src/vespa/vespalib/data/fileheader.h new file mode 100644 index 00000000000..89b63949a35 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/data/fileheader.h @@ -0,0 +1,333 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <map> +#include "databuffer.h" +#include <vespa/fastos/file.h> +#include <vespa/vespalib/util/exception.h> +#include <vespa/vespalib/stllike/asciistream.h> + +namespace vespalib { + +/** + * This exception can be thrown when serializing or deserializing header content. + */ +VESPA_DEFINE_EXCEPTION(IllegalHeaderException, vespalib::Exception); + +/** + * This class implements a collection of GenericHeader::Tag objects that can be set and retrieved by name. The + * interfaces GenericHeader::IDataReader or GenericHeader::IDataWriter define the api to allow deserialization + * and serialization of an instance of this class to any underlying data buffer. + */ +class GenericHeader { +public: + static const uint32_t MAGIC; + static const uint32_t VERSION; + + /** + * This class implements a immutable named value of a specified type. This type can be any of + * Tag::Type. There is no enforcement of type, so using asInteger() on a TYPE_STRING instance will simple + * return the default integer value. + */ + class Tag { + public: + enum Type { + TYPE_EMPTY = 'e', + TYPE_FLOAT = 'f', + TYPE_INTEGER = 'i', + TYPE_STRING = 's' + }; + + private: + Type _type; + vespalib::string _name; + double _fVal; + int64_t _iVal; + vespalib::string _sVal; + + public: + Tag(); + Tag(const vespalib::string &name, float val); + Tag(const vespalib::string &name, double val); + Tag(const vespalib::string &name, int8_t val); + Tag(const vespalib::string &name, uint8_t val); + Tag(const vespalib::string &name, int16_t val); + Tag(const vespalib::string &name, uint16_t val); + Tag(const vespalib::string &name, int32_t val); + Tag(const vespalib::string &name, uint32_t val); + Tag(const vespalib::string &name, int64_t val); + Tag(const vespalib::string &name, uint64_t val); + Tag(const vespalib::string &name, const vespalib::string &val); + + size_t read(DataBuffer &buf); + size_t write(DataBuffer &buf) const; + size_t getSize() const; + + bool isEmpty() const { return _type == TYPE_EMPTY; } + Type getType() const { return _type; } + const vespalib::string &getName() const { return _name; } + + double asFloat() const { return _fVal; } + int64_t asInteger() const { return _iVal; } + const vespalib::string &asString() const { return _sVal; } + }; + + /** + * This class defines the interface used by GenericHeader to deserialize content. It has implementation + * GenericHeader::BufferReader for reading from a buffer, and FileHeader::FileReader for reading from a + * file. + */ + class IDataReader { + public: + virtual ~IDataReader() { /* empty */ } + virtual size_t getData(char *buf, size_t len) = 0; + }; + + /** + * This class defines the interface used by GenericHeader to serialize content. It has implementation + * GenericHeader::BufferWriter for reading from a buffer, and FileHeader::FileWriter for reading from a + * file. + */ + class IDataWriter { + public: + virtual ~IDataWriter() { /* empty */ } + virtual size_t putData(const char *buf, size_t len) = 0; + }; + + /** + * Implements the GenericHeader::IDataReader interface for deserializing header content from a + * DataBuffer instance. + */ + class BufferReader : public IDataReader { + private: + DataBuffer &_buf; + + public: + BufferReader(DataBuffer &buf); + size_t getData(char *buf, size_t len); + }; + + /** + * Implements the GenericHeader::IDataWriter interface for serializing header content to a + * DataBuffer instance. + */ + class BufferWriter : public IDataWriter { + private: + DataBuffer &_buf; + + public: + BufferWriter(DataBuffer &buf); + size_t putData(const char *buf, size_t len); + }; + + class MMapReader : public IDataReader + { + const char *_buf; + size_t _sz; + + public: + MMapReader(const char *buf, size_t sz); + + size_t + getData(char *buf, size_t len); + }; + +private: + static const Tag EMPTY; + + typedef std::map<vespalib::string, Tag> TagMap; + TagMap _tags; + +public: + /** + * Constructs a new instance of this class. + */ + GenericHeader(); + + /** + * Virtual destructor required for inheritance. + */ + virtual ~GenericHeader() { /* empty */ } + + /** + * Returns the number of tags contained in this header. + * + * @return The size of the tag map. + */ + size_t getNumTags() const { return _tags.size(); } + + /** + * Returns the tag at the given index. This can be used along with getNumTags() to iterate over all the + * tags in a header. This is not an efficient way of accessing tags, since the underlying map does not + * support random access. If you are interested in a specific tag, use getTag() by name instead. + * + * @param idx The index of the tag to return. + * @return The tag at the given index. + */ + const Tag &getTag(size_t idx) const; + + /** + * Returns a reference to the named tag. If hasTag() returned false for the same key, this method returns + * a tag that is of type Tag::TYPE_EMPTY and returns true for Tag::isEmpty(). + * + * @param key The name of the tag to return. + * @return A reference to the named tag. + */ + const Tag &getTag(const vespalib::string &key) const; + + /** + * Returns whether or not there exists a tag with the given name. + * + * @param key The name of the tag to look for. + * @return True if the named tag exists. + */ + bool hasTag(const vespalib::string &key) const; + + /** + * Adds the given tag to this header. If a tag already exists with the given name, this method replaces + * that tag and returns false. + * + * @param tag The tag to add. + * @return True if no tag was overwritten. + */ + bool putTag(const Tag &tag); + + /** + * Removes a named tag. If no tag exists with the given name, this method returns false. + * + * @param key The name of the tag to remove. + * @return True if a tag was removed. + */ + bool removeTag(const vespalib::string &key); + + /** + * Returns whether or not this header contains any data. The current implementation only checks for tags, + * but as this class evolves it might include other data as well. + * + * @return True if this header has no data. + */ + bool isEmpty() const { return _tags.empty(); } + + static size_t + getMinSize(void); + + /** + * Returns the number of bytes required to hold the content of this when calling write(). + * + * @return The number of bytes. + */ + virtual size_t getSize() const; + + static size_t + readSize(IDataReader &reader); + + /** + * Deserializes header content from the given provider into this. + * + * @param reader The provider to read from. + * @return The number of bytes read. + */ + size_t read(IDataReader &reader); + + /** + * Serializes the content of this into the given consumer. + * + * @param writer The consumer to write to. + * @return The number of bytes written. + */ + size_t write(IDataWriter &writer) const; +}; + +/** + * This class adds file-specific functionality to the GenericHeader class. This includes alignment of size to + * some set number of bytes, as well as the ability to update a header in-place (see FileHeader::rewrite()). + */ +class FileHeader : public GenericHeader { +public: + /** + * Implements the GenericHeader::IDataReader interface for deserializing header content from a + * FastOS_FileInterface instance. + */ + class FileReader : public IDataReader { + private: + FastOS_FileInterface &_file; + + public: + FileReader(FastOS_FileInterface &file); + size_t getData(char *buf, size_t len); + }; + + /** + * Implements the GenericHeader::IDataWriter interface for serializing header content to a + * FastOS_FileInterface instance. + */ + class FileWriter : public IDataWriter { + private: + FastOS_FileInterface &_file; + + public: + FileWriter(FastOS_FileInterface &file); + size_t putData(const char *buf, size_t len); + }; + +private: + size_t _alignTo; + size_t _minSize; + size_t _fileSize; + +public: + /** + * Constructs a new instance of this class. + * + * @param alignTo The number of bytes to which the serialized size must be aligned to. + * @param minSize The minimum number of bytes of the serialized size. + */ + FileHeader(size_t alignTo = 8u, size_t minSize = 0u); + + /** + * This function overrides GenericHeader::getSize() to align header size to the number of bytes supplied + * to the constructor of this class. Furthermore, it will attempt to keep the header size constant after + * an initial read() or write() so that one can later rewrite() it. + * + * @return The number of bytes required to hold the content of this. + */ + size_t getSize() const; + + /** + * Deserializes header content from the given file into this. This requires that the file is open in read + * mode, and that it is positioned at the start of the file. + * + * @param file The file to read from. + * @return The number of bytes read. + */ + size_t readFile(FastOS_FileInterface &file); + + /** + * Serializes the content of this into the given file. This requires that the file is open in write mode, + * and that it is positioned at the start of the file. + * + * @param file The file to write to. + * @return The number of bytes written. + */ + size_t writeFile(FastOS_FileInterface &file) const; + + /** + * Serializes the content of this into the given file. This requires that the file is open in read-write + * mode. This method reads the first 64 bits of the file to ensure that it is compatible with this, then + * write its content to it. Finally, it moves the file position back to where it was before. + * + * @param file The file to write to. + * @return The number of bytes written. + */ + size_t rewriteFile(FastOS_FileInterface &file); +}; + +/** + * Implements ostream operator for GenericHeader::Tag class. This method will only output the actual value of + * the tag, not its name or type. Without this operator you would have to switch on tag type to decide which + * value accessor to use. + */ +vespalib::asciistream &operator<<(vespalib::asciistream &out, const GenericHeader::Tag &tag); + +} // namespace + diff --git a/staging_vespalib/src/vespa/vespalib/encoding/.gitignore b/staging_vespalib/src/vespa/vespalib/encoding/.gitignore new file mode 100644 index 00000000000..ee8938b6bf4 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/encoding/.gitignore @@ -0,0 +1,6 @@ +*.So +*.exe +*.ilk +*.pdb +.depend* +Makefile diff --git a/staging_vespalib/src/vespa/vespalib/encoding/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/encoding/CMakeLists.txt new file mode 100644 index 00000000000..6eccc098d9f --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/encoding/CMakeLists.txt @@ -0,0 +1,6 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(staging_vespalib_vespalib_encoding OBJECT + SOURCES + base64.cpp + DEPENDS +) diff --git a/staging_vespalib/src/vespa/vespalib/encoding/base64.cpp b/staging_vespalib/src/vespa/vespalib/encoding/base64.cpp new file mode 100644 index 00000000000..cafa84f2b50 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/encoding/base64.cpp @@ -0,0 +1,166 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * + * $Id$ + * This class convert a string to a base64 representation of the string. + * + */ + +#include <vespa/fastos/fastos.h> +#include <assert.h> +#include <vespa/vespalib/encoding/base64.h> +#include <vespa/vespalib/util/exceptions.h> + +namespace vespalib { + +static const char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789+/="; + +// Set -1 for illegal chars that will cause an error. +// Set -2 for illegal chars that will be ignored. (whitespace " \r\t\f\n") +static const char base64Backwards[] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -2, -2, -1, -2, -2, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -2, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, + -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }; + +std::string +Base64::encode(const char* source, int len) +{ + // Assign a string that we know is long enough + std::string result(getMaximumEncodeLength(len), '\0'); + int outlen = encode(source, len, &result[0], result.size()); + assert(outlen >= 0); // Make sure buffer was big enough. + result.resize(outlen); + return result; +} + +std::string +Base64::decode(const char* source, int len) +{ + std::string result(getMaximumDecodeLength(len), '\0'); + int outlen = decode(source, len, &result[0], len); + assert(outlen >= 0); + result.resize(outlen); + return result; +} + +int +Base64::encode(const char *inBuffer, int inLen, char *outBuffer, int outBufLen) +{ + int i; + int outLen = 0; + for (i = 0; inLen >= 3; inLen -= 3) { + if (outBufLen - outLen < 4) { + return -1; + } + // Do this to keep chars > 127 + unsigned char a = inBuffer[i]; + unsigned char b = inBuffer[i+1]; + unsigned char c = inBuffer[i+2]; + i += 3; + + outBuffer[outLen ] = base64Chars[ a >> 2 ]; + outBuffer[outLen + 1] = base64Chars[ (a << 4 & 0x30) | (b >> 4) ]; + outBuffer[outLen + 2] = base64Chars[ (b << 2 & 0x3c) | (c >> 6) ]; + outBuffer[outLen + 3] = base64Chars[ c & 0x3f ]; + + outLen += 4; + } + + if (inLen) { + if (outBufLen - outLen < 4) { + return -1; + } + // Do this to keep chars with value>127 + unsigned char a = inBuffer[i]; + + outBuffer[outLen] = base64Chars[ a >> 2 ]; + + if (inLen == 1) { + outBuffer[outLen + 1] = base64Chars[ (a << 4 & 0x30) ]; + outBuffer[outLen + 2] = '='; + } else { + unsigned char b = inBuffer[i + 1]; + outBuffer[outLen + 1] = base64Chars[ (a << 4 & 0x30) | (b >> 4) ]; + outBuffer[outLen + 2] = base64Chars[ b << 2 & 0x3c ]; + } + + outBuffer[outLen + 3] = '='; + + outLen += 4; + } + + if (outLen >= outBufLen) + return -1; + + outBuffer[outLen] = '\0'; + + return outLen; +} + +int +Base64::decode(const char* inBuffer, int inLen, char* outBuffer, int outLen) +{ + // Read char by char to better support skipping of illegal character. + int readbytes = 0; + int num_valid_chars = 0; + const char* thischar = inBuffer; + char curchar; + int curOut = 0; + char tmp = 0; + + while( (readbytes++ < inLen) && (*thischar != '\0') && (*thischar != '=')) { + curchar = base64Backwards[ (unsigned int)(*thischar++) ]; + + if (curchar == -2) { + continue; // Some illegal chars will be skipped. + } else if (curchar == -1) { + // Other illegal characters will generate failure + throw vespalib::IllegalArgumentException(vespalib::make_string( + "Illegal base64 character %u found.", + (unsigned int) *thischar), VESPA_STRLOC); + } else { + + // Four bytes from input (eqals three bytes in output) + if (outLen <= curOut) { + return -1; + } + switch( num_valid_chars % 4 ) { + case 0: + tmp = (curchar << 2); + break; + case 1: + outBuffer[curOut++] = tmp | ((curchar >> 4) & 0x03); + tmp = ((curchar & 0xf) << 4); + break; + case 2: + outBuffer[curOut++] = tmp | ((curchar >> 2) & 0x0f); + tmp = ((curchar & 0x03 ) << 6); + break; + case 3: + outBuffer[curOut++] = tmp | curchar; + tmp = 0; + break; + } + num_valid_chars++; + } + } + return curOut; +} + +} // namespace vespalib + diff --git a/staging_vespalib/src/vespa/vespalib/encoding/base64.h b/staging_vespalib/src/vespa/vespalib/encoding/base64.h new file mode 100644 index 00000000000..11c74261627 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/encoding/base64.h @@ -0,0 +1,124 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @class vespalib::Base64 + * @group util + * + * Utility class for conversion between binary and base64 encoded data. + * + * $Id$ + */ + +#pragma once + +#include <string> +#include <algorithm> + +namespace vespalib { + +struct Base64 { + + /** + * @param sourcelen The length of the source string. + * + * @return The maximum number of characters needed to encode a string of + * given length, including terminating '\0' byte. + * + * @todo This seems to be more than needed. Inspect what encode() does and + * make this the exact size. + **/ + static int getMaximumEncodeLength(int sourcelen) + { return std::max(6, 2 * sourcelen + 2); } + + /** + * @param The length of the base64 encoded string to decode. + * + * @return The maximum size of the decoded data of a base64 string of the + * given length. + * + * @todo This seems to be more than needed. Inspect what decode() does and + * make this the exact size. + **/ + static int getMaximumDecodeLength(int sourcelen) + { return sourcelen; } + + /** + * Encodes a string of binary data to base 64. + * + * @param source The buffer to convert. + * @param len The length of the buffer. + * + * @return The base64 encoded string. + */ + static std::string encode(const std::string& source) + { return encode(source.c_str(), source.size()); } + + /** + * Encodes binary data to base 64. + * + * @param source The buffer to convert. + * @param len The length of the buffer. + * + * @return The base64 encoded string. + */ + static std::string encode(const char* source, int len); + + /** + * Encodes binary data pointed to by source, to base 64 data + * written into dest. + * + * @param source The input buffer. + * @param sourcelen The length of the input buffer. + * @param dest The buffer to write the encoded data to. + * @param destlen The length of the output buffer. This may need to be + * up to getMaximumEncodeLength(sourcelen) bytes. + * + * @return The number of characters used in dest to store the encoded + * data. Excluding '\0' termination of the string (which is always + * added). -1 is returned if there was not enough space in the dest + * buffer to store all of the data. + */ + static int encode(const char* source, int sourcelen, + char* dest, int destlen); + + /** + * Decodes base64 data to binary format. + * + * @param source The buffer to convert. + * @param len The length of the buffer. + * + * @return The base64 decoded string. + * + * @throws Throw IllegalArgumentException if source contains illegal base 64 + * characters that are not whitespace. + */ + static std::string decode(const std::string& source) + { return decode(source.c_str(), source.size()); } + + /** + * Decodes base64 data to binary format. + * + * @param source The buffer to convert. + * @param len The length of the buffer. + * + * @return The base64 decoded string. + */ + static std::string decode(const char* source, int len); + + /** + * Decodes base 64 data in source to binary format written into dest. + * + * @param source The input buffer. + * @param sourcelen The length of the input buffer. + * @param dest The buffer to write the encoded data to. + * @param destlen The length of the output buffer. + * + * @return The number of bytes used in dest to store the binary + * representation, or -1 if there wasn't enough bytes available in + * dest. + */ + static int decode(const char* source, int sourcelen, + char* dest, int destlen); +}; + +} // namespace vespalib + diff --git a/staging_vespalib/src/vespa/vespalib/hwaccelrated/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/hwaccelrated/CMakeLists.txt new file mode 100644 index 00000000000..da1b76090a0 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/hwaccelrated/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(staging_vespalib_vespalib_hwaccelrated OBJECT + SOURCES + iaccelrated.cpp + generic.cpp + sse2.cpp + avx.cpp + DEPENDS +) diff --git a/staging_vespalib/src/vespa/vespalib/hwaccelrated/avx.cpp b/staging_vespalib/src/vespa/vespalib/hwaccelrated/avx.cpp new file mode 100644 index 00000000000..00708edf5b7 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/hwaccelrated/avx.cpp @@ -0,0 +1,113 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright (C) 2003 Fast Search & Transfer ASA +// Copyright (C) 2003 Overture Services Norway AS + +#include <vespa/vespalib/hwaccelrated/avx.h> + +namespace vespalib { + +namespace hwaccelrated { + +namespace { + +bool validAlignment32(const void * p) { + return (reinterpret_cast<uint64_t>(p) & 0x1ful) == 0; +} + +} + +template <unsigned AlignA, unsigned AlignB> +float +AvxAccelrator::computeDotProduct(const float * af, const float * bf, size_t sz) +{ + typedef float v8saf __attribute__ ((vector_size (32))); + const size_t ChunkSize(32); + const size_t VectorsPerChunk(ChunkSize/8); + v8saf partial[VectorsPerChunk] = { {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, + {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0} }; + typedef float A __attribute__ ((vector_size (32), aligned(AlignA))); + typedef float B __attribute__ ((vector_size (32), aligned(AlignB))); + const A * a = reinterpret_cast<const A *>(af); + const B * b = reinterpret_cast<const B *>(bf); + + const size_t numChunks(sz/ChunkSize); + for (size_t i(0); i < numChunks; i++) { + for (size_t j(0); j < VectorsPerChunk; j++) { + partial[j] += a[VectorsPerChunk*i+j] * b[VectorsPerChunk*i+j]; + } + } + float sum(0); + for (size_t i(numChunks*ChunkSize); i < sz; i++) { + sum += af[i] * bf[i]; + } + for (size_t i(1); i < VectorsPerChunk; i++) { + partial[0] += partial[i]; + } + sum += partial[0][0] + partial[0][1] + partial[0][2] + partial[0][3] + + partial[0][4] + partial[0][5] + partial[0][6] + partial[0][7]; + return sum; +} + +template <unsigned AlignA, unsigned AlignB> +double +AvxAccelrator::computeDotProduct(const double * af, const double * bf, size_t sz) +{ + typedef double v4sd __attribute__ ((vector_size (32))); + const size_t ChunkSize(16); + const size_t VectorsPerChunk(ChunkSize/4); + v4sd partial[VectorsPerChunk] = { {0.0, 0.0, 0.0, 0.0}, {0.0, 0.0, 0.0, 0.0}, {0.0, 0.0, 0.0, 0.0}, {0.0, 0.0, 0.0, 0.0} }; + typedef double A __attribute__ ((vector_size (32), aligned(AlignA))); + typedef double B __attribute__ ((vector_size (32), aligned(AlignB))); + const A * a = reinterpret_cast<const A *>(af); + const B * b = reinterpret_cast<const B *>(bf); + + const size_t numChunks(sz/ChunkSize); + for (size_t i(0); i < numChunks; i++) { + for (size_t j(0); j < VectorsPerChunk; j++) { + partial[j] += a[VectorsPerChunk*i+j] * b[VectorsPerChunk*i+j]; + } + } + double sum(0); + for (size_t i(numChunks*ChunkSize); i < sz; i++) { + sum += af[i] * bf[i]; + } + for (size_t i(1); i < VectorsPerChunk; i++) { + partial[0] += partial[i]; + } + sum += partial[0][0] + partial[0][1] + partial[0][2] + partial[0][3]; + return sum; +} + +template <typename T> +T +AvxAccelrator::dotProductSelectAlignment(const T * af, const T * bf, size_t sz) +{ + if (validAlignment32(af)) { + if (validAlignment32(bf)) { + return computeDotProduct<32, 32>(af, bf, sz); + } else { + return computeDotProduct<32, 1>(af, bf, sz); + } + } else { + if (validAlignment32(bf)) { + return computeDotProduct<1, 32>(af, bf, sz); + } else { + return computeDotProduct<1, 1>(af, bf, sz); + } + } +} + +float +AvxAccelrator::dotProduct(const float * af, const float * bf, size_t sz) const +{ + return dotProductSelectAlignment(af, bf, sz); +} + +double +AvxAccelrator::dotProduct(const double * af, const double * bf, size_t sz) const +{ + return dotProductSelectAlignment(af, bf, sz); +} + +} +} diff --git a/staging_vespalib/src/vespa/vespalib/hwaccelrated/avx.h b/staging_vespalib/src/vespa/vespalib/hwaccelrated/avx.h new file mode 100644 index 00000000000..bd4816eb3a1 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/hwaccelrated/avx.h @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright (C) 2003 Fast Search & Transfer ASA +// Copyright (C) 2003 Overture Services Norway AS + +#pragma once + +#include <vespa/vespalib/hwaccelrated/sse2.h> +#include <vespa/fastos/dynamiclibrary.h> + +namespace vespalib { + +namespace hwaccelrated { + +/** + * Generic cpu agnostic implementation. + */ +class AvxAccelrator : public Sse2Accelrator +{ +public: + virtual float dotProduct(const float * a, const float * b, size_t sz) const; + virtual double dotProduct(const double * a, const double * b, size_t sz) const; +private: + template <typename T> + VESPA_DLL_LOCAL static T dotProductSelectAlignment(const T * af, const T * bf, size_t sz); + template <unsigned AlignA, unsigned AlignB> + VESPA_DLL_LOCAL static double computeDotProduct(const double * af, const double * bf, size_t sz) __attribute__((noinline)); + template <unsigned AlignA, unsigned AlignB> + VESPA_DLL_LOCAL static float computeDotProduct(const float * af, const float * bf, size_t sz) __attribute__((noinline)); +}; + +} +} diff --git a/staging_vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp b/staging_vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp new file mode 100644 index 00000000000..f218e4172f9 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp @@ -0,0 +1,125 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright (C) 2003 Fast Search & Transfer ASA +// Copyright (C) 2003 Overture Services Norway AS + +#include <vespa/vespalib/hwaccelrated/generic.h> + +namespace vespalib { + +namespace hwaccelrated { + +namespace { + +template <typename ACCUM, typename T, size_t UNROLL> +ACCUM +multiplyAdd(const T * a, const T * b, size_t sz) +{ + ACCUM partial[UNROLL]; + for (size_t i(0); i < UNROLL; i++) { + partial[i] = 0; + } + size_t i(0); + for (; i + UNROLL <= sz; i+= UNROLL) { + for (size_t j(0); j < UNROLL; j++) { + partial[j] += a[i+j] * b[i+j]; + } + } + for (;i < sz; i++) { + partial[i%UNROLL] += a[i] * b[i]; + } + ACCUM sum(0); + for (size_t j(0); j < UNROLL; j++) { + sum += partial[j]; + } + return sum; +} + +} + +float +GenericAccelrator::dotProduct(const float * a, const float * b, size_t sz) const +{ + return multiplyAdd<float, float, 4>(a, b, sz); +} + +double +GenericAccelrator::dotProduct(const double * a, const double * b, size_t sz) const +{ + return multiplyAdd<double, double, 4>(a, b, sz); +} + +int64_t +GenericAccelrator::dotProduct(const int32_t * a, const int32_t * b, size_t sz) const +{ + return multiplyAdd<int64_t, int32_t, 4>(a, b, sz); +} + +long long +GenericAccelrator::dotProduct(const int64_t * a, const int64_t * b, size_t sz) const +{ + return multiplyAdd<long long, int64_t, 4>(a, b, sz); +} + +void +GenericAccelrator::orBit(void * aOrg, const void * bOrg, size_t bytes) const +{ + uint64_t *a(static_cast<uint64_t *>(aOrg)); + const uint64_t *b(static_cast<const uint64_t *>(bOrg)); + const size_t sz(bytes/sizeof(uint64_t)); + for (size_t i(0); i < sz; i++) { + a[i] |= b[i]; + } + uint8_t *ac(static_cast<uint8_t *>(aOrg)); + const uint8_t *bc(static_cast<const uint8_t *>(bOrg)); + for (size_t i(sz*sizeof(uint64_t)); i < bytes; i++) { + ac[i] |= bc[i]; + } +} + +void +GenericAccelrator::andBit(void * aOrg, const void * bOrg, size_t bytes) const +{ + uint64_t *a(static_cast<uint64_t *>(aOrg)); + const uint64_t *b(static_cast<const uint64_t *>(bOrg)); + const size_t sz(bytes/sizeof(uint64_t)); + for (size_t i(0); i < sz; i++) { + a[i] &= b[i]; + } + uint8_t *ac(static_cast<uint8_t *>(aOrg)); + const uint8_t *bc(static_cast<const uint8_t *>(bOrg)); + for (size_t i(sz*sizeof(uint64_t)); i < bytes; i++) { + ac[i] &= bc[i]; + } +} +void +GenericAccelrator::andNotBit(void * aOrg, const void * bOrg, size_t bytes) const +{ + uint64_t *a(static_cast<uint64_t *>(aOrg)); + const uint64_t *b(static_cast<const uint64_t *>(bOrg)); + const size_t sz(bytes/sizeof(uint64_t)); + for (size_t i(0); i < sz; i++) { + a[i] &= ~b[i]; + } + uint8_t *ac(static_cast<uint8_t *>(aOrg)); + const uint8_t *bc(static_cast<const uint8_t *>(bOrg)); + for (size_t i(sz*sizeof(uint64_t)); i < bytes; i++) { + ac[i] &= ~bc[i]; + } +} + +void +GenericAccelrator::notBit(void * aOrg, size_t bytes) const +{ + uint64_t *a(static_cast<uint64_t *>(aOrg)); + const size_t sz(bytes/sizeof(uint64_t)); + for (size_t i(0); i < sz; i++) { + a[i] = ~a[i]; + } + uint8_t *ac(static_cast<uint8_t *>(aOrg)); + for (size_t i(sz*sizeof(uint64_t)); i < bytes; i++) { + ac[i] = ~ac[i]; + } +} + +} +} diff --git a/staging_vespalib/src/vespa/vespalib/hwaccelrated/generic.h b/staging_vespalib/src/vespa/vespalib/hwaccelrated/generic.h new file mode 100644 index 00000000000..0cb21b70ca3 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/hwaccelrated/generic.h @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright (C) 2003 Fast Search & Transfer ASA +// Copyright (C) 2003 Overture Services Norway AS + +#pragma once + +#include <vespa/vespalib/hwaccelrated/iaccelrated.h> + +namespace vespalib { + +namespace hwaccelrated { + +/** + * Generic cpu agnostic implementation. + */ +class GenericAccelrator : public IAccelrated +{ +public: + float dotProduct(const float * a, const float * b, size_t sz) const override; + double dotProduct(const double * a, const double * b, size_t sz) const override; + int64_t dotProduct(const int32_t * a, const int32_t * b, size_t sz) const override; + long long dotProduct(const int64_t * a, const int64_t * b, size_t sz) const override; + void orBit(void * a, const void * b, size_t bytes) const override; + void andBit(void * a, const void * b, size_t bytes) const override; + void andNotBit(void * a, const void * b, size_t bytes) const override; + void notBit(void * a, size_t bytes) const override; +}; + +} +} diff --git a/staging_vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp b/staging_vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp new file mode 100644 index 00000000000..fd4fc706b89 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.cpp @@ -0,0 +1,124 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright (C) 2003 Fast Search & Transfer ASA +// Copyright (C) 2003 Overture Services Norway AS + +#include <vespa/vespalib/hwaccelrated/iaccelrated.h> +#include <vespa/vespalib/hwaccelrated/generic.h> +#include <vespa/vespalib/hwaccelrated/sse2.h> +#include <vespa/vespalib/hwaccelrated/avx.h> +#include <assert.h> + +namespace vespalib { + +namespace hwaccelrated { + +#if 0 +__attribute__ ((target ("default"), noinline)) +vespalib::hwaccelrated::IAccelrated::UP selectAccelrator() { return vespalib::hwaccelrated::IAccelrated::UP(new vespalib::hwaccelrated::GenericAccelrator()); } + +__attribute__ ((target ("sse2"), noinline)) +vespalib::hwaccelrated::IAccelrated::UP selectAccelrator() { return vespalib::hwaccelrated::IAccelrated::UP(new vespalib::hwaccelrated::Sse2Accelrator()); } + +__attribute__ ((target ("avx"), noinline)) +vespalib::hwaccelrated::IAccelrated::UP selectAccelrator() { return vespalib::hwaccelrated::IAccelrated::UP(new AvxAccelrator()); } +#endif + +namespace { + +class Factory { +public: + virtual ~Factory() { } + virtual IAccelrated::UP create() const = 0; +}; + +class GenericFactory :public Factory{ +public: + virtual IAccelrated::UP create() const { return IAccelrated::UP(new GenericAccelrator()); } +}; + +class Sse2Factory :public Factory{ +public: + virtual IAccelrated::UP create() const { return IAccelrated::UP(new Sse2Accelrator()); } +}; + +class AvxFactory :public Factory{ +public: + virtual IAccelrated::UP create() const { return IAccelrated::UP(new AvxAccelrator()); } +}; + +template<typename T> +void verifyAccelrator(const IAccelrated & accel) +{ + const size_t testLength(127); + T * a = new T[testLength]; + T * b = new T[testLength]; + for (size_t j(0); j < 0x20; j++) { + T sum(0); + for (size_t i(j); i < testLength; i++) { + a[i] = b[i] = i; + sum += i*i; + } + T hwComputedSum(accel.dotProduct(&a[j], &b[j], testLength - j)); + assert(sum == hwComputedSum); + } + delete [] a; + delete [] b; +} + +class RuntimeVerificator +{ +public: + RuntimeVerificator(); +}; + +RuntimeVerificator::RuntimeVerificator() +{ + GenericAccelrator generic; + verifyAccelrator<float>(generic); + verifyAccelrator<double>(generic); + verifyAccelrator<int32_t>(generic); + verifyAccelrator<int64_t>(generic); + + IAccelrated::UP thisCpu(IAccelrated::getAccelrator()); + verifyAccelrator<float>(*thisCpu); + verifyAccelrator<double>(*thisCpu); + verifyAccelrator<int32_t>(*thisCpu); + verifyAccelrator<int64_t>(*thisCpu); + +} + +class Selector +{ +public: + Selector() __attribute__((noinline)); + IAccelrated::UP create() { return _factory->create(); } +private: + std::unique_ptr<Factory> _factory; +}; + +Selector::Selector() : + _factory(new GenericFactory()) +{ + __builtin_cpu_init (); + if (__builtin_cpu_supports("avx")) { + _factory.reset(new AvxFactory()); + } else if (__builtin_cpu_supports("sse2")) { + _factory.reset(new Sse2Factory()); + } +} + +} + +static Selector _G_selector; + +RuntimeVerificator _G_verifyAccelrator; + + +IAccelrated::UP +IAccelrated::getAccelrator() +{ + return _G_selector.create(); +} + +} +} diff --git a/staging_vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h b/staging_vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h new file mode 100644 index 00000000000..b8c7794a386 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright (C) 2003 Fast Search & Transfer ASA +// Copyright (C) 2003 Overture Services Norway AS + +#pragma once + +#include <memory> +#include <stdint.h> + +namespace vespalib { + +namespace hwaccelrated { + +/** + * This contains an interface to all primitives that has different cpu supported accelrations. + * The actual implementation you get by calling the the static getAccelrator method. + */ +class IAccelrated +{ +public: + virtual ~IAccelrated() { } + typedef std::unique_ptr<IAccelrated> UP; + virtual float dotProduct(const float * a, const float * b, size_t sz) const = 0; + virtual double dotProduct(const double * a, const double * b, size_t sz) const = 0; + virtual int64_t dotProduct(const int32_t * a, const int32_t * b, size_t sz) const = 0; + virtual long long dotProduct(const int64_t * a, const int64_t * b, size_t sz) const = 0; + virtual void orBit(void * a, const void * b, size_t bytes) const = 0; + virtual void andBit(void * a, const void * b, size_t bytes) const = 0; + virtual void andNotBit(void * a, const void * b, size_t bytes) const = 0; + virtual void notBit(void * a, size_t bytes) const = 0; + + static IAccelrated::UP getAccelrator() __attribute__((noinline)); +}; + +} +} diff --git a/staging_vespalib/src/vespa/vespalib/hwaccelrated/sse2.cpp b/staging_vespalib/src/vespa/vespalib/hwaccelrated/sse2.cpp new file mode 100644 index 00000000000..11333d62f78 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/hwaccelrated/sse2.cpp @@ -0,0 +1,84 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright (C) 2003 Fast Search & Transfer ASA +// Copyright (C) 2003 Overture Services Norway AS + +#include <vespa/vespalib/hwaccelrated/sse2.h> + +namespace vespalib { + +namespace hwaccelrated { + +namespace { + +bool validAlignment16(const void * p) { + return (reinterpret_cast<uint64_t>(p) & 0xful) == 0; +} + +bool validAlignment16(const void * a, const void * b) { + return validAlignment16(a) && validAlignment16(b); +} + +} + +float +Sse2Accelrator::dotProduct(const float * af, const float * bf, size_t sz) const +{ + if ( ! validAlignment16(af, bf)) { + return GenericAccelrator::dotProduct(af, bf, sz); + } + typedef float v4sf __attribute__ ((vector_size (16))); + const size_t ChunkSize(16); + const size_t VectorsPerChunk(ChunkSize/4); + v4sf partial[VectorsPerChunk] = { {0.0, 0.0, 0.0, 0.0}, {0.0, 0.0, 0.0, 0.0}, {0.0, 0.0, 0.0, 0.0}, {0.0, 0.0, 0.0, 0.0} }; + const v4sf * a = reinterpret_cast<const v4sf *>(af); + const v4sf * b = reinterpret_cast<const v4sf *>(bf); + + const size_t numChunks(sz/ChunkSize); + for (size_t i(0); i < numChunks; i++) { + for (size_t j(0); j < VectorsPerChunk; j++) { + partial[j] += a[VectorsPerChunk*i+j] * b[VectorsPerChunk*i+j]; + } + } + float sum(0); + for (size_t i(numChunks*ChunkSize); i < sz; i++) { + sum += af[i] * bf[i]; + } + for (size_t i(1); i < VectorsPerChunk; i++) { + partial[0] += partial[i]; + } + sum += partial[0][0] + partial[0][1] + partial[0][2] + partial[0][3]; + return sum; +} + +double +Sse2Accelrator::dotProduct(const double * af, const double * bf, size_t sz) const +{ + if ( ! validAlignment16(af, bf)) { + return GenericAccelrator::dotProduct(af, bf, sz); + } + typedef double v2sd __attribute__ ((vector_size (16))); + const size_t ChunkSize(8); + const size_t VectorsPerChunk(ChunkSize/2); + v2sd partial[VectorsPerChunk] = { {0.0, 0.0}, {0.0, 0.0}, {0.0, 0.0}, {0.0, 0.0} }; + const v2sd * a = reinterpret_cast<const v2sd *>(af); + const v2sd * b = reinterpret_cast<const v2sd *>(bf); + + const size_t numChunks(sz/ChunkSize); + for (size_t i(0); i < numChunks; i++) { + for (size_t j(0); j < VectorsPerChunk; j++) { + partial[j] += a[VectorsPerChunk*i+j] * b[VectorsPerChunk*i+j]; + } + } + double sum(0); + for (size_t i(numChunks*ChunkSize); i < sz; i++) { + sum += af[i] * bf[i]; + } + for (size_t i(1); i < VectorsPerChunk; i++) { + partial[0] += partial[i]; + } + sum += partial[0][0] + partial[0][1]; + return sum; +} + +} +} diff --git a/staging_vespalib/src/vespa/vespalib/hwaccelrated/sse2.h b/staging_vespalib/src/vespa/vespalib/hwaccelrated/sse2.h new file mode 100644 index 00000000000..86fbc41a486 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/hwaccelrated/sse2.h @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright (C) 2003 Fast Search & Transfer ASA +// Copyright (C) 2003 Overture Services Norway AS + +#pragma once + +#include <vespa/vespalib/hwaccelrated/generic.h> + +namespace vespalib { + +namespace hwaccelrated { + +/** + * Generic cpu agnostic implementation. + */ +class Sse2Accelrator : public GenericAccelrator +{ +public: + virtual float dotProduct(const float * a, const float * b, size_t sz) const; + virtual double dotProduct(const double * a, const double * b, size_t sz) const; +}; + +} +} diff --git a/staging_vespalib/src/vespa/vespalib/net/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/net/CMakeLists.txt new file mode 100644 index 00000000000..f576a391b35 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/CMakeLists.txt @@ -0,0 +1,16 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(staging_vespalib_vespalib_net OBJECT + SOURCES + generic_state_handler.cpp + http_server.cpp + json_handler_repo.cpp + simple_component_config_producer.cpp + simple_health_producer.cpp + simple_metric_snapshot.cpp + simple_metrics_producer.cpp + slime_explorer.cpp + state_api.cpp + state_explorer.cpp + state_server.cpp + DEPENDS +) diff --git a/staging_vespalib/src/vespa/vespalib/net/component_config_producer.h b/staging_vespalib/src/vespa/vespalib/net/component_config_producer.h new file mode 100644 index 00000000000..19f13f4c470 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/component_config_producer.h @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +namespace vespalib { + +struct ComponentConfigProducer { + struct Config { + vespalib::string name; + size_t gen; + vespalib::string msg; + Config(const vespalib::string &n, size_t g) : name(n), gen(g), msg() {} + Config(const vespalib::string &n, size_t g, const vespalib::string &m) + : name(n), gen(g), msg(m) {} + }; + struct Consumer { + virtual void add(const Config &config) = 0; + virtual ~Consumer() {} + }; + virtual void getComponentConfig(Consumer &consumer) = 0; + virtual ~ComponentConfigProducer() {} +}; + +} // namespace vespalib + diff --git a/staging_vespalib/src/vespa/vespalib/net/generic_state_handler.cpp b/staging_vespalib/src/vespa/vespalib/net/generic_state_handler.cpp new file mode 100644 index 00000000000..b0310c91c64 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/generic_state_handler.cpp @@ -0,0 +1,121 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "generic_state_handler.h" +#include <vespa/vespalib/data/slime/slime.h> + +namespace vespalib { + +namespace { + +std::vector<vespalib::string> split_path(const vespalib::string &path) { + vespalib::string tmp; + std::vector<vespalib::string> items; + for (size_t i = 0; (i < path.size()) && (path[i] != '?'); ++i) { + if (path[i] == '/') { + if (!tmp.empty()) { + items.push_back(tmp); + tmp.clear(); + } + } else { + tmp.push_back(path[i]); + } + } + if (!tmp.empty()) { + items.push_back(tmp); + } + return items; +} + +bool is_prefix(const std::vector<vespalib::string> &root, const std::vector<vespalib::string> &full) { + if (root.size() > full.size()) { + return false; + } + for (size_t i = 0; i < root.size(); ++i) { + if (root[i] != full[i]) { + return false; + } + } + return true; +} + +vespalib::string make_url(const vespalib::string &host, const std::vector<vespalib::string> &items) { + vespalib::string url = "http://" + host; + if (items.empty()) { + url += "/"; + } + for (const vespalib::string &item: items) { + url += "/" + item; + } + return url; +} + +void inject_children(const StateExplorer &state, const vespalib::string &url, slime::Cursor &self); + +Slime child_state(const StateExplorer &state, const vespalib::string &url) { + Slime child_state; + state.get_state(slime::SlimeInserter(child_state), false); + if (child_state.get().type().getId() == slime::NIX::ID) { + inject_children(state, url, child_state.setObject()); + } else { + child_state.get().setString("url", url); + } + return child_state; +} + +void inject_children(const StateExplorer &state, const vespalib::string &url, slime::Cursor &self) { + std::vector<vespalib::string> children_names = state.get_children_names(); + for (const vespalib::string &child_name: children_names) { + std::unique_ptr<StateExplorer> child = state.get_child(child_name); + if (child) { + vespalib::string child_url = url + "/" + child_name; + Slime fragment = child_state(*child, child_url); + slime::inject(fragment.get(), slime::ObjectInserter(self, child_name)); + } + } +} + +vespalib::string render(const StateExplorer &state, const vespalib::string &url) { + Slime top; + state.get_state(slime::SlimeInserter(top), true); + if (top.get().type().getId() == slime::NIX::ID) { + top.setObject(); + } + inject_children(state, url, top.get()); + slime::SimpleBuffer buf; + slime::JsonFormat::encode(top, buf, true); + return buf.get().make_string(); +} + +vespalib::string explore(const StateExplorer &state, const vespalib::string &host, + const std::vector<vespalib::string> &items, size_t pos) { + if (pos == items.size()) { + return render(state, make_url(host, items)); + } + std::unique_ptr<StateExplorer> child = state.get_child(items[pos]); + if (!child) { + return ""; + } + return explore(*child, host, items, pos + 1); +} + +} // namespace vespalib::<unnamed> + +GenericStateHandler::GenericStateHandler(const vespalib::string &root_path, const StateExplorer &state) + : _root(split_path(root_path)), + _state(state) +{ +} + +vespalib::string +GenericStateHandler::get(const vespalib::string &host, + const vespalib::string &path, + const std::map<vespalib::string,vespalib::string> &) const +{ + std::vector<vespalib::string> items = split_path(path); + if (!is_prefix(_root, items)) { + return ""; + } + return explore(_state, host, items, _root.size()); +} + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/generic_state_handler.h b/staging_vespalib/src/vespa/vespalib/net/generic_state_handler.h new file mode 100644 index 00000000000..2e9880da6ee --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/generic_state_handler.h @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "json_get_handler.h" +#include "state_explorer.h" +#include <vespa/vespalib/stllike/string.h> +#include <vector> +#include <map> + +namespace vespalib { + +/** + * An implementation of the json get handler interface that exposes + * the state represented by the given state explorer as a browsable + * REST sub-API located below the given root path. + **/ +class GenericStateHandler : public JsonGetHandler +{ +private: + std::vector<vespalib::string> _root; + const StateExplorer &_state; + +public: + GenericStateHandler(const vespalib::string &root_path, const StateExplorer &state); + virtual vespalib::string get(const vespalib::string &host, + const vespalib::string &path, + const std::map<vespalib::string,vespalib::string> ¶ms) const override; +}; + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/health_producer.h b/staging_vespalib/src/vespa/vespalib/net/health_producer.h new file mode 100644 index 00000000000..f61afa3b716 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/health_producer.h @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +namespace vespalib { + +struct HealthProducer { + struct Health { + bool ok; + vespalib::string msg; + Health(bool o, const vespalib::string &m) : ok(o), msg(m) {} + }; + virtual Health getHealth() const = 0; + virtual ~HealthProducer() {} +}; + +} // namespace vespalib + diff --git a/staging_vespalib/src/vespa/vespalib/net/http_server.cpp b/staging_vespalib/src/vespa/vespalib/net/http_server.cpp new file mode 100644 index 00000000000..0696d65a3cb --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/http_server.cpp @@ -0,0 +1,82 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "http_server.h" +#include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/host_name.h> +#include <algorithm> + +namespace vespalib { + +namespace { + +void respond_not_found(Fast_HTTPConnection &conn) { + conn.Output(conn.GetHTTPVersion().c_str()); + conn.Output(" 404 Not Found\r\n"); + conn.Output("Connection: close\r\n\r\n"); +} + +void write_json_header(Fast_HTTPConnection &conn) { + conn.Output(conn.GetHTTPVersion().c_str()); + conn.Output(" 200 OK\r\n"); + conn.Output("Connection: close\r\n"); + conn.Output("Content-Type: application/json\r\n\r\n"); +} + +} // namespace vespalib::<unnamed> + +void +HttpServer::handle_get(const string &url, const string &host_in, + Fast_HTTPConnection &conn) const +{ + std::map<vespalib::string,vespalib::string> params; + vespalib::string json_result = _handler_repo.get(host_in.empty() ? _my_host : host_in, url, params); + if (json_result.empty()) { + respond_not_found(conn); + } else { + write_json_header(conn); + conn.OutputData(json_result.data(), json_result.size()); + } +} + +//----------------------------------------------------------------------------- + +HttpServer::HttpServer(int port_in) + : _requested_port(port_in), + _started(false), + _actual_port(0), + _my_host(), + _handler_repo(), + _server(new Server(port_in, *this)) +{ + _server->SetKeepAlive(false); +} + +void +HttpServer::start() +{ + if (_started) { + return; + } + int ret_code = _server->Start(); + if (ret_code != FASTLIB_SUCCESS && + ret_code != FASTLIB_HTTPSERVER_ALREADYSTARTED) + { + if (ret_code == FASTLIB_HTTPSERVER_BADLISTEN) { + throw PortListenException(_requested_port, "HTTP"); + } else { + throw FatalException("failed to start vespalib HTTP server"); + } + } + _actual_port = _server->getListenPort(); + _my_host = make_string("%s:%d", HostName::get().c_str(), _actual_port); + _started = true; +} + +void +HttpServer::stop() +{ + _server.reset(nullptr); +} + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/http_server.h b/staging_vespalib/src/vespa/vespalib/net/http_server.h new file mode 100644 index 00000000000..a889dd9e091 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/http_server.h @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/fastlib/net/httpserver.h> +#include "json_handler_repo.h" + +namespace vespalib { + +/** + * A simple HTTP server that can be used to handle GET requests + * returning json (typically simple read-only REST APIs). Either pass + * a specific port to the constructor or use 0 to bind to a random + * port. Note that you may not ask about the actual port until after + * the server has been started. Request dispatching is done using a + * JsonHandlerRepo. + **/ +class HttpServer +{ +private: + struct Server : Fast_HTTPServer { + typedef std::unique_ptr<Server> UP; + const HttpServer &parent; + virtual void onGetRequest(const vespalib::string &url, const vespalib::string &host, + Fast_HTTPConnection &conn) override + { + parent.handle_get(url, host, conn); + } + Server(int port, const HttpServer &parent_in) : Fast_HTTPServer(port), parent(parent_in) {} + }; + + int _requested_port; + volatile bool _started; + int _actual_port; + vespalib::string _my_host; + JsonHandlerRepo _handler_repo; + Server::UP _server; // need separate object for controlled shutdown + + void handle_get(const vespalib::string &url, const vespalib::string &host, + Fast_HTTPConnection &conn) const; +public: + typedef std::unique_ptr<HttpServer> UP; + HttpServer(int port_in); + const vespalib::string &host() const { return _my_host; } + JsonHandlerRepo &repo() { return _handler_repo; } + void start(); + int port() const { return (_actual_port != 0) ? _actual_port : _requested_port; } + void stop(); +}; + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/json_get_handler.h b/staging_vespalib/src/vespa/vespalib/net/json_get_handler.h new file mode 100644 index 00000000000..836b5b4d67a --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/json_get_handler.h @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <map> + +namespace vespalib { + +struct JsonGetHandler { + virtual vespalib::string get(const vespalib::string &host, + const vespalib::string &path, + const std::map<vespalib::string,vespalib::string> ¶ms) const = 0; + virtual ~JsonGetHandler() {} +}; + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/json_handler_repo.cpp b/staging_vespalib/src/vespa/vespalib/net/json_handler_repo.cpp new file mode 100644 index 00000000000..7ab6d8f1e75 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/json_handler_repo.cpp @@ -0,0 +1,88 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "json_handler_repo.h" +#include <algorithm> + +namespace vespalib { + +namespace { + +template <typename T> +void remove_seq(T &collection, size_t seq) { + collection.erase(std::remove_if(collection.begin(), collection.end(), + [seq](const typename T::value_type &item) + { return (item.seq == seq); }), + collection.end()); +} + +} // namespace vespalib::<unnamed> + +size_t +JsonHandlerRepo::State::bind(const vespalib::stringref &path_prefix, + const JsonGetHandler &get_handler) +{ + std::lock_guard<std::mutex> guard(lock); + size_t my_seq = ++seq; + hooks.emplace_back(my_seq, path_prefix, get_handler); + std::sort(hooks.begin(), hooks.end()); + return my_seq; +} + +size_t +JsonHandlerRepo::State::add_root_resource(const vespalib::stringref &path) +{ + std::lock_guard<std::mutex> guard(lock); + size_t my_seq = ++seq; + root_resources.emplace_back(my_seq, path); + return my_seq; +} + +void +JsonHandlerRepo::State::unbind(size_t my_seq) { + std::lock_guard<std::mutex> guard(lock); + remove_seq(hooks, my_seq); + remove_seq(root_resources, my_seq); +} + +//----------------------------------------------------------------------------- + +JsonHandlerRepo::Token::UP +JsonHandlerRepo::bind(const vespalib::stringref &path_prefix, + const JsonGetHandler &get_handler) +{ + return Token::UP(new Unbinder(_state, _state->bind(path_prefix, get_handler))); +} + +JsonHandlerRepo::Token::UP +JsonHandlerRepo::add_root_resource(const vespalib::stringref &path) +{ + return Token::UP(new Unbinder(_state, _state->add_root_resource(path))); +} + +std::vector<vespalib::string> +JsonHandlerRepo::get_root_resources() const +{ + std::lock_guard<std::mutex> guard(_state->lock); + std::vector<vespalib::string> result; + for (const Resource &resource: _state->root_resources) { + result.push_back(resource.path); + } + return result; +} + +vespalib::string +JsonHandlerRepo::get(const vespalib::string &host, + const vespalib::string &path, + const std::map<vespalib::string,vespalib::string> ¶ms) const +{ + std::lock_guard<std::mutex> guard(_state->lock); + for (const auto &hook: _state->hooks) { + if (path.find(hook.path_prefix) == 0) { + return hook.handler->get(host, path, params); + } + } + return ""; +} + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/json_handler_repo.h b/staging_vespalib/src/vespa/vespalib/net/json_handler_repo.h new file mode 100644 index 00000000000..0e27d895f52 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/json_handler_repo.h @@ -0,0 +1,90 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "json_get_handler.h" +#include <mutex> +#include <memory> + +namespace vespalib { + +/** + * A repository of json get handlers that is also a json get + * handler. The get function will dispatch the request to the + * appropriate get handler in the repository. The bind function will + * register a handler and return a token that can later be deleted to + * unbind the handler. Each handler is registered with a path + * prefix. If the requested path matches multiple handlers, the one + * with the longest prefix will be selected. If multiple handlers are + * tied for longest prefix, the most recently added handler will be + * selected. + **/ +class JsonHandlerRepo : public JsonGetHandler +{ +public: + struct Token { + typedef std::unique_ptr<Token> UP; + virtual ~Token() {} + }; + +private: + struct Hook { + size_t seq; + vespalib::string path_prefix; + const JsonGetHandler *handler; + Hook(size_t seq_in, + const vespalib::stringref &prefix_in, + const JsonGetHandler &handler_in) + : seq(seq_in), path_prefix(prefix_in), handler(&handler_in) {} + bool operator <(const Hook &rhs) const { + if (path_prefix.size() == rhs.path_prefix.size()) { + return (seq > rhs.seq); + } + return (path_prefix.size() > rhs.path_prefix.size()); + } + }; + + struct Resource { + size_t seq; + vespalib::string path; + Resource(size_t seq_in, vespalib::stringref path_in) + : seq(seq_in), path(path_in) {} + }; + + struct State { + typedef std::shared_ptr<State> SP; + std::mutex lock; + size_t seq; + std::vector<Hook> hooks; + std::vector<Resource> root_resources; + State() : lock(), seq(0), hooks(), root_resources() {} + size_t bind(const vespalib::stringref &path_prefix, + const JsonGetHandler &get_handler); + size_t add_root_resource(const vespalib::stringref &path); + void unbind(size_t my_seq); + }; + + struct Unbinder : Token { + State::SP state; + size_t my_seq; + Unbinder(State::SP state_in, size_t my_seq_in) + : state(state_in), my_seq(my_seq_in) {} + ~Unbinder() override { + state->unbind(my_seq); + } + }; + + std::shared_ptr<State> _state; + +public: + JsonHandlerRepo() : _state(std::make_shared<State>()) {} + Token::UP bind(const vespalib::stringref &path_prefix, + const JsonGetHandler &get_handler); + Token::UP add_root_resource(const vespalib::stringref &path); + std::vector<vespalib::string> get_root_resources() const; + vespalib::string get(const vespalib::string &host, + const vespalib::string &path, + const std::map<vespalib::string,vespalib::string> ¶ms) const override; +}; + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/metrics_producer.h b/staging_vespalib/src/vespa/vespalib/net/metrics_producer.h new file mode 100644 index 00000000000..bb222057f0b --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/metrics_producer.h @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +namespace vespalib { + +struct MetricsProducer { + virtual vespalib::string getMetrics(const vespalib::string &consumer) = 0; + virtual vespalib::string getTotalMetrics(const vespalib::string &consumer) = 0; + virtual ~MetricsProducer() {} +}; + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_component_config_producer.cpp b/staging_vespalib/src/vespa/vespalib/net/simple_component_config_producer.cpp new file mode 100644 index 00000000000..7f873e26dd1 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/simple_component_config_producer.cpp @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "simple_component_config_producer.h" + +namespace vespalib { + +SimpleComponentConfigProducer::SimpleComponentConfigProducer() + : _lock(), + _state() +{ +} + +void +SimpleComponentConfigProducer::addConfig(const Config &config) +{ + LockGuard guard(_lock); + _state.insert(std::make_pair(config.name, config)).first->second = config; +} + +void +SimpleComponentConfigProducer::removeConfig(const vespalib::string &name) +{ + LockGuard guard(_lock); + _state.erase(name); +} + +void +SimpleComponentConfigProducer::getComponentConfig(Consumer &consumer) +{ + typedef std::map<vespalib::string, Config>::const_iterator ITR; + LockGuard guard(_lock); + for (ITR itr = _state.begin(); itr != _state.end(); ++itr) { + consumer.add(itr->second); + } +} + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_component_config_producer.h b/staging_vespalib/src/vespa/vespalib/net/simple_component_config_producer.h new file mode 100644 index 00000000000..8561b6fb008 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/simple_component_config_producer.h @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "component_config_producer.h" +#include <vespa/vespalib/util/sync.h> +#include <map> + +namespace vespalib { + +class SimpleComponentConfigProducer : public ComponentConfigProducer +{ +private: + Lock _lock; + std::map<vespalib::string, Config> _state; + +public: + SimpleComponentConfigProducer(); + void addConfig(const Config &config); + void removeConfig(const vespalib::string &name); + virtual void getComponentConfig(Consumer &consumer); +}; + +} // namespace vespalib + diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_health_producer.cpp b/staging_vespalib/src/vespa/vespalib/net/simple_health_producer.cpp new file mode 100644 index 00000000000..002710b6e14 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/simple_health_producer.cpp @@ -0,0 +1,36 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "simple_health_producer.h" + +namespace vespalib { + +SimpleHealthProducer::SimpleHealthProducer() + : _lock(), + _health(true, "") +{ + setOk(); +} + +void +SimpleHealthProducer::setOk() +{ + LockGuard guard(_lock); + _health = Health(true, "All OK"); +} + +void +SimpleHealthProducer::setFailed(const vespalib::string &msg) +{ + LockGuard guard(_lock); + _health = Health(false, msg); +} + +HealthProducer::Health +SimpleHealthProducer::getHealth() const +{ + LockGuard guard(_lock); + return _health; +} + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_health_producer.h b/staging_vespalib/src/vespa/vespalib/net/simple_health_producer.h new file mode 100644 index 00000000000..254b4ce29b7 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/simple_health_producer.h @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "health_producer.h" +#include <vespa/vespalib/util/sync.h> + +namespace vespalib { + +class SimpleHealthProducer : public HealthProducer +{ +private: + Lock _lock; + HealthProducer::Health _health; + +public: + SimpleHealthProducer(); + void setOk(); + void setFailed(const vespalib::string &msg); + virtual Health getHealth() const; +}; + +} // namespace vespalib + diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_metric_snapshot.cpp b/staging_vespalib/src/vespa/vespalib/net/simple_metric_snapshot.cpp new file mode 100644 index 00000000000..de9db86e69a --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/simple_metric_snapshot.cpp @@ -0,0 +1,51 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "simple_metric_snapshot.h" + +namespace vespalib { + +SimpleMetricSnapshot::SimpleMetricSnapshot(uint32_t prevTime, uint32_t currTime) + : _data(), + _metrics(_data.setObject()), + _values(_metrics.setArray("values")), + _snapLen(currTime - prevTime) +{ + vespalib::slime::Cursor& snapshot = _metrics.setObject("snapshot"); + snapshot.setLong("from", prevTime); + snapshot.setLong("to", currTime); + if (_snapLen < 1.0) { + _snapLen = 1.0; + } +} + + +void +SimpleMetricSnapshot::addCount(const char *name, const char *desc, uint32_t count) +{ + using namespace vespalib::slime::convenience; + Cursor& value = _values.addObject(); + value.setString("name", name); + value.setString("description", desc); + Cursor& inner = value.setObject("values"); + inner.setLong("count", count); + inner.setDouble("rate", count / _snapLen); +} + +void +SimpleMetricSnapshot::addGauge(const char *name, const char *desc, long gauge) +{ + using namespace vespalib::slime::convenience; + Cursor& value = _values.addObject(); + value.setString("name", name); + value.setString("description", desc); + Cursor& inner = value.setObject("values"); + inner.setLong("average", gauge); + inner.setLong("min", gauge); + inner.setLong("max", gauge); + inner.setLong("last", gauge); + inner.setLong("count", 1); + inner.setDouble("rate", 1.0 / _snapLen); +} + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_metric_snapshot.h b/staging_vespalib/src/vespa/vespalib/net/simple_metric_snapshot.h new file mode 100644 index 00000000000..605f51f55af --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/simple_metric_snapshot.h @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/data/slime/slime.h> + +namespace vespalib { + +class SimpleMetricSnapshot +{ +private: + vespalib::Slime _data; + vespalib::slime::Cursor& _metrics; + vespalib::slime::Cursor& _values; + double _snapLen; + +public: + SimpleMetricSnapshot(uint32_t prevTime, uint32_t currTime); + void addCount(const char *name, const char *desc, uint32_t count); + void addGauge(const char *name, const char *desc, long gauge); + + vespalib::string asString() const { + return _data.toString(); + } +}; + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_metrics_producer.cpp b/staging_vespalib/src/vespa/vespalib/net/simple_metrics_producer.cpp new file mode 100644 index 00000000000..586226d5277 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/simple_metrics_producer.cpp @@ -0,0 +1,43 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "simple_metrics_producer.h" + +namespace vespalib { + +SimpleMetricsProducer::SimpleMetricsProducer() + : _lock(), + _metrics(), + _totalMetrics() +{ +} + +void +SimpleMetricsProducer::setMetrics(const vespalib::string &metrics) +{ + LockGuard guard(_lock); + _metrics = metrics; +} + +vespalib::string +SimpleMetricsProducer::getMetrics(const vespalib::string &) +{ + LockGuard guard(_lock); + return _metrics; +} + +void +SimpleMetricsProducer::setTotalMetrics(const vespalib::string &metrics) +{ + LockGuard guard(_lock); + _totalMetrics = metrics; +} + +vespalib::string +SimpleMetricsProducer::getTotalMetrics(const vespalib::string &) +{ + LockGuard guard(_lock); + return _totalMetrics; +} + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_metrics_producer.h b/staging_vespalib/src/vespa/vespalib/net/simple_metrics_producer.h new file mode 100644 index 00000000000..67e34e1ae4e --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/simple_metrics_producer.h @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "metrics_producer.h" +#include <vespa/vespalib/util/sync.h> + +namespace vespalib { + +class SimpleMetricsProducer : public MetricsProducer +{ +private: + Lock _lock; + vespalib::string _metrics; + vespalib::string _totalMetrics; + +public: + SimpleMetricsProducer(); + void setMetrics(const vespalib::string &metrics); + virtual vespalib::string getMetrics(const vespalib::string &consumer) override; + void setTotalMetrics(const vespalib::string &metrics); + virtual vespalib::string getTotalMetrics(const vespalib::string &consumer) override; +}; + +} // namespace vespalib + diff --git a/staging_vespalib/src/vespa/vespalib/net/slime_explorer.cpp b/staging_vespalib/src/vespa/vespalib/net/slime_explorer.cpp new file mode 100644 index 00000000000..a99ef240833 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/slime_explorer.cpp @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "slime_explorer.h" +#include <vespa/vespalib/data/slime/inject.h> + +namespace vespalib { + +namespace { + +struct SelfState : slime::ObjectTraverser { + Slime result; + SelfState() : result() { result.setObject(); } + virtual void field(const slime::Memory &key, const slime::Inspector &value) { + if (value.type().getId() != slime::OBJECT::ID) { + slime::inject(value, slime::ObjectInserter(result.get(), key)); + } + } +}; + +struct ChildrenNames : slime::ObjectTraverser { + std::vector<vespalib::string> result; + virtual void field(const slime::Memory &key, const slime::Inspector &value) { + if (value.type().getId() == slime::OBJECT::ID) { + result.push_back(key.make_string()); + } + } +}; + +} // namespace vespalib::<unnamed> + +void +SlimeExplorer::get_state(const slime::Inserter &inserter, bool full) const +{ + SelfState state; + _self.traverse(state); + if (state.result.get().fields() > 0) { + if (full) { + state.result.get().setBool("full", true); + } + slime::inject(state.result.get(), inserter); + } +} + +std::vector<vespalib::string> +SlimeExplorer::get_children_names() const +{ + ChildrenNames names; + _self.traverse(names); + return names.result; +} + +std::unique_ptr<StateExplorer> +SlimeExplorer::get_child(vespalib::stringref name) const +{ + slime::Inspector &child = _self[name]; + if (!child.valid()) { + return std::unique_ptr<StateExplorer>(nullptr); + } + return std::unique_ptr<StateExplorer>(new SlimeExplorer(child)); +} + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/slime_explorer.h b/staging_vespalib/src/vespa/vespalib/net/slime_explorer.h new file mode 100644 index 00000000000..798ce132c9e --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/slime_explorer.h @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "state_explorer.h" +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vector> +#include <memory> + +namespace vespalib { + +/** + * Simple class exposing the contents of a Slime object through the + * StateExplorer interface (to be used when testing clients of the + * StateExplorer interface). + **/ +class SlimeExplorer : public StateExplorer +{ +private: + const slime::Inspector &_self; + +public: + SlimeExplorer(const slime::Inspector &self) : _self(self) {} + virtual void get_state(const slime::Inserter &inserter, bool full) const override; + virtual std::vector<vespalib::string> get_children_names() const override; + virtual std::unique_ptr<StateExplorer> get_child(vespalib::stringref name) const override; +}; + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/state_api.cpp b/staging_vespalib/src/vespa/vespalib/net/state_api.cpp new file mode 100644 index 00000000000..50a8a969e9c --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/state_api.cpp @@ -0,0 +1,166 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "state_api.h" +#include <vespa/vespalib/util/jsonwriter.h> + +namespace vespalib { + +namespace { + +struct ConfigRenderer : ComponentConfigProducer::Consumer { + JSONStringer &json; + ConfigRenderer(JSONStringer &j) : json(j) {} + virtual void add(const ComponentConfigProducer::Config &config) { + json.appendKey(config.name); + json.beginObject(); + json.appendKey("generation"); + json.appendInt64(config.gen); + if (!config.msg.empty()) { + json.appendKey("message"); + json.appendString(config.msg); + } + json.endObject(); + } +}; + +struct ConfigGenerationObserver : ComponentConfigProducer::Consumer { + size_t maxGen; + bool seenSome; + ConfigGenerationObserver() : maxGen(0), seenSome(false) {} + virtual void add(const ComponentConfigProducer::Config &config) { + if (seenSome) { + maxGen = std::max(maxGen, config.gen); + } else { + maxGen = config.gen; + seenSome = true; + } + } +}; + +void build_health_status(JSONStringer &json, const HealthProducer &healthProducer) { + HealthProducer::Health health = healthProducer.getHealth(); + json.appendKey("status"); + json.beginObject(); + json.appendKey("code"); + if (health.ok) { + json.appendString("up"); + } else { + json.appendString("down"); + json.appendKey("message"); + json.appendString(health.msg); + } + json.endObject(); +} + +vespalib::string get_consumer(const std::map<vespalib::string,vespalib::string> ¶ms) { + auto consumer_lookup = params.find("consumer"); + if (consumer_lookup == params.end()) { + return ""; + } + return consumer_lookup->second; +} + +void render_link(JSONStringer &json, const vespalib::string &host, const vespalib::string &path) { + json.beginObject(); + json.appendKey("url"); + json.appendString("http://" + host + path); + json.endObject(); +} + +vespalib::string respond_root(const JsonHandlerRepo &repo, const vespalib::string &host) { + JSONStringer json; + json.beginObject(); + json.appendKey("resources"); + json.beginArray(); + for (auto path: {"/state/v1/health", "/state/v1/metrics", "/state/v1/config"}) { + render_link(json, host, path); + } + for (const vespalib::string &path: repo.get_root_resources()) { + render_link(json, host, path); + } + json.endArray(); + json.endObject(); + return json.toString(); +} + +vespalib::string respond_health(const HealthProducer &healthProducer) { + JSONStringer json; + json.beginObject(); + build_health_status(json, healthProducer); + json.endObject(); + return json.toString(); +} + +vespalib::string respond_metrics(const vespalib::string &consumer, + const HealthProducer &healthProducer, + MetricsProducer &metricsProducer) +{ + JSONStringer json; + json.beginObject(); + build_health_status(json, healthProducer); + { // metrics + vespalib::string metrics = metricsProducer.getMetrics(consumer); + if (!metrics.empty()) { + json.appendKey("metrics"); + json.appendJSON(metrics); + } + } + json.endObject(); + return json.toString(); +} + +vespalib::string respond_config(ComponentConfigProducer &componentConfigProducer) { + JSONStringer json; + json.beginObject(); + { // config + ConfigRenderer renderer(json); + json.appendKey("config"); + json.beginObject(); + ConfigGenerationObserver observer; + componentConfigProducer.getComponentConfig(observer); + if (observer.seenSome) { + json.appendKey("generation"); + json.appendInt64(observer.maxGen); + } + componentConfigProducer.getComponentConfig(renderer); + json.endObject(); + } + json.endObject(); + return json.toString(); +} + +} // namespace vespalib::<unnamed> + +vespalib::string +StateApi::get(const vespalib::string &host, + const vespalib::string &path, + const std::map<vespalib::string,vespalib::string> ¶ms) const +{ + if (path == "/state/v1/" || path == "/state/v1") { + return respond_root(_handler_repo, host); + } else if (path == "/state/v1/health") { + return respond_health(_healthProducer); + } else if (path == "/state/v1/metrics") { + return respond_metrics(get_consumer(params), _healthProducer, _metricsProducer); + } else if (path == "/state/v1/config") { + return respond_config(_componentConfigProducer); + } else if (path == "/metrics/total") { + return _metricsProducer.getTotalMetrics(get_consumer(params)); + } else { + return _handler_repo.get(host, path, params); + } +} + +//----------------------------------------------------------------------------- + +StateApi::StateApi(const HealthProducer &hp, + MetricsProducer &mp, + ComponentConfigProducer &ccp) + : _healthProducer(hp), + _metricsProducer(mp), + _componentConfigProducer(ccp) +{ +} + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/state_api.h b/staging_vespalib/src/vespa/vespalib/net/state_api.h new file mode 100644 index 00000000000..565fa9469fd --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/state_api.h @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "json_get_handler.h" +#include "health_producer.h" +#include "metrics_producer.h" +#include "component_config_producer.h" +#include <memory> +#include "json_handler_repo.h" + +namespace vespalib { + +/** + * This class uses the underlying producer interfaces passed to the + * constructor to implement the 'state' REST API. The get function is + * a simple abstraction of a GET request returning json and can be + * wired into the HttpServer or called directly. + **/ +class StateApi : public JsonGetHandler +{ +private: + const HealthProducer &_healthProducer; + MetricsProducer &_metricsProducer; + ComponentConfigProducer &_componentConfigProducer; + JsonHandlerRepo _handler_repo; + +public: + StateApi(const HealthProducer &hp, + MetricsProducer &mp, + ComponentConfigProducer &ccp); + virtual vespalib::string get(const vespalib::string &host, + const vespalib::string &path, + const std::map<vespalib::string,vespalib::string> ¶ms) const override; + JsonHandlerRepo &repo() { return _handler_repo; } +}; + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/state_explorer.cpp b/staging_vespalib/src/vespa/vespalib/net/state_explorer.cpp new file mode 100644 index 00000000000..d3a03882f37 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/state_explorer.cpp @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "state_explorer.h" + +namespace vespalib { + +std::vector<vespalib::string> +StateExplorer::get_children_names() const +{ + return std::vector<vespalib::string>(); +} + +std::unique_ptr<StateExplorer> +StateExplorer::get_child(vespalib::stringref) const +{ + return std::unique_ptr<StateExplorer>(nullptr); +} + +StateExplorer::~StateExplorer() +{ +} + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/state_explorer.h b/staging_vespalib/src/vespa/vespalib/net/state_explorer.h new file mode 100644 index 00000000000..7955d49d7e1 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/state_explorer.h @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/data/slime/inserter.h> +#include <vector> +#include <memory> + +namespace vespalib { + +/** + * Interface used to traverse and expose state of a given component and its children. + */ +struct StateExplorer { + virtual void get_state(const slime::Inserter &inserter, bool full) const = 0; + virtual std::vector<vespalib::string> get_children_names() const; + virtual std::unique_ptr<StateExplorer> get_child(vespalib::stringref name) const; + virtual ~StateExplorer(); +}; + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/state_server.cpp b/staging_vespalib/src/vespa/vespalib/net/state_server.cpp new file mode 100644 index 00000000000..8529c0c1f84 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/state_server.cpp @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "state_server.h" + +namespace vespalib { + +StateServer::StateServer(int port, + const HealthProducer &hp, + MetricsProducer &mp, + ComponentConfigProducer &ccp) + : _api(hp, mp, ccp), + _server(port), + _tokens() +{ + _tokens.push_back(_server.repo().bind("/state/v1", _api)); + _tokens.push_back(_server.repo().bind("/metrics/total", _api)); + _server.start(); +} + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/state_server.h b/staging_vespalib/src/vespa/vespalib/net/state_server.h new file mode 100644 index 00000000000..63ee899fd84 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/net/state_server.h @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "http_server.h" +#include "state_api.h" +#include "json_get_handler.h" +#include "health_producer.h" +#include "metrics_producer.h" +#include "component_config_producer.h" +#include <memory> +#include "json_handler_repo.h" + +namespace vespalib { + +/** + * An all-in-one server making it simple for applications to serve the + * 'state' REST API over HTTP. + **/ +class StateServer +{ +private: + StateApi _api; + HttpServer _server; + std::vector<JsonHandlerRepo::Token::UP> _tokens; + +public: + typedef std::unique_ptr<StateServer> UP; + StateServer(int port, + const HealthProducer &hp, + MetricsProducer &mp, + ComponentConfigProducer &ccp); + int getListenPort() { return _server.port(); } + JsonHandlerRepo &repo() { return _api.repo(); } +}; + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/objects/.gitignore b/staging_vespalib/src/vespa/vespalib/objects/.gitignore new file mode 100644 index 00000000000..ee8938b6bf4 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/.gitignore @@ -0,0 +1,6 @@ +*.So +*.exe +*.ilk +*.pdb +.depend* +Makefile diff --git a/staging_vespalib/src/vespa/vespalib/objects/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/objects/CMakeLists.txt new file mode 100644 index 00000000000..093d5e2151f --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/CMakeLists.txt @@ -0,0 +1,18 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(staging_vespalib_vespalib_objects OBJECT + SOURCES + identifiable.cpp + namedobject.cpp + objectvisitor.cpp + objectdumper.cpp + visit.cpp + objectpredicate.cpp + objectoperation.cpp + fieldbase.cpp + nboserializer.cpp + serializer.cpp + deserializer.cpp + asciiserializer.cpp + floatingpointtype.cpp + DEPENDS +) diff --git a/staging_vespalib/src/vespa/vespalib/objects/asciiserializer.cpp b/staging_vespalib/src/vespa/vespalib/objects/asciiserializer.cpp new file mode 100644 index 00000000000..10c26eea3cf --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/asciiserializer.cpp @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/objects/asciiserializer.h> + +namespace vespalib { + +AsciiSerializer &AsciiSerializer::put(const IFieldBase &, bool value) { + _stream << value; + return *this; +} + +AsciiSerializer &AsciiSerializer::put(const IFieldBase &, uint8_t value) { + _stream << value; + return *this; +} + +AsciiSerializer &AsciiSerializer::put(const IFieldBase &, uint16_t value) { + _stream << value; + return *this; +} + +AsciiSerializer &AsciiSerializer::put(const IFieldBase &, uint32_t value) { + _stream << value; + return *this; +} + +AsciiSerializer &AsciiSerializer::put(const IFieldBase &, uint64_t value) { + _stream << value; + return *this; +} + +AsciiSerializer &AsciiSerializer::put(const IFieldBase &, float value) { + _stream << value; + return *this; +} + +AsciiSerializer &AsciiSerializer::put(const IFieldBase &, double value) { + _stream << value; + return *this; +} + +AsciiSerializer &AsciiSerializer::put(const IFieldBase &, const stringref & value) { + _stream << value; + return *this; +} + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/objects/asciiserializer.h b/staging_vespalib/src/vespa/vespalib/objects/asciiserializer.h new file mode 100644 index 00000000000..06283ba1b61 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/asciiserializer.h @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/objects/serializer.h> +#include <vespa/vespalib/objects/deserializer.h> +#include <vespa/vespalib/stllike/asciistream.h> + +namespace vespalib { +class AsciiSerializer : public Serializer { +public: + AsciiSerializer(asciistream &stream) : _stream(stream) { } + virtual AsciiSerializer &put(const IFieldBase &field, bool value); + virtual AsciiSerializer &put(const IFieldBase &field, uint8_t value); + virtual AsciiSerializer &put(const IFieldBase &field, uint16_t value); + virtual AsciiSerializer &put(const IFieldBase &field, uint32_t value); + virtual AsciiSerializer &put(const IFieldBase &field, uint64_t value); + virtual AsciiSerializer &put(const IFieldBase &field, float value); + virtual AsciiSerializer &put(const IFieldBase &field, double value); + virtual AsciiSerializer &put(const IFieldBase &field, const stringref &val); + + const asciistream &getStream() const { return _stream; } + asciistream &getStream() { return _stream; } +private: + asciistream &_stream; +}; + +} // namespace vespalib + diff --git a/staging_vespalib/src/vespa/vespalib/objects/cloneable.h b/staging_vespalib/src/vespa/vespalib/objects/cloneable.h new file mode 100644 index 00000000000..c0b12272797 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/cloneable.h @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +/** + * @class vespalib::Cloneable + * @brief Superclass for objects implementing clone() deep copy. + */ + +#include <memory> + +namespace vespalib { + +class Cloneable { +public: + /** + * @brief Creates a clone of this instance. + * + * Note that the caller takes ownership of the returned object. It + * is not an unique_ptr since that would not support covariant + * return types. A class T that inherits this interface should + * define T* clone() const, such that people cloning a T object + * don't need to cast it to get the correct type. + */ + virtual Cloneable* clone() const = 0; + virtual ~Cloneable() {} +}; + +} // namespace vespalib + diff --git a/staging_vespalib/src/vespa/vespalib/objects/deserializer.cpp b/staging_vespalib/src/vespa/vespalib/objects/deserializer.cpp new file mode 100644 index 00000000000..3a87c616362 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/deserializer.cpp @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "deserializer.h" +#include "identifiable.h" + +namespace vespalib { + +Deserializer & Deserializer::get(const IFieldBase & field, Identifiable & value) +{ + (void) field; + return value.deserializeDirect(*this); +} +Deserializer & Deserializer::get(const IFieldBase & field, int8_t & value) +{ + uint8_t v(0); + get(field, v); + value = v; + return *this; +} +Deserializer & Deserializer::get(const IFieldBase & field, int16_t & value) +{ + uint16_t v(0); + get(field, v); + value = v; + return *this; +} +Deserializer & Deserializer::get(const IFieldBase & field, int32_t & value) +{ + uint32_t v(0); + get(field, v); + value = v; + return *this; +} +Deserializer & Deserializer::get(const IFieldBase & field, int64_t & value) +{ + uint64_t v(0); + get(field, v); + value = v; + return *this; +} +Deserializer & Deserializer::get(const IFieldBase & field, std::string & value) +{ + string v; + get(field, v); + value = v; + return *this; +} +} diff --git a/staging_vespalib/src/vespa/vespalib/objects/deserializer.h b/staging_vespalib/src/vespa/vespalib/objects/deserializer.h new file mode 100644 index 00000000000..3e3eb3cc526 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/deserializer.h @@ -0,0 +1,83 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/objects/fieldbase.h> +#include <vespa/vespalib/util/array.h> +#include <vector> +#include <stdint.h> + +namespace vespalib +{ + +class Identifiable; + +class Deserializer : virtual SerializerCommon +{ +public: + virtual ~Deserializer() { } + virtual Deserializer & get(const IFieldBase & field, bool & value) = 0; + virtual Deserializer & get(const IFieldBase & field, uint8_t & value) = 0; + virtual Deserializer & get(const IFieldBase & field, uint16_t & value) = 0; + virtual Deserializer & get(const IFieldBase & field, uint32_t & value) = 0; + virtual Deserializer & get(const IFieldBase & field, uint64_t & value) = 0; + virtual Deserializer & get(const IFieldBase & field, double & value) = 0; + virtual Deserializer & get(const IFieldBase & field, float & value) = 0; + virtual Deserializer & get(const IFieldBase & field, string & value) = 0; + + virtual Deserializer & get(const IFieldBase & field, Identifiable & value); + virtual Deserializer & get(const IFieldBase & field, int8_t & value); + virtual Deserializer & get(const IFieldBase & field, int16_t & value); + virtual Deserializer & get(const IFieldBase & field, int32_t & value); + virtual Deserializer & get(const IFieldBase & field, int64_t & value); + + uint8_t getBool(const IFieldBase & field) { bool v; get(field, v); return v; } + uint8_t getUInt8(const IFieldBase & field) { uint8_t v; get(field, v); return v; } + int8_t getInt8(const IFieldBase & field) { int8_t v; get(field, v); return v; } + uint16_t getUInt16(const IFieldBase & field) { uint16_t v; get(field, v); return v; } + int16_t getInt16(const IFieldBase & field) { int16_t v; get(field, v); return v; } + uint32_t getUInt32(const IFieldBase & field) { uint32_t v; get(field, v); return v; } + int32_t getInt32(const IFieldBase & field) { int32_t v; get(field, v); return v; } + uint64_t getUInt64(const IFieldBase & field) { uint64_t v; get(field, v); return v; } + int64_t getInt64(const IFieldBase & field) { int64_t v; get(field, v); return v; } + float getFloat(const IFieldBase & field) { float v; get(field, v); return v; } + double getDouble(const IFieldBase & field) { double v; get(field, v); return v; } + string getString(const IFieldBase & field) { string v; get(field, v); return v; } + + Deserializer & get(const IFieldBase & field, std::string & value); + Deserializer & operator >> (bool & value) { return get(_unspecifiedField, value); } + Deserializer & operator >> (uint8_t & value) { return get(_unspecifiedField, value); } + Deserializer & operator >> (int8_t & value) { return get(_unspecifiedField, value); } + Deserializer & operator >> (uint16_t & value) { return get(_unspecifiedField, value); } + Deserializer & operator >> (int16_t & value) { return get(_unspecifiedField, value); } + Deserializer & operator >> (uint32_t & value) { return get(_unspecifiedField, value); } + Deserializer & operator >> (int32_t & value) { return get(_unspecifiedField, value); } + Deserializer & operator >> (uint64_t & value) { return get(_unspecifiedField, value); } + Deserializer & operator >> (int64_t & value) { return get(_unspecifiedField, value); } + Deserializer & operator >> (float & value) { return get(_unspecifiedField, value); } + Deserializer & operator >> (double & value) { return get(_unspecifiedField, value); } + Deserializer & operator >> (string & value) { return get(_unspecifiedField, value); } + template <typename T> + Deserializer & operator >> (vespalib::Array<T> & v) { + uint32_t sz; + get(_sizeField, sz); + v.resize(sz); + for(size_t i(0); i < sz; i++) { + (*this) >> v[i]; + } + return *this; + } + template <typename T> + Deserializer & operator >> (std::vector<T> & v) { + uint32_t sz; + get(_sizeField, sz); + v.resize(sz); + for(size_t i(0); i < sz; i++) { + (*this) >> v[i]; + } + return *this; + } + +}; + +} + diff --git a/staging_vespalib/src/vespa/vespalib/objects/fieldbase.cpp b/staging_vespalib/src/vespa/vespalib/objects/fieldbase.cpp new file mode 100644 index 00000000000..a8011adf5ad --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/fieldbase.cpp @@ -0,0 +1,10 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "fieldbase.h" + +namespace vespalib { + +FieldBase SerializerCommon::_unspecifiedField("unspecified"); +FieldBase SerializerCommon::_sizeField("size"); + +} diff --git a/staging_vespalib/src/vespa/vespalib/objects/fieldbase.h b/staging_vespalib/src/vespa/vespalib/objects/fieldbase.h new file mode 100644 index 00000000000..9592566b7d4 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/fieldbase.h @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> + +namespace vespalib +{ + +class IFieldBase +{ +public: + virtual ~IFieldBase() { } + virtual stringref getName() const = 0; +}; + +class FieldBase : public IFieldBase +{ +public: + FieldBase(stringref name) : _name(name) { } + virtual stringref getName() const { return _name; } +private: + string _name; +}; + +class SerializerCommon +{ +protected: + static FieldBase _unspecifiedField; + static FieldBase _sizeField; +}; + +} + diff --git a/staging_vespalib/src/vespa/vespalib/objects/floatingpointtype.cpp b/staging_vespalib/src/vespa/vespalib/objects/floatingpointtype.cpp new file mode 100644 index 00000000000..6d3c93dbb66 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/floatingpointtype.cpp @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/objects/floatingpointtype.h> +#include <iostream> + +namespace vespalib { + +template<typename Number> +std::ostream& operator<<(std::ostream& out, FloatingPointType<Number> number) +{ + return out << number.getValue(); +} + +template<typename Number> +vespalib::asciistream & operator<<(vespalib::asciistream & out, FloatingPointType<Number> number) +{ + return out << number.getValue(); +} + +template std::ostream& operator<<(std::ostream& out, FloatingPointType<float> number); +template std::ostream& operator<<(std::ostream& out, FloatingPointType<double> number); + +template vespalib::asciistream& operator<<(vespalib::asciistream& out, FloatingPointType<float> number); +template vespalib::asciistream& operator<<(vespalib::asciistream& out, FloatingPointType<double> number); + +} diff --git a/staging_vespalib/src/vespa/vespalib/objects/floatingpointtype.h b/staging_vespalib/src/vespa/vespalib/objects/floatingpointtype.h new file mode 100644 index 00000000000..49d13f5e35d --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/floatingpointtype.h @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +/** + * \class vespalib::FloatingPointType + * \ingroup object + * + * \brief Wrapper class for floating point types taking care of comparisons. + * + * Due to floating point values not being able to represent a lot of numbers + * exactly, one should always compare floating point values, allowing slight + * variations. + * + * To avoid having to handle this everywhere in the code, some wrapper classes + * exist here to take care of this for you. They will automatically convert + * primitive values to the wrapper class, so you can still use primitive + * constants in your code. + * + * Node that epsilon is currently just set to 10^(-6). We can reduce it or + * adjust it based on type or value later if we see the need. But the class + * should define it, at least by default, to make the interface easy to use as + * most use cases don't really care that much. + */ + +#pragma once + +#include <vespa/vespalib/stllike/asciistream.h> +#include <iosfwd> // To get std::ostream for output operator + +namespace vespalib { + +template<typename Number> +class FloatingPointType { + Number _value; + +public: + typedef FloatingPointType<Number> Type; + + FloatingPointType() : _value(0.0) {} + FloatingPointType(Number n) : _value(n) {} + + Number getValue() const { return _value; } + + Number abs() const { return (_value < 0 ? -1 * _value : _value); } + + bool operator==(Type n) const { return ((*this - n).abs() < 0.000001); } + bool operator!=(Type n) const { return ((*this - n).abs() > 0.000001); } + bool operator<(Type n) const { return (n._value - 0.000001 > _value); } + bool operator>(Type n) const { return (n._value + 0.000001 < _value); } + bool operator<=(Type n) const { return (n._value + 0.000001 > _value); } + bool operator>=(Type n) const { return (n._value - 0.000001 < _value); } + + Type operator-(Type n) const { return Type(_value - n._value); } + Type operator+(Type n) const { return Type(_value + n._value); } + Type operator*(Type n) const { return Type(_value * n._value); } + Type operator/(Type n) const { return Type(_value / n._value); } + + Type& operator+=(Type n) { _value += n._value; return *this; } + Type& operator-=(Type n) { _value -= n._value; return *this; } + Type& operator*=(Type n) { _value *= n._value; return *this; } + Type& operator/=(Type n) { _value /= n._value; return *this; } + + Type& operator++() { ++_value; return *this; } + Type operator++(int) { Type t(_value); ++_value; return t; } + Type& operator--() { --_value; return *this; } + Type operator--(int) { Type t(_value); --_value; return t; } +}; + +typedef FloatingPointType<double> Double; +typedef FloatingPointType<double> Float; + +template<typename Number> +std::ostream& operator<<(std::ostream& out, FloatingPointType<Number> number); + +template<typename Number> +vespalib::asciistream & operator<<(vespalib::asciistream & out, FloatingPointType<Number> number); + +} // vespalib + diff --git a/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp b/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp new file mode 100644 index 00000000000..bac071c8048 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp @@ -0,0 +1,267 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <cassert> +#include <vespa/vespalib/util/stringfmt.h> +#include <stdexcept> +#include <algorithm> +#include <vespa/vespalib/objects/identifiable.h> +#include "objectdumper.h" +#include "visit.h" +#include "objectpredicate.h" +#include "objectoperation.h" +#include <cxxabi.h> + +namespace vespalib { + +Identifiable::Register * Identifiable::_register = NULL; +Identifiable::ILoader * Identifiable::_classLoader = NULL; +FieldBase Identifiable::hasObjectField("hasObject"); +FieldBase Identifiable::sizeField("size"); +FieldBase Identifiable::classIdField("classId"); +FieldBase Identifiable::objectField("object"); + +IMPLEMENT_IDENTIFIABLE(Identifiable, Identifiable); + +Identifiable::RuntimeClass::RuntimeClass(RuntimeInfo * info_) : + _rt(info_) +{ + if (_rt->_factory) { + Identifiable::UP tmp(create()); + assert(id() == tmp->getClass().id()); + //printf("Class %s has typeinfo %s\n", name(), typeid(*tmp).name()); + for (const RuntimeInfo * curr = _rt; curr && curr != curr->_base; curr = curr->_base) { + //printf("\tinherits %s : typeinfo = %s\n", curr->_name, curr->_typeId().name()); + if ( ! curr->_tryCast(tmp.get()) ) { + throw std::runtime_error(make_string("(%s, %s) is not a baseclass of (%s, %s)", curr->_name, curr->_typeId().name(), name(), typeid(*tmp).name())); + } + } + } + if (Identifiable::_register == NULL) { + Identifiable::_register = new Register(); + } + if (! Identifiable::_register->append(this)) { + const RuntimeClass * old = Identifiable::_register->classFromId(id()); + throw std::runtime_error(make_string("Duplicate Identifiable object(%s, %s, %d) being registered. Choose a unique id. Object (%s, %s, %d) is using it.", name(), info(), id(), old->name(), old->info(), old->id())); + } +} + +Identifiable::RuntimeClass::~RuntimeClass() +{ + if ( ! Identifiable::_register->erase(this) ) { + assert(0); + } + if (Identifiable::_register->empty()) { + delete Identifiable::_register; + Identifiable::_register = NULL; + } +} + +bool Identifiable::RuntimeClass::inherits(unsigned cid) const +{ + const RuntimeInfo *cur; + for (cur = _rt; (cur != &Identifiable::_RTInfo) && cid != cur->_id; cur = cur->_base) { } + return (cid == cur->_id); +} + +Identifiable::Register::Register() : + _listById(), + _listByName() +{ +} + +Identifiable::Register::~Register() +{ +} + +bool Identifiable::Register::erase(Identifiable::RuntimeClass * c) +{ + _listById.erase(c); + _listByName.erase(c); + return true; +} + +class SortById : public std::binary_function<const Identifiable::RuntimeClass *, const Identifiable::RuntimeClass *, bool> { +public: + bool operator() (const Identifiable::RuntimeClass * x, const Identifiable::RuntimeClass * y) const { + return x->id() < y->id(); + } +}; + +class SortByName : public std::binary_function<const Identifiable::RuntimeClass *, const Identifiable::RuntimeClass *, bool> { +public: + bool operator() (const Identifiable::RuntimeClass * x, const Identifiable::RuntimeClass * y) const { + return strcmp(x->name(), y->name()) < 0; + } +}; + +bool Identifiable::Register::append(Identifiable::RuntimeClass * c) +{ + bool ok((_listById.find(c) == _listById.end()) && ((_listByName.find(c) == _listByName.end()))); + if (ok) { + _listById.insert(c); + _listByName.insert(c); + } + return ok; +} + +const Identifiable::RuntimeClass * Identifiable::Register::classFromId(unsigned id) const +{ + IdList::const_iterator it(_listById.find<uint32_t, GetId, hash<uint32_t>, std::equal_to<uint32_t> >(id)); + return (it != _listById.end()) ? *it : NULL; +} + +const Identifiable::RuntimeClass * Identifiable::Register::classFromName(const char *name) const +{ + NameList::const_iterator it(_listByName.find<const char *, GetName, hash<const char *>, std::equal_to<const char *> >(name)); + return (it != _listByName.end()) ? *it : NULL; +} + +Serializer & operator << (Serializer & os, const Identifiable & obj) +{ + os.put(Identifiable::classIdField, obj.getClass().id()); + obj.serialize(os); + return os; +} + +nbostream & operator << (nbostream & os, const Identifiable & obj) +{ + NBOSerializer nos(os); + nos << obj; + return os; +} + +nbostream & operator >> (nbostream & is, Identifiable & obj) +{ + NBOSerializer nis(is); + nis >> obj; + return is; +} + +Deserializer & operator >> (Deserializer & os, Identifiable & obj) +{ + uint32_t cid(0); + os.get(Identifiable::classIdField, cid); + if (cid == obj.getClass().id()) { + obj.deserialize(os); + } else { + throw std::runtime_error(make_string("Failed deserializing %s : Received cid %d(%0x) != %d(%0x)", + obj.getClass().name(), + cid, cid, + obj.getClass().id(), obj.getClass().id())); + //Should mark as failed + } + return os; +} + +Identifiable::UP Identifiable::create(Deserializer & is) +{ + uint32_t cid(0); + is.get(classIdField, cid); + UP obj; + const Identifiable::RuntimeClass *rtc = Identifiable::classFromId(cid); + if (rtc == NULL) { + if ((_classLoader != NULL) && _classLoader->hasClass(cid)) { + _classLoader->loadClass(cid); + rtc = Identifiable::classFromId(cid); + if (rtc == NULL) { + throw std::runtime_error(make_string("Failed loading class for Identifiable with classId %d(%0x)", cid, cid)); + } + } + } + if (rtc != NULL) { + obj.reset(rtc->create()); + if (obj.get()) { + obj->deserialize(is); + } else { + throw std::runtime_error( + make_string("Failed deserializing an Identifiable for classId %d(%0x). " + "It is abstract, so it can not be instantiated. Does it need to be abstract ?", + cid, cid)); + } + } else { + throw std::runtime_error(make_string("Failed deserializing an Identifiable with unknown classId %d(%0x)", cid, cid)); + } + return obj; +} + +string +Identifiable::getNativeClassName() const +{ + string name(typeid(*this).name()); + int status = 0; + size_t size = 0; + char *unmangled = abi::__cxa_demangle(name.c_str(), 0, &size, &status); + string result(unmangled); + free(unmangled); + return result; +} + +string +Identifiable::asString() const +{ + ObjectDumper dumper; + visit(dumper, "", this); + return dumper.toString(); +} + +int Identifiable::onCmp(const Identifiable& b) const +{ + int diff(0); + nbostream as, bs; + NBOSerializer nas(as), nbs(bs); + nas << *this; + nbs << b; + size_t minLength(std::min(as.size(), bs.size())); + if (minLength > 0) { + diff = memcmp(as.c_str(), bs.c_str(), minLength); + } + if (diff == 0) { + diff = as.size() - bs.size(); + } + return diff; +} + +void +Identifiable::visitMembers(ObjectVisitor &visitor) const +{ + visitor.visitNotImplemented(); +} + +void +Identifiable::select(const ObjectPredicate &predicate, ObjectOperation &operation) +{ + if (predicate.check(*this)) { + operation.execute(*this); + } else { + selectMembers(predicate, operation); + } +} + +void +Identifiable::selectMembers(const ObjectPredicate &predicate, ObjectOperation &operation) +{ + (void) predicate; + (void) operation; +} + +Serializer & Identifiable::serialize(Serializer & os) const +{ + return os.put(objectField, *this); +} + +Deserializer & Identifiable::deserialize(Deserializer & is) +{ + return is.get(objectField, *this); +} + +Serializer & Identifiable::onSerialize(Serializer & os) const +{ + return os; +} + +Deserializer & Identifiable::onDeserialize(Deserializer & is) +{ + return is; +} + +} diff --git a/staging_vespalib/src/vespa/vespalib/objects/identifiable.h b/staging_vespalib/src/vespa/vespalib/objects/identifiable.h new file mode 100644 index 00000000000..67f5917c68d --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/identifiable.h @@ -0,0 +1,566 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +/** + * \class vespalib::Identifiable + * \ingroup util + * + * \brief Superclass for objects adding some runtime type information. + * + * This class is a superclass used by many other classes to add some runtime + * information. + * + * This can be used to verify type to be able to do cheap static casts + * instead of dynamic casts. It can be used to identify type of object, and + * it can also be used to generate an object of the given type (If it's not + * an identifiable abstract type). + * + */ + +#include <string> +#include <vector> +#include <memory> +#include <typeinfo> + +#define CID_Identifiable 1 + +#include "ids.h" +#include "nboserializer.h" +#include "objectvisitor.h" + +#include <vespa/vespalib/util/memory.h> +#include <vespa/vespalib/util/linkedptr.h> +#include <vespa/vespalib/stllike/hash_set.h> + +#define IDENTIFIABLE_CLASSID(cclass) CID_##cclass +#define IDENTIFIABLE_CLASSID_NS(ns, cclass) CID_##ns##_##cclass +#define IDENTIFIABLE_CLASSID_NS2(ns1, ns2, cclass) CID_##ns1##_##ns2##_##cclass +#define IDENTIFIABLE_CLASSID_NS3(ns1, ns2, ns3, cclass) CID_##ns1##_##ns2##_##ns3##_##cclass + +#define DECLARE_IDENTIFIABLE_BASE_COMMON(cclass) \ + static vespalib::Identifiable::RuntimeInfo _RTInfo; \ + static vespalib::Identifiable::RuntimeClass _RTClass; \ + static const std::type_info & typeId() { return typeid(cclass); } \ + static bool tryCast(const vespalib::Identifiable * v) { return dynamic_cast<const cclass *>(v) != NULL; } \ + static cclass *identifyClassAsIdentifiable() { return NULL; } /* no implementation */ \ + virtual const vespalib::Identifiable::RuntimeClass & getClass() const; + +#define DECLARE_IDENTIFIABLE_BASE(cclass, classid) \ + public: \ + enum { classId=classid }; \ + DECLARE_IDENTIFIABLE_BASE_COMMON(cclass) + +#define DECLARE_IDENTIFIABLE_ABSTRACT(cclass) DECLARE_IDENTIFIABLE_BASE(cclass, IDENTIFIABLE_CLASSID(cclass)) +#define DECLARE_IDENTIFIABLE_ABSTRACT_NS(ns, cclass) DECLARE_IDENTIFIABLE_BASE(ns::cclass, IDENTIFIABLE_CLASSID_NS(ns, cclass)) +#define DECLARE_IDENTIFIABLE_ABSTRACT_NS2(ns1, ns2, cclass) DECLARE_IDENTIFIABLE_BASE(ns1::ns2::cclass, IDENTIFIABLE_CLASSID_NS2(ns1, ns2, cclass)) +#define DECLARE_IDENTIFIABLE_ABSTRACT_NS3(ns1, ns2, ns3, cclass) DECLARE_IDENTIFIABLE_BASE(ns1::ns2::ns3::cclass, IDENTIFIABLE_CLASSID_NS3(ns1, ns2, ns3, cclass)) + +#define DECLARE_IDENTIFIABLE_COMMON(cclass) \ + virtual void assign(const vespalib::Identifiable & rhs); \ + static cclass * create() { return new cclass(); } \ + static Identifiable * createAsIdentifiable() { return cclass::create(); } + +#define DECLARE_IDENTIFIABLE(cclass) \ + DECLARE_IDENTIFIABLE_BASE(cclass, IDENTIFIABLE_CLASSID(cclass)) \ + DECLARE_IDENTIFIABLE_COMMON(cclass) + +#define DECLARE_IDENTIFIABLE_NS(ns, cclass) \ + DECLARE_IDENTIFIABLE_BASE(ns::cclass, IDENTIFIABLE_CLASSID_NS(ns, cclass)) \ + DECLARE_IDENTIFIABLE_COMMON(ns::cclass) + +#define DECLARE_IDENTIFIABLE_NS2(ns1, ns2, cclass) \ + DECLARE_IDENTIFIABLE_BASE(ns1::ns2::cclass, IDENTIFIABLE_CLASSID_NS2(ns1, ns2, cclass)) \ + DECLARE_IDENTIFIABLE_COMMON(ns1::ns2::cclass) + +#define DECLARE_IDENTIFIABLE_NS3(ns1, ns2, ns3, cclass) \ + DECLARE_IDENTIFIABLE_BASE(ns1::ns2::ns3::cclass, IDENTIFIABLE_CLASSID_NS3(ns1, ns2, ns3, cclass)) \ + DECLARE_IDENTIFIABLE_COMMON(ns1::ns2::ns3::cclass) + +#define IMPLEMENT_IDENTIFIABLE_COMMON(cclass) \ + vespalib::Identifiable::RuntimeClass cclass::_RTClass(&_RTInfo); \ + const vespalib::Identifiable::RuntimeClass & cclass::getClass() const { return _RTClass; } + +#define IMPLEMENT_IDENTIFIABLE_CONCRET(cclass) \ + void cclass::assign(const vespalib::Identifiable & rhs) { \ + if (rhs.inherits(classId)) { \ + *this = static_cast<const cclass &>(rhs); \ + } \ + } + +#define IMPLEMENT_IDENTIFIABLE_BASE(cclass, name, base, id, factory, ctypeInfo, tryCast, typeinfo) \ + vespalib::Identifiable::RuntimeInfo cclass::_RTInfo = {name, typeinfo, id, factory, ctypeInfo, tryCast, &base::_RTInfo }; \ + IMPLEMENT_IDENTIFIABLE_COMMON(cclass) + +#define IMPLEMENT_IDENTIFIABLE_ABSTRACT(cclass, base) \ + IMPLEMENT_IDENTIFIABLE_BASE(cclass, #cclass, base, IDENTIFIABLE_CLASSID(cclass), \ + NULL, cclass::typeId, cclass::tryCast, "") +#define IMPLEMENT_IDENTIFIABLE(cclass, base) \ + IMPLEMENT_IDENTIFIABLE_CONCRET(cclass) \ + IMPLEMENT_IDENTIFIABLE_BASE(cclass, #cclass, base, IDENTIFIABLE_CLASSID(cclass), \ + cclass::createAsIdentifiable, cclass::typeId, cclass::tryCast, "") +#define IMPLEMENT_IDENTIFIABLE_ABSTRACT_NS(ns, cclass, base) \ + IMPLEMENT_IDENTIFIABLE_BASE(ns::cclass, #ns"::"#cclass, base, IDENTIFIABLE_CLASSID_NS(ns, cclass), \ + NULL, cclass::typeId, cclass::tryCast, "") +#define IMPLEMENT_IDENTIFIABLE_NS(ns, cclass, base) \ + IMPLEMENT_IDENTIFIABLE_CONCRET(ns::cclass) \ + IMPLEMENT_IDENTIFIABLE_BASE(ns::cclass, #ns"::"#cclass, base, IDENTIFIABLE_CLASSID_NS(ns, cclass), \ + cclass::createAsIdentifiable, cclass::typeId, cclass::tryCast, "") +#define IMPLEMENT_IDENTIFIABLE_ABSTRACT_NS2(ns1, ns2, cclass, base) \ + IMPLEMENT_IDENTIFIABLE_BASE(ns1::ns2::cclass, #ns1"::"#ns2"::"#cclass, base, IDENTIFIABLE_CLASSID_NS2(ns1, ns2, cclass), \ + NULL, cclass::typeId, cclass::tryCast, "") +#define IMPLEMENT_IDENTIFIABLE_NS2(ns1, ns2, cclass, base) \ + IMPLEMENT_IDENTIFIABLE_CONCRET(ns1::ns2::cclass) \ + IMPLEMENT_IDENTIFIABLE_BASE(ns1::ns2::cclass, #ns1"::"#ns2"::"#cclass, base, IDENTIFIABLE_CLASSID_NS2(ns1, ns2, cclass), \ + cclass::createAsIdentifiable, cclass::typeId, cclass::tryCast, "") +#define IMPLEMENT_IDENTIFIABLE_ABSTRACT_NS3(ns1, ns2, ns3, cclass, base) \ + IMPLEMENT_IDENTIFIABLE_BASE(ns1::ns2::ns3::cclass, #ns1"::"#ns2"::"#ns3"::"#cclass, base, IDENTIFIABLE_CLASSID_NS3(ns1, ns2, ns3, cclass), \ + NULL, cclass::typeId, cclass::tryCast, "") +#define IMPLEMENT_IDENTIFIABLE_NS3(ns1, ns2, ns3, cclass, base) \ + IMPLEMENT_IDENTIFIABLE_CONCRET(ns1::ns2::ns3::cclass) \ + IMPLEMENT_IDENTIFIABLE_BASE(ns1::ns2::ns3::cclass, #ns1"::"#ns2"::"#ns3"::"#cclass, base, IDENTIFIABLE_CLASSID_NS3(ns1, ns2, ns3, cclass), \ + cclass::createAsIdentifiable, cclass::typeId, cclass::tryCast, "") + +#define DECLARE_NBO_SERIALIZE \ + virtual vespalib::Serializer & onSerialize(vespalib::Serializer & os) const; \ + virtual vespalib::Deserializer & onDeserialize(vespalib::Deserializer & is); + + +namespace vespalib +{ + +class ObjectPredicate; +class ObjectOperation; + +class Identifiable { + protected: + struct RuntimeInfo { + const char * _name; + const char * _info; + unsigned _id; + Identifiable * (* _factory)(); + const std::type_info & (* _typeId)(); + bool (* _tryCast)(const Identifiable *); + const RuntimeInfo * _base; + }; +public: + typedef std::unique_ptr<Identifiable> UP; + static FieldBase hasObjectField; + static FieldBase sizeField; + static FieldBase classIdField; + static FieldBase objectField; + class ILoader + { + public: + virtual ~ILoader() { } + virtual bool hasClass(unsigned classId) const = 0; + virtual bool hasClass(const char * className) const = 0; + virtual void loadClass(unsigned classId) = 0; + virtual void loadClass(const char * className) = 0; + }; + struct RuntimeClass : public IFieldBase { + public: + RuntimeClass(RuntimeInfo * info); + ~RuntimeClass(); + const char * name() const { return _rt->_name; } + const char * info() const { return _rt->_info; } + unsigned id() const { return _rt->_id; } + Identifiable * create() const { return _rt->_factory ? _rt->_factory() : 0; } + const std::type_info & typeId() const { return _rt->_typeId(); } + bool tryCast(const Identifiable *o) const { return _rt->_tryCast(o); } + const RuntimeInfo * base() const { return _rt->_base; } + bool inherits(unsigned id) const; + bool equal(unsigned cid) const { return id() == cid; } + int compare(const RuntimeClass& other) const { return (id() - other.id()); } + private: + virtual stringref getName() const { return stringref(_rt->_name); } + RuntimeInfo * _rt; + }; + DECLARE_IDENTIFIABLE(Identifiable); + virtual ~Identifiable() { } + + /** + * Will produce the full demangled className + */ + string getNativeClassName() const; + + /** + * This returns the innermost class that you represent. Default is that that is yourself. + * However when you are a vector containing other objects, it might be feasible + * to let the world know about them too. + * @return the class info for the innermost object. + */ + virtual const RuntimeClass & getBaseClass() const { return getClass(); } + /** + * Checks if this object inherits from a class with the given id. + * @param id The id of the class to check if is an anchestor. + * @return true if the object does inherit from it. Significantly faster than using dynamic cast. + */ + bool inherits(unsigned id) const { return getClass().inherits(id); } + /** + * Checks if this object inherits from a class with the given name. + * @param name The name of the class to check if is an anchestor. + * @return true if the object does inherit from it. Significantly faster than using dynamic cast. + */ + bool inherits(const char * name) const; + /** + * Identifiable::cast<T> behaves like dynamic_cast<T> when trying + * to cast between Identifiable objects, using the inherits() + * function defined above to check if the cast should succeed. + */ + template <typename T> struct BaseType { typedef T type; }; + template <typename T> struct BaseType<T &> { typedef T type; }; + template <typename T> struct BaseType<T *> { typedef T type; }; + template <typename T> struct BaseType<const T &> { typedef T type; }; + template <typename T> struct BaseType<const T *> { typedef T type; }; + template <typename Type> static void ERROR_Type_is_not_Identifiable() { + Type *(*foo)() = &Type::identifyClassAsIdentifiable; + (void) foo; + } + template <typename T, typename Base> + static T cast(Base &p) { + typedef typename BaseType<T>::type Type; + ERROR_Type_is_not_Identifiable<Type>(); // Help diagnose errors. + if (p.inherits(Type::classId)) { return static_cast<T>(p); } + else { throw std::bad_cast(); } + } + template <typename T, typename Base> + static T cast(Base *p) { + typedef typename BaseType<T>::type Type; + ERROR_Type_is_not_Identifiable<Type>(); // Help diagnose errors. + if (p && p->inherits(Type::classId)) { return static_cast<T>(p); } + else { return 0; } + } + /** + * Given the unique registered id of a class it will look up the object describing it. + * @return object describing the class. + */ + static const RuntimeClass * classFromId(unsigned id) { return _register->classFromId(id); } + /** + * Given the unique registered name of a class it will look up the object describing it. + * @return object describing the class. + */ + static const RuntimeClass * classFromName(const char * name) { return _register->classFromName(name); } + /** + * Here you can provide an optional classloader. + */ + static void registerClassLoader(ILoader & loader) { _classLoader = &loader; } + static void clearClassLoader() { _classLoader = NULL; } + + /** + * Create a human-readable representation of this object. This + * method will use object visitation internally to capture the + * full structure of this object. + * + * @return structured human-readable representation of this object + **/ + string asString() const; + + /** + * Visit each of the members of this object. This method should be + * overridden by subclasses and should present all appropriate + * internal structure of this object to the given visitor. Note + * that while each level of a class hierarchy may cooperate to + * visit all object members (invoking superclass method within + * method), this method, as implemented in the Identifiable class + * should not be invoked, since its default implementation is + * there to signal about the method not being overridden. + * + * @param visitor the visitor of this object + **/ + virtual void visitMembers(ObjectVisitor &visitor) const; + + /** + * Compares 2 objects. First tests classId, then lives it to the objects and onCmp + * if classes are the same. + * @param b the object to compare with. + * @return <0 if this comes before b, 0 for equality, and >0 fi b comes before *this. + */ + int cmp(const Identifiable& b) const { + int r(cmpClassId(b)); + return (r==0) ? onCmp(b) : r; + } + /** + * Compares 2 objects. This is faster method that cmp as classId tests is not done. + * onCmp is called directly. + * @param b the object to compare with. + * @return <0 if this comes before b, 0 for equality, and >0 fi b comes before *this. + */ + int cmpFast(const Identifiable& b) const { return onCmp(b); } + + /** + * Apply the predicate to this object. If the predicate returns + * true, pass this object to the operation, otherwise invoke the + * @ref selectMemebers method to locate sub-elements that might + * trigger the predicate. + * + * @param predicate component used to select (sub-)objects + * @param operation component performing some operation + * on the selected (sub-)objects + **/ + void select(const ObjectPredicate &predicate, + ObjectOperation &operation); + + /** + * Invoke @ref select on any member objects this object wants to + * expose through the selection mechanism. Overriding this method + * is optional, and which objects to expose is determined by the + * application logic of the object itself. + * + * @param predicate component used to select (sub-)objects + * @param operation component performing some operation + * on the selected (sub-)objects + **/ + virtual void selectMembers(const ObjectPredicate &predicate, + ObjectOperation &operation); + + /** + * This will serialize the object by calling the virtual onSerialize. + * The id of the object is not serialized. + * @param os The nbostream used for output. + * @return The nbostream after serialization. + */ + Serializer & serialize(Serializer & os) const; + /** + * This will deserialize the object by calling the virtual onDeserialize. + * It is symetric with serialize. + * @param is The nbostream used for input. + * @return The nbostream after deserialization. + */ + Deserializer & deserialize(Deserializer & is); + /** + * This will read the first 4 bytes containing the classid. It will then create the + * correct object based on that class, and then call deserialize with the rest. + * It is symetric with the << operator, and does the same as >>, except instead of checking the id + * it uses it to construct an object. + * @param is The nbostream used for input. + * @return The object created and constructed by deserialize. NULL if there are any errors. + */ + static UP create(Deserializer & is); + static UP create(nbostream & is) { NBOSerializer nis(is); return create(nis); } + /** + * This will serialize the object by first serializing the 4 byte classid. + * Then the rest is serialized by calling serialize. + * @param os The nbostream used for output. + * @param obj The object to serialize. + * @return The nbostream after serialization. + */ + friend Serializer & operator << (Serializer & os, const Identifiable & obj); + friend nbostream & operator << (nbostream & os, const Identifiable & obj); + /** + * This will read the first 4 bytes containing the classid. It will then check if it matches its own. + * if not it will mark the nbostream as bad. Then it will deserialize he content. + * It is symetric with the << operator + * @param is The nbostream used for input. + * @param obj The object that the stream will be deserialized into. + * @return The object created and constructed by deserialize. NULL if there are any errors. + */ + friend Deserializer & operator >> (Deserializer & os, Identifiable & obj); + friend nbostream & operator >> (nbostream & os, Identifiable & obj); + + /** + * This will serialize a 4 byte num element before it will serialize all elements. + * This is used when you have a vector of objects of known class. + * It is symetric with deserialize template below + * @param v is vector of non polymorf Identifiable. + * @param os is output stream + * @return outputstream + */ + template <typename T> + static Serializer & serialize(const T & v, Serializer & os); + + /** + * This will serialize a 4 byte num element before it will serialize all elements. + * This is used when you have a vector of objects of known class. + * It is symetric with deserialize template below + * @param v is vector of non polymorf Identifiable. + * @param os is output stream + * @return outputstream + */ + template <typename T> + static Deserializer & deserialize(T & v, Deserializer & is); + + Serializer & serializeDirect(Serializer & os) const { return onSerialize(os); } + Deserializer & deserializeDirect(Deserializer & is) { return onDeserialize(is); } +protected: + /** + * Returns the diff of the classid. Used for comparing classes. + * @param b the object to compare with. + * @return getClass().id() - b.getClass().id() + */ + int cmpClassId(const Identifiable& b) const + { return getClass().id() - b.getClass().id(); } + +private: + /** + * Interface for comparing objects to each other. Default implementation + * is to serialise the two objects and use memcmp to verify equality. + * Classes should overide this method if they have special needs. + * It might also be an idea to override for speed, as serialize is 'expensive'. + * @param b the object to compare with. + * @return <0 if this comes before b, 0 for equality, and >0 fi b comes before *this. + */ + virtual int onCmp(const Identifiable& b) const; + virtual Serializer & onSerialize(Serializer & os) const; + virtual Deserializer & onDeserialize(Deserializer & is); + + class Register { + public: + Register(); + ~Register(); + bool append(RuntimeClass * c); + bool erase(RuntimeClass * c); + const RuntimeClass * classFromId(unsigned id) const; + const RuntimeClass * classFromName(const char * name) const; + const char * id2Name(unsigned id) const; + unsigned name2Id(const char * name) const; + bool empty() const { return _listById.empty(); } + private: + struct GetId { uint32_t operator() (const RuntimeClass * f) const { return f->id(); } }; + struct HashId { size_t operator() (const RuntimeClass * f) const { return f->id(); } }; + struct EqualId { bool operator() (const RuntimeClass * a, const RuntimeClass * b) const { return a->id() == b->id(); } }; + struct GetName { const char * operator() (const RuntimeClass * f) const { return f->name(); } }; + struct HashName { size_t operator() (const RuntimeClass * f) const { return hashValue(f->name()); } }; + struct EqualName { bool operator() (const RuntimeClass * a, const RuntimeClass * b) const { return strcmp(a->name(), b->name()) == 0; } }; + typedef hash_set<RuntimeClass *, HashId, EqualId> IdList; + typedef hash_set<RuntimeClass *, HashName, EqualName> NameList; + IdList _listById; + NameList _listByName; + }; + static Register * _register; + static ILoader * _classLoader; +}; + +template <typename T> +Serializer & Identifiable::serialize(const T & v, Serializer & os) { + uint32_t sz(v.size()); + os.put(sizeField, sz); + for(size_t i(0); i < sz; i++) { + v[i].serialize(os); + } + return os; +} + +template <typename T> +Deserializer & Identifiable::deserialize(T & v, Deserializer & is) { + uint32_t sz(0); + is.get(sizeField, sz); + v.resize(sz); + for(size_t i(0); i < sz; i++) { + v[i].deserialize(is); + } + return is; +} + +template <typename T> +class IdentifiablePtr : public CloneablePtr<T> +{ +public: + IdentifiablePtr(const T &t) : CloneablePtr<T>(t.clone()) {} + IdentifiablePtr(T * p=NULL) : CloneablePtr<T>(p) { } + int cmp(const IdentifiablePtr<T> &rhs) const { + const T *a = this->get(); + const T *b = rhs.get(); + if (a == 0) { + return (b == 0) ? 0 : -1; + } + return (b == 0) ? 1 : a->cmp(*b); + } + bool operator < (const IdentifiablePtr<T> &rhs) const { return (cmp(rhs) < 0); } + bool operator > (const IdentifiablePtr<T> &rhs) const { return (cmp(rhs) > 0); } + bool operator == (const IdentifiablePtr<T> &rhs) const { return (cmp(rhs) == 0); } + bool operator != (const IdentifiablePtr<T> &rhs) const { return (cmp(rhs) != 0); } + Serializer & serialize(Serializer & os) const { + if (this->get()) { + os.put(Identifiable::hasObjectField, uint8_t(1)) << *this->get(); + } else { + os.put(Identifiable::hasObjectField, uint8_t(0)); + } + return os; + } + Deserializer & deserialize(Deserializer & is) { + uint8_t hasObject; + is.get(Identifiable::hasObjectField, hasObject); + if (hasObject) { + this->reset(static_cast<T *>(Identifiable::create(is).release())); + } + return is; + } + friend Serializer & operator << (Serializer & os, const IdentifiablePtr<T> & agg) { return agg.serialize(os); } + friend Deserializer & operator >> (Deserializer & is, IdentifiablePtr<T> & agg) { return agg.deserialize(is); } +}; + +template <typename T> +class IdentifiableSharedPtr : public std::shared_ptr<T> +{ +public: + IdentifiableSharedPtr(const T &t) : std::shared_ptr<T>(t.clone()) {} + IdentifiableSharedPtr(T * p=NULL) : std::shared_ptr<T>(p) { } + int cmp(const IdentifiableSharedPtr<T> &rhs) const { + const T *a = this->get(); + const T *b = rhs.get(); + if (a == 0) { + return (b == 0) ? 0 : -1; + } + return (b == 0) ? 1 : a->cmp(*b); + } + bool operator < (const IdentifiableSharedPtr<T> &rhs) const { + return (cmp(rhs) < 0); + } + Serializer & serialize(Serializer & os) const { + if (this->get()) { + os.put(Identifiable::hasObjectField, uint8_t(1)) << *this->get(); + } else { + os.put(Identifiable::hasObjectField, uint8_t(0)); + } + return os; + } + Deserializer & deserialize(Deserializer & is) { + uint8_t hasObject; + is.get(Identifiable::hasObjectField, hasObject); + if (hasObject) { + reset(static_cast<T *>(Identifiable::create(is).release())); + } + return is; + } + friend Serializer & operator << (Serializer & os, const IdentifiableSharedPtr<T> & agg) { return agg.serialize(os); } + friend Deserializer & operator >> (Deserializer & is, IdentifiableSharedPtr<T> & agg) { return agg.deserialize(is); } +}; + +template <typename T> +class IdentifiableLinkedPtr : public LinkedPtr<T> +{ +public: + IdentifiableLinkedPtr(const T &t) : LinkedPtr<T>(t.clone()) {} + IdentifiableLinkedPtr(T * p=NULL) : LinkedPtr<T>(p) { } + int cmp(const IdentifiableLinkedPtr<T> &rhs) const { + const T *a = this->get(); + const T *b = rhs.get(); + if (a == 0) { + return (b == 0) ? 0 : -1; + } + return (b == 0) ? 1 : a->cmp(*b); + } + bool operator < (const IdentifiableLinkedPtr<T> &rhs) const { + return (cmp(rhs) < 0); + } + Serializer & serialize(Serializer & os) const { + if (this->get()) { + os.put(Identifiable::hasObjectField, uint8_t(1)) << *this->get(); + } else { + os.put(Identifiable::hasObjectField, uint8_t(0)); + } + return os; + } + Deserializer & deserialize(Deserializer & is) { + uint8_t hasObject; + is.get(Identifiable::hasObjectField, hasObject); + if (hasObject) { + this->reset(static_cast<T *>(Identifiable::create(is).release())); + } + return is; + } + friend Serializer & operator << (Serializer & os, const IdentifiableLinkedPtr<T> & agg) { return agg.serialize(os); } + friend Deserializer & operator >> (Deserializer & is, IdentifiableLinkedPtr<T> & agg) { return agg.deserialize(is); } +}; + +} + diff --git a/staging_vespalib/src/vespa/vespalib/objects/ids.h b/staging_vespalib/src/vespa/vespalib/objects/ids.h new file mode 100644 index 00000000000..7cc6ab85648 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/ids.h @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +/* + * These are the namespaces of ids used by identifiable classes. + * Should correspond to actual C++ namespace names. + */ + +#define DOCUMENT_CID(v) (0x1000 + v) +#define STORAGEAPI_CID(v) (0x2000 + v) +#define STORAGECLIENT_CID(v) (0x3000 + v) +#define SEARCHLIB_CID(v) (0x4000 + v) +#define VESPALIB_CID(v) (0x5000 + v) +#define CONFIGD_CID(v) (0x6000 + v) +#define VESPA_CONFIGMODEL_CID(v) (0x7000 + v) + +/* + * Here are all the ids in the vespalib namespace: + */ + +#define CID_vespalib_NamedObject VESPALIB_CID(9) + diff --git a/staging_vespalib/src/vespa/vespalib/objects/memberinfo.h b/staging_vespalib/src/vespa/vespalib/objects/memberinfo.h new file mode 100644 index 00000000000..9c37554796b --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/memberinfo.h @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +namespace vespalib { + +#define MI_TYPEID(t) MemberInfo::##t##_id +#define MI_M_OFFSET(c, m) static_cast<long>(&static_cast<const c *>(0)->m) +#define MI_M_SIZEOF(c, m) sizeof(static_cast<const c *>(0)->m) +#define MI_BASE_OFFSET(c,b) static_cast<long>(static_cast<const b *>(static_cast<const * c>(1)) - 1) + +#define MI_DECLARE \ + static const MemberInfo _memberInfo[]; \ + static const unsigned _memberCount; + +#define MI_IMPL_BEGIN(c) const MemberInfo c::_memberInfo[] = { \ +#define MI_IMPL_MEM(c, t, m) {#m, MI_TYPEID(t), MI_M_OFFSET(c, m), sizeof(m) } +#define MI_IMPL_END(c) }; const unsigned c::_memberCount = sizeof(cl::__memberDescription)/sizeof(cl::__memberDescription[0]) + + +class MemberInfo { + public: + enum TypeId { + bool_id = 1, + char_id, + int8_t_id, + uint8_t_id, + int16_t_id, + uint16_t_id, + int32_t_id, + uint32_t_id, + int64_t_id, + uint64_t_id, + float_id, + double_id, + std_string_id, + vespalib_Identifiable_id + }; + const char * _name; + TypeId _type; + size_t _offset + size_t _sizeof; +}; + +} + diff --git a/staging_vespalib/src/vespa/vespalib/objects/namedobject.cpp b/staging_vespalib/src/vespa/vespalib/objects/namedobject.cpp new file mode 100644 index 00000000000..4424f3949a0 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/namedobject.cpp @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "namedobject.h" + +namespace vespalib { + +IMPLEMENT_IDENTIFIABLE_NS(vespalib, NamedObject, Identifiable); + +static FieldBase _G_nameField("name"); + +Serializer & NamedObject::onSerialize(Serializer & os) const +{ + return os.put(_G_nameField, _name); +} + +Deserializer & NamedObject::onDeserialize(Deserializer & is) +{ + return is.get(_G_nameField, _name); +} + +} diff --git a/staging_vespalib/src/vespa/vespalib/objects/namedobject.h b/staging_vespalib/src/vespa/vespalib/objects/namedobject.h new file mode 100644 index 00000000000..c9b49c92aa5 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/namedobject.h @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/objects/identifiable.h> +#include <string> + +namespace vespalib +{ + +class NamedObject : public Identifiable +{ +public: + DECLARE_IDENTIFIABLE_NS(vespalib, NamedObject); + DECLARE_NBO_SERIALIZE; + NamedObject() : _name() { } + NamedObject(const string & name) : _name(name) { } + const string & getName() const { return _name; } +private: + string _name; +}; + +} + diff --git a/staging_vespalib/src/vespa/vespalib/objects/nboserializer.cpp b/staging_vespalib/src/vespa/vespalib/objects/nboserializer.cpp new file mode 100644 index 00000000000..00e7a9c7edd --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/nboserializer.cpp @@ -0,0 +1,87 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "nboserializer.h" + +namespace vespalib { + +NBOSerializer &NBOSerializer::put(const IFieldBase &, bool value) { + _stream << value; + return *this; +} + +NBOSerializer &NBOSerializer::put(const IFieldBase &, uint8_t value) { + _stream << value; + return *this; +} + +NBOSerializer &NBOSerializer::put(const IFieldBase &, uint16_t value) { + _stream << value; + return *this; +} + +NBOSerializer &NBOSerializer::put(const IFieldBase &, uint32_t value) { + _stream << value; + return *this; +} + +NBOSerializer &NBOSerializer::put(const IFieldBase &, uint64_t value) { + _stream << value; + return *this; +} + +NBOSerializer &NBOSerializer::put(const IFieldBase &, float value) { + _stream << value; + return *this; +} + +NBOSerializer &NBOSerializer::put(const IFieldBase &, double value) { + _stream << value; + return *this; +} + +NBOSerializer &NBOSerializer::put(const IFieldBase &, const stringref & value) { + _stream << value; + return *this; +} + + +NBOSerializer &NBOSerializer::get(const IFieldBase &, bool & value) { + _stream >> value; + return *this; +} + +NBOSerializer &NBOSerializer::get(const IFieldBase &, uint8_t & value) { + _stream >> value; + return *this; +} + +NBOSerializer &NBOSerializer::get(const IFieldBase &, uint16_t & value) { + _stream >> value; + return *this; +} + +NBOSerializer &NBOSerializer::get(const IFieldBase &, uint32_t & value) { + _stream >> value; + return *this; +} + +NBOSerializer &NBOSerializer::get(const IFieldBase &, uint64_t & value) { + _stream >> value; + return *this; +} + +NBOSerializer &NBOSerializer::get(const IFieldBase &, double & value) { + _stream >> value; + return *this; +} + +NBOSerializer &NBOSerializer::get(const IFieldBase &, float & value) { + _stream >> value; + return *this; +} + +NBOSerializer &NBOSerializer::get(const IFieldBase &, string & value) { + _stream >> value; + return *this; +} +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/objects/nboserializer.h b/staging_vespalib/src/vespa/vespalib/objects/nboserializer.h new file mode 100644 index 00000000000..bac768fec2c --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/nboserializer.h @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/objects/serializer.h> +#include <vespa/vespalib/objects/deserializer.h> +#include <vespa/vespalib/objects/nbostream.h> + +namespace vespalib { +class NBOSerializer : public Serializer, public Deserializer { +public: + NBOSerializer(nbostream &stream) : _stream(stream) { } + virtual NBOSerializer &put(const IFieldBase &field, bool value); + virtual NBOSerializer &put(const IFieldBase &field, uint8_t value); + virtual NBOSerializer &put(const IFieldBase &field, uint16_t value); + virtual NBOSerializer &put(const IFieldBase &field, uint32_t value); + virtual NBOSerializer &put(const IFieldBase &field, uint64_t value); + virtual NBOSerializer &put(const IFieldBase &field, float value); + virtual NBOSerializer &put(const IFieldBase &field, double value); + virtual NBOSerializer &put(const IFieldBase &field, const stringref &val); + + virtual NBOSerializer &get(const IFieldBase &field, bool &value); + virtual NBOSerializer &get(const IFieldBase &field, uint8_t &value); + virtual NBOSerializer &get(const IFieldBase &field, uint16_t &value); + virtual NBOSerializer &get(const IFieldBase &field, uint32_t &value); + virtual NBOSerializer &get(const IFieldBase &field, uint64_t &value); + virtual NBOSerializer &get(const IFieldBase &field, double &value); + virtual NBOSerializer &get(const IFieldBase &field, float &value); + virtual NBOSerializer &get(const IFieldBase &field, string &value); + + const char *peek() const { return _stream.peek(); } + + const nbostream &getStream() const { return _stream; } + nbostream &getStream() { return _stream; } +private: + nbostream &_stream; +}; +} // namespace vespalib + diff --git a/staging_vespalib/src/vespa/vespalib/objects/objectdumper.cpp b/staging_vespalib/src/vespa/vespalib/objects/objectdumper.cpp new file mode 100644 index 00000000000..a80c69fd800 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/objectdumper.cpp @@ -0,0 +1,101 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "objectdumper.h" +#include <vespa/vespalib/util/stringfmt.h> + +namespace vespalib { + +void +ObjectDumper::addIndent() +{ + int n = _currIndent; + if (n < 0) { + n = 0; + } + _str.append(vespalib::string(n, ' ')); +} + +void +ObjectDumper::addLine(const vespalib::string &line) +{ + addIndent(); + _str.append(line); + _str.push_back('\n'); +} + +void +ObjectDumper::openScope() +{ + _currIndent += _indent; +} + +void +ObjectDumper::closeScope() +{ + _currIndent -= _indent; +} + +ObjectDumper::ObjectDumper(int indent) + : _str(), + _indent(indent), + _currIndent(0) +{ +} + +//----------------------------------------------------------------------------- + +void +ObjectDumper::openStruct(const vespalib::string &name, const vespalib::string &type) +{ + if (name.empty()) { + addLine(make_string("%s {", type.c_str())); + } else { + addLine(make_string("%s: %s {", name.c_str(), type.c_str())); + } + openScope(); +} + +void +ObjectDumper::closeStruct() +{ + closeScope(); + addLine("}"); +} + +void +ObjectDumper::visitBool(const vespalib::string &name, bool value) +{ + addLine(make_string("%s: %s", name.c_str(), value? "true" : "false")); +} + +void +ObjectDumper::visitInt(const vespalib::string &name, int64_t value) +{ + addLine(make_string("%s: %" PRId64 "", name.c_str(), value)); +} + +void +ObjectDumper::visitFloat(const vespalib::string &name, double value) +{ + addLine(make_string("%s: %g", name.c_str(), value)); +} + +void +ObjectDumper::visitString(const vespalib::string &name, const vespalib::string &value) +{ + addLine(make_string("%s: '%s'", name.c_str(), value.c_str())); +} + +void +ObjectDumper::visitNull(const vespalib::string &name) +{ + addLine(make_string("%s: <NULL>", name.c_str())); +} + +void +ObjectDumper::visitNotImplemented() +{ + addLine("<member visit not implemented>"); +} + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/objects/objectdumper.h b/staging_vespalib/src/vespa/vespalib/objects/objectdumper.h new file mode 100644 index 00000000000..7edb017c3a8 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/objectdumper.h @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "objectvisitor.h" + +namespace vespalib { + +/** + * This is a concrete object visitor that will build up a structured + * human-readable string representation of an object. + **/ +class ObjectDumper : public ObjectVisitor +{ +private: + vespalib::string _str; + int _indent; + int _currIndent; + + /** + * Add a number of spaces equal to the current indent to the + * string we are building. + **/ + void addIndent(); + + /** + * Add a complete line of output. Appropriate indentation will be + * added before the given string and a newline will be added after + * it. + * + * @param line the line we want to add + **/ + void addLine(const vespalib::string &line); + + /** + * Open a subscope by increasing the current indent level + **/ + void openScope(); + + /** + * Close a subscope by decreasing the current indent level + **/ + void closeScope(); + +public: + /** + * Create an object dumper with the given indent size; default is + * 4 spaces per indent level. + * + * @param indent indent size in number of spaces + **/ + ObjectDumper(int indent = 4); + + /** + * Obtain the created object string representation. This object + * should be invoked after the complete object structure has been + * visited. + * + * @return object string representation + **/ + vespalib::string toString() const { return _str; } + + // inherit doc + virtual void openStruct(const vespalib::string &name, const vespalib::string &type); + // inherit doc + virtual void closeStruct(); + // inherit doc + virtual void visitBool(const vespalib::string &name, bool value); + // inherit doc + virtual void visitInt(const vespalib::string &name, int64_t value); + // inherit doc + virtual void visitFloat(const vespalib::string &name, double value); + // inherit doc + virtual void visitString(const vespalib::string &name, const vespalib::string &value); + // inherit doc + virtual void visitNull(const vespalib::string &name); + // inherit doc + virtual void visitNotImplemented(); +}; + +} // namespace vespalib + diff --git a/staging_vespalib/src/vespa/vespalib/objects/objectoperation.cpp b/staging_vespalib/src/vespa/vespalib/objects/objectoperation.cpp new file mode 100644 index 00000000000..016a8f7832f --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/objectoperation.cpp @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "objectoperation.h" + +namespace vespalib { + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/objects/objectoperation.h b/staging_vespalib/src/vespa/vespalib/objects/objectoperation.h new file mode 100644 index 00000000000..734397ff670 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/objectoperation.h @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +namespace vespalib { + +class Identifiable; + +/** + * An operation that is able to operate on a generic object. + **/ +class ObjectOperation +{ +public: + /** + * Apply this operation to the given object. + * + * @param obj the object to operate on + **/ + virtual void execute(Identifiable &obj) = 0; + + /** + * empty + **/ + virtual ~ObjectOperation() { } +}; + +} // namespace vespalib + diff --git a/staging_vespalib/src/vespa/vespalib/objects/objectpredicate.cpp b/staging_vespalib/src/vespa/vespalib/objects/objectpredicate.cpp new file mode 100644 index 00000000000..282f5550585 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/objectpredicate.cpp @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "objectpredicate.h" + +namespace vespalib { + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/objects/objectpredicate.h b/staging_vespalib/src/vespa/vespalib/objects/objectpredicate.h new file mode 100644 index 00000000000..92a7987d85c --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/objectpredicate.h @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +namespace vespalib { + +class Identifiable; + +/** + * A predicate that is able to say either true or false when presented + * with a generic object. + **/ +class ObjectPredicate +{ +public: + /** + * Apply this predicate to the given object. + * + * @return true or false + * @param obj the object to check + **/ + virtual bool check(const Identifiable &obj) const = 0; + + /** + * empty + **/ + virtual ~ObjectPredicate() { } +}; + +} // namespace vespalib + diff --git a/staging_vespalib/src/vespa/vespalib/objects/objectvisitor.cpp b/staging_vespalib/src/vespa/vespalib/objects/objectvisitor.cpp new file mode 100644 index 00000000000..d025a596a08 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/objectvisitor.cpp @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "objectvisitor.h" +#include "identifiable.h" + +namespace vespalib { + +ObjectVisitor::~ObjectVisitor() +{ +} + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/objects/objectvisitor.h b/staging_vespalib/src/vespa/vespalib/objects/objectvisitor.h new file mode 100644 index 00000000000..fd11effef89 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/objectvisitor.h @@ -0,0 +1,89 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <stdint.h> + +namespace vespalib { + +/** + * This is an abstract class used to visit structured objects. It + * contains a basic interface that is intended to be overridden by + * subclasses. As an extension to this class, the visit.hpp file + * contains various versions of the visit method that maps visitation + * of various types into invocations of the basic interface defined by + * this class. + **/ +class ObjectVisitor +{ +public: + /** + * Open a (sub-)structure + * + * @param name name of structure + * @param type type of structure + **/ + virtual void openStruct(const vespalib::string &name, const vespalib::string &type) = 0; + + /** + * Close a (sub-)structure + **/ + virtual void closeStruct() = 0; + + /** + * Visit a boolean value + * + * @param name variable name + * @param value variable value + **/ + virtual void visitBool(const vespalib::string &name, bool value) = 0; + + /** + * Visit an integer value + * + * @param name variable name + * @param value variable value + **/ + virtual void visitInt(const vespalib::string &name, int64_t value) = 0; + + /** + * Visit a floating-point value + * + * @param name variable name + * @param value variable value + **/ + virtual void visitFloat(const vespalib::string &name, double value) = 0; + + /** + * Visit a string value + * + * @param name variable name + * @param value variable value + **/ + virtual void visitString(const vespalib::string &name, const vespalib::string &value) = 0; + + /** + * Visit method used to indicate that an optional substructure is + * not present. + * + * @param name variable name + **/ + virtual void visitNull(const vespalib::string &name) = 0; + + /** + * Visit method invoked by the default implementation of member + * visitation to signal that member visitation is not yet implemented. + * + * @param name variable name + * @param value variable value + **/ + virtual void visitNotImplemented() = 0; + + /** + * empty + **/ + virtual ~ObjectVisitor(); +}; + +} // namespace vespalib + diff --git a/staging_vespalib/src/vespa/vespalib/objects/serializer.cpp b/staging_vespalib/src/vespa/vespalib/objects/serializer.cpp new file mode 100644 index 00000000000..4583423eed6 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/serializer.cpp @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "serializer.h" +#include "identifiable.h" + +namespace vespalib { + +Serializer & Serializer::put(const IFieldBase & field, const Identifiable & value) +{ + (void) field; + return value.serializeDirect(*this); +} + +Serializer & Serializer::put(const IFieldBase & field, int8_t value) { return put(field, static_cast< uint8_t>(value)); } +Serializer & Serializer::put(const IFieldBase & field, int16_t value) { return put(field, static_cast<uint16_t>(value)); } +Serializer & Serializer::put(const IFieldBase & field, int32_t value) { return put(field, static_cast<uint32_t>(value)); } +Serializer & Serializer::put(const IFieldBase & field, int64_t value) { return put(field, static_cast<uint64_t>(value)); } + +} diff --git a/staging_vespalib/src/vespa/vespalib/objects/serializer.h b/staging_vespalib/src/vespa/vespalib/objects/serializer.h new file mode 100644 index 00000000000..c846123ce45 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/serializer.h @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/objects/fieldbase.h> +#include <vespa/vespalib/util/array.h> +#include <vector> +#include <stdint.h> + +namespace vespalib +{ + +class Identifiable; + +class Serializer : virtual SerializerCommon +{ +public: + virtual ~Serializer() { } + virtual Serializer & put(const IFieldBase & field, bool value) = 0; + virtual Serializer & put(const IFieldBase & field, uint8_t value) = 0; + virtual Serializer & put(const IFieldBase & field, uint16_t value) = 0; + virtual Serializer & put(const IFieldBase & field, uint32_t value) = 0; + virtual Serializer & put(const IFieldBase & field, uint64_t value) = 0; + virtual Serializer & put(const IFieldBase & field, float value) = 0; + virtual Serializer & put(const IFieldBase & field, double value) = 0; + virtual Serializer & put(const IFieldBase & field, const stringref & value) = 0; + + virtual Serializer & put(const IFieldBase & field, const Identifiable & value); + virtual Serializer & put(const IFieldBase & field, int8_t value); + virtual Serializer & put(const IFieldBase & field, int16_t value); + virtual Serializer & put(const IFieldBase & field, int32_t value); + virtual Serializer & put(const IFieldBase & field, int64_t value); + + Serializer & operator << (bool value) { return put(_unspecifiedField, value); } + Serializer & operator << (uint8_t value) { return put(_unspecifiedField, value); } + Serializer & operator << (int8_t value) { return put(_unspecifiedField, value); } + Serializer & operator << (uint16_t value) { return put(_unspecifiedField, value); } + Serializer & operator << (int16_t value) { return put(_unspecifiedField, value); } + Serializer & operator << (uint32_t value) { return put(_unspecifiedField, value); } + Serializer & operator << (int32_t value) { return put(_unspecifiedField, value); } + Serializer & operator << (uint64_t value) { return put(_unspecifiedField, value); } + Serializer & operator << (int64_t value) { return put(_unspecifiedField, value); } + Serializer & operator << (float value) { return put(_unspecifiedField, value); } + Serializer & operator << (double value) { return put(_unspecifiedField, value); } + Serializer & operator << (const stringref & value) { return put(_unspecifiedField, value); } + template <typename T> + Serializer & operator << (const vespalib::Array<T> & v) { + uint32_t sz(v.size()); + put(_sizeField, sz); + for(size_t i(0); i < sz; i++) { + (*this) << v[i]; + } + return *this; + } + template <typename T> + Serializer & operator << (const std::vector<T> & v) { + uint32_t sz(v.size()); + put(_sizeField, sz); + for(size_t i(0); i < sz; i++) { + (*this) << v[i]; + } + return *this; + } +}; + +} + diff --git a/staging_vespalib/src/vespa/vespalib/objects/simpledynamicdata.h b/staging_vespalib/src/vespa/vespalib/objects/simpledynamicdata.h new file mode 100644 index 00000000000..17b93b8f0e9 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/simpledynamicdata.h @@ -0,0 +1,113 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vector> +#include <algorithm> +#include <memory> +#include "dynamicdatavalue.h" + +namespace vespalib +{ + +class SimpleDynamicData : public DynamicDataValue +{ +private: + struct DataUnit { + uint32_t id; + Value *v; + DataUnit(uint32_t id_, Value *v_) : id(id_), v(v_) {} + }; + typedef std::vector<DataUnit> DataUnits; + DataUnits _values; + + virtual bool setValueIfExisting(uint32_t id, const Value& v) { + for (DataUnits::iterator it = _values.begin(); it != _values.end(); ++it) { + if (it->id == id) { + delete it->v; + it->v = v.clone(); + return true; + } + } + return false; + } + virtual void addNewValue(uint32_t id, const Value& v) { + _values.push_back(DataUnit(id, v.clone())); + } + + void swap(SimpleDynamicData& other) { + std::swap(_values, other._values); + } + +public: + virtual ~SimpleDynamicData() { + // we own contained data, must delete it + for (DataUnits::iterator it = _values.begin(); it != _values.end(); ++it) { + delete it->v; + } + } + virtual bool hasValue(uint32_t id) const { + for (DataUnits::const_iterator it = _values.begin(); it != _values.end(); ++it) { + if (it->id == id) return true; + } + return false; + } + virtual void deleteValue(uint32_t id) { + for (DataUnits::iterator it = _values.begin(); it != _values.end(); ++it) { + if (it->id == id) { + delete it->v; + _values.erase(it); + return; + } + } + } + virtual const Value& getValue(uint32_t id) const { + for (DataUnits::const_iterator it = _values.begin(); it != _values.end(); ++it) { + if (it->id == id) { + return *(it->v); + } + } + throw IllegalArgumentException("id not found"); + } + virtual Value *getValueRef(uint32_t id) const { + for (DataUnits::const_iterator it = _values.begin(); it != _values.end(); ++it) { + if (it->id == id) { + return it->v; + } + } + return NULL; + } + virtual void visitValues(ValueReceiverI& visitor) const { + for (DataUnits::const_iterator it = _values.begin(); it != _values.end(); ++it) { + visitor(it->id, *(it->v)); + } + } + + SimpleDynamicData() : _values() {} + SimpleDynamicData(const SimpleDynamicData& other) : + DynamicDataValue(), + _values() + { + for (DataUnits::const_iterator it = other._values.begin(); + it != other._values.end(); + ++it) + { + addNewValue(it->id, *(it->v)); + } + } + + SimpleDynamicData& operator= (const SimpleDynamicData& other) { + SimpleDynamicData tmp(other); + swap(tmp); + return *this; + } + + virtual SimpleDynamicData *clone() const { + SimpleDynamicData *ret = new SimpleDynamicData(*this); + return ret; + } +}; + + + +} // namespace vespalib + diff --git a/staging_vespalib/src/vespa/vespalib/objects/valuetype.cpp b/staging_vespalib/src/vespa/vespalib/objects/valuetype.cpp new file mode 100644 index 00000000000..ec7cf3132b9 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/valuetype.cpp @@ -0,0 +1,26 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "valuetype.h" + +namespace vespalib { + +const StringValueType StringValueType::t; +const DynamicDataValueType DynamicDataValueType::t; +const Int16ValueType Int16ValueType::t; +const Int32ValueType Int32ValueType::t; +const Int64ValueType Int64ValueType::t; +const UnsignedByteValueType UnsignedByteValueType::t; +const Uint128ValueType Uint128ValueType::t; +const Uint256ValueType Uint256ValueType::t; +const Uint96ValueType Uint96ValueType::t; +const ArrayOfStringValueType ArrayOfStringValueType::t; +const ArrayOfDynamicDataValueType ArrayOfDynamicDataValueType::t; +const ArrayOfInt16ValueType ArrayOfInt16ValueType::t; +const ArrayOfInt32ValueType ArrayOfInt32ValueType::t; +const ArrayOfInt64ValueType ArrayOfInt64ValueType::t; +const ArrayOfUnsignedByteValueType ArrayOfUnsignedByteValueType::t; +const ArrayOfUint128ValueType ArrayOfUint128ValueType::t; +const ArrayOfUint256ValueType ArrayOfUint256ValueType::t; +const ArrayOfUint96ValueType ArrayOfUint96ValueType::t; + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/objects/visit.cpp b/staging_vespalib/src/vespa/vespalib/objects/visit.cpp new file mode 100644 index 00000000000..3b160961a01 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/visit.cpp @@ -0,0 +1,81 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "visit.h" + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::Identifiable *obj) { + if (obj != 0) { + self.openStruct(name, obj->getClass().name()); + obj->visitMembers(self); + self.closeStruct(); + } else { + self.visitNull(name); + } +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::Identifiable &obj) { + visit(self, name, &obj); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, bool value) { + self.visitBool(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, int8_t value) +{ + self.visitInt(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, uint8_t value) +{ + self.visitInt(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, int16_t value) +{ + self.visitInt(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, uint16_t value) +{ + self.visitInt(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, int32_t value) { + self.visitInt(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, uint32_t value) { + self.visitInt(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, int64_t value) { + self.visitInt(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, uint64_t value) { + self.visitInt(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, float value) { + self.visitFloat(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, double value) { + self.visitFloat(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::string &value) { + self.visitString(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const std::string &value) { + self.visitString(name, value); +} + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const char *value) { + if (value != 0) { + visit(self, name, vespalib::string(value)); + } else { + self.visitNull(name); + } +} diff --git a/staging_vespalib/src/vespa/vespalib/objects/visit.h b/staging_vespalib/src/vespa/vespalib/objects/visit.h new file mode 100644 index 00000000000..42826c7ff74 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/objects/visit.h @@ -0,0 +1,95 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <string> +#include <vector> +#include <vespa/vespalib/util/stringfmt.h> +#include "objectvisitor.h" +#include "identifiable.h" + +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::Identifiable *obj); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::Identifiable &obj); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, bool value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, int8_t value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, uint8_t value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, int16_t value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, uint16_t value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, int32_t value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, uint32_t value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, int64_t value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, uint64_t value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, float value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, double value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::string &value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const std::string &value); +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const char *value); + +template<typename T> +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::CloneablePtr<T> &ptr) { + if (ptr.get()) { + visit(self, name, *ptr); + } else { + self.visitNull(name); + } +} + +template<typename T> +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const std::shared_ptr<T> &ptr) { + if (ptr.get()) { + visit(self, name, *ptr); + } else { + self.visitNull(name); + } +} + +template<typename T> +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const std::unique_ptr<T> &ptr) { + if (ptr.get()) { + visit(self, name, *ptr); + } else { + self.visitNull(name); + } +} + +template<typename T> +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::LinkedPtr<T> &ptr) { + if (ptr.get()) { + visit(self, name, *ptr); + } else { + self.visitNull(name); + } +} + +template<typename T> +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::IdentifiablePtr<T> &ptr) { + visit(self, name, ptr.get()); +} + +template<typename T> +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::IdentifiableLinkedPtr<T> &ptr) { + visit(self, name, ptr.get()); +} + +template<typename T> +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::IdentifiableSharedPtr<T> &ptr) { + visit(self, name, ptr.get()); +} + +template<typename T> +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const std::vector<T> &list) { + self.openStruct(name, "std::vector"); + for (uint32_t i = 0; i < list.size(); ++i) { + visit(self, vespalib::make_string("[%u]", i), list[i]); + } + self.closeStruct(); +} + +template<typename T> +void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const vespalib::Array<T> &list) { + self.openStruct(name, "vespalib::Array"); + for (uint32_t i = 0; i < list.size(); ++i) { + visit(self, vespalib::make_string("[%u]", i), list[i]); + } + self.closeStruct(); +} + diff --git a/staging_vespalib/src/vespa/vespalib/stllike/.gitignore b/staging_vespalib/src/vespa/vespalib/stllike/.gitignore new file mode 100644 index 00000000000..ee8938b6bf4 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/stllike/.gitignore @@ -0,0 +1,6 @@ +*.So +*.exe +*.ilk +*.pdb +.depend* +Makefile diff --git a/staging_vespalib/src/vespa/vespalib/stllike/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/stllike/CMakeLists.txt new file mode 100644 index 00000000000..cf36a7f210f --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/stllike/CMakeLists.txt @@ -0,0 +1,6 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(staging_vespalib_vespalib_stllike OBJECT + SOURCES + smallvector.cpp + DEPENDS +) diff --git a/staging_vespalib/src/vespa/vespalib/stllike/avl.h b/staging_vespalib/src/vespa/vespalib/stllike/avl.h new file mode 100644 index 00000000000..be8ccbd3497 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/stllike/avl.h @@ -0,0 +1,367 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vector> +#include <iterator> +#include <bits/stl_algo.h> +#include <bits/stl_function.h> + +#include "hash_fun.h" + +namespace vespalib { + +/** + Yet another avl implementation. This one is justified by + different memory management. + + The interface is tried to keep similar to stl version. However the + are some major differences. In order to avoid an allocation / + deallocation for every object inserted / erased, it stores all + objects in a std::vector. This should significantly speed up + things. However it does remove properties that the stl versions + have. The most obvious is that insert might invalidate iterators. + That is due to possible resizing of memory store. That can be + avoided by using a deque. However a deque is more complex and will + be slower. Since speed is here preferred over functionality that is + not yet done. + The same trick could be done for tree based map/set. + An entry can be valid or invalid(not set, or erased). + The entry contains the element + a next index. + + After selecting the proper prime number for modulo operation, the + vector reserves requested space. Resize (triggered by full vector) + doubles size. Then the first 'prime' entries are initialized to + invalid. Then the objects are filled in. If the selected bucket is + invalid the element is inserted directly. If not it is + 'push_back'ed, on the vector and linked in after the head element + for that bucket. Erased elements are marked invalid, but not + reused. They are reclaimed on resize. + + Advantage: + - Significantly faster insert on average. No alloc/dealloc per element. + Disadvantage: + - insert spikes due to possible resize. + - not fully stl-compliant. + Conclusion: + - Probably very good for typical use. Fx duplicate removal. + Advice: + - If you know how many elements you are going to put in, construct + the hash with 2 times the amount. Since a hash will never be + fully filled. + ( hash_set<T>(2*num_expected_elements) ). +**/ + +class avl_base +{ +public: + typedef unsigned int next_t; +}; + +template< typename Key, typename Value, typename Compare, typename KeyExtract > +class avl : public avl_base +{ +protected: + class Node { + public: + enum {npos=-1u, invalid=-2u}; + Node() : _parent(npos), _left(npos), _right(npos) { } + Node(const Value & node, next_t parent=npos, next_t left=npos, next_t right=npos) : _parent(parent), _left(left), _right(right), _node(node) { } + Value & getValue() { return _node; } + const Value & getValue() const { return _node; } + next_t getParent() const { return _parent; } + void setParent(next_t v) { _parent = v; } + next_t getLeft() const { return _left; } + void setLeft(next_t v) { _left = v; } + next_t getRight() const { return _right; } + void setRight(next_t v) { _right = v; } + private: + next_t _parent; + next_t _left; + next_t _right; + Value _node; + }; + typedef std::vector<Node> NodeStore; + virtual void move(const NodeStore & oldStore); +public: + class iterator { + public: + iterator(avl * avl, next_t start) : _node(start), _avl(avl) { } + Value & operator * () const { return _avl->get(_node); } + Value * operator -> () const { return & _avl->get(_node); } + iterator & operator ++ () { + _node = _avl->getNextRight(_node); + return *this; + } + iterator operator ++ (int) { + iterator prev = *this; + ++(*this); + return prev; + } + bool operator==(const iterator& rhs) const { return (_node == rhs._node); } + bool operator!=(const iterator& rhs) const { return (_node != rhs._node); } + private: + next_t _node; + avl * _avl; + + friend class avl::const_iterator; + }; + class const_iterator { + public: + const_iterator(const avl * avl, next_t start) : _node(start), _avl(avl) { } + const_iterator(const iterator &i) : _node(i._node), _avl(i._avl) {} + Value & operator * () const { return _avl->get(_node); } + Value * operator -> () const { return & _avl->get(_node); } + iterator & operator ++ () { + _node = _avl->getNextRight(_node); + return *this; + } + iterator operator ++ (int) { + iterator prev = *this; + ++(*this); + return prev; + } + bool operator==(const iterator& rhs) const { return (_node == rhs._node); } + bool operator!=(const iterator& rhs) const { return (_node != rhs._node); } + private: + next_t _node; + avl * _avl; + + friend class avl::const_iterator; + }; + +public: + avl(size_t reservedSpace); + iterator begin() { return iterator(this, getLeftMost(_root)); } + iterator end() { return iterator(this, Node::npos); } + const_iterator begin() const { return const_iterator(this, getLeftMost(_root)); } + const_iterator end() const { return const_iterator(this, Node::npos); } + size_t capacity() const { return _nodes.capacity(); } + size_t size() const { return _nodes.size(); } + bool empty() const { return _root == npos; } + template< typename AltKey, typename AltExtract, typename AltCompare > + iterator find(const AltKey & key); + iterator find(const Key & key); + iterator find(const Key & key) { return iterator(this, internalFind(key)); } + template< typename AltKey, typename AltExtract, typename AltCompare > + const_iterator find(const AltKey & key) const; + const_iterator find(const Key & key) const { return const_iterator(this, internalFind(key)); } + void insert(const Value & node); + void erase(const Key & key); + void clear() { _nodes.clear(); } + void reserve(size_t newReserve) { _nodes.reserve(newReserve); } + void swap(avl & rhs); + /** + * Get an approximate number of memory consumed by hash set. Not including + * any data K would store outside of sizeof(K) of course. + */ + size_t getMemoryConsumption() const; + +protected: + /// These two methods are only for the ones that know what they are doing. + /// valid input here are stuff returned from iterator.getInternalIndex. + next_t insertInternal(const Value & node); + Value & getByInternalIndex(size_t index) { return _nodes[index].getValue(); } + const Value & getByInternalIndex(size_t index) const { return _nodes[index].getValue(); } + template <typename MoveHandler> + void erase(MoveHandler & moveHandler, const Key & key); +private: + next_t _begin; + next_t _root; + NodeStore _nodes; + Compare _compare; + KeyExtract _keyExtractor; + next_t internalFind(const Key & key) const; + Value & get(size_t index) { return _nodes[index].getValue(); } + const Value & get(size_t index) const { return _nodes[index].getValue(); } + next_t getRightMost(next_t n) { + while(_nodes[n].hasRight()) { + n = _nodes[n].getRight()); + } + return n; + } + next_t getLeftMost(next_t n) { + while(_nodes[n].hasLeft()) { + n = _nodes[n].getleft()); + } + return n; + } + next_t getNextRight(next_t n) const { + if (_nodes[n].hasParent()) { + next_t parent = _nodes[_node].getParent(); + if (_nodes[parent].getLeft() == _node) { + return parent; + } else { + return getNextRight(parent); + } + } else if (_nodes[n].hasRight()) { + return getLeftMost(_nodes[n].getRight()); + } else { + return npos; + } + } + next_t getNextLeft(next_t n) const { + if (_nodes[n].hasParent()) { + next_t parent = _nodes[_node].getParent(); + if (_nodes[parent].getRight() == _node) { + return parent; + } else { + return getNextLeft(parent); + } + } else if (_nodes[n].hasLeft()) { + return getRightMost(_nodes[n].getLeft()); + } else { + return npos; + } + } +}; + +template< typename Key, typename Value, typename Compare, typename KeyExtract > +void avl<Key, Value, Compare, KeyExtract>::swap(avl & rhs) +{ + std::swap(_root, rhs._root); + _nodes.swap(rhs._nodes); + std::swap(_compare, rhs._compare); + std::swap(_keyExtractor, rhs._keyExtractor); +} + +template< typename Key, typename Value, typename Compare, typename KeyExtract > +avl<Key, Value, Compare, KeyExtract>::avl(size_t reservedSpace) : + _root(npos), + _nodes() +{ + if (reservedSpace > 0) { + reserve(reservedSpace); + } +} + +template< typename Key, typename Value, typename Compare, typename KeyExtract > +typename avl<Key, Value, Compare, KeyExtract>::iterator +avl<Key, Value, Compare, KeyExtract>::internalFind(const Key & key) +{ + next_t found = npos; // Last node which is not less than key. + + for (next_t n(_begin); n != npos; ) { + if (!_compare(_keyExtractor(_nodes[n]), key)) { + found = n; + n = getNextLeft(n); + } else { + n = getNextRight(n); + } + } + return ((found != npos) && ! _compare(key, _keyExtractor(_nodes[found]))) + ? found + : npos; +} + +template< typename Key, typename Value, typename Compare, typename KeyExtract > +typename avl<Key, Value, Compare, KeyExtract>::next_t +avl<Key, Value, Compare, KeyExtract>::insert(const Value & node) +{ + next_t n = _begin; + next_t e = npos; + Key key(_keyExtractor(node); + while (n != npos) { + e = n; + n = _compare(_keyExtractor(_nodes[n]), key) + ? getNextLeft(n) + : getNextRight(n); + } + return insert(n, e, node); +} + +template< typename Key, typename Value, typename Compare, typename KeyExtract > +typename avl<Key, Value, Compare, KeyExtract>::next_t +avl<Key, Value, Compare, KeyExtract>::insert(next_t n, next_t e, const Value & value) +{ + bool insert_left = (n != npos) || + (e == npos) || + _compare(_keyExtractor(value), _keyExtractor(_nodes[e])); + + next_t newN = _nodes.size(); + Node node(value); + _nodes.push_back(node); + + insert_and_rebalance(insert_left, newN, e, this->_M_impl._M_header); + return iterator(newN); +} + +template< typename Key, typename Value, typename Compare, typename KeyExtract > +void +avl<Key, Value, Compare, KeyExtract>::erase(const Key & key) +{ + next_t found = internalFind(key); + if (found != npos) { + // Link out + erase_and_rebalance(found); + // Swap with last + std::swap(_nodes[found], _nodes.back()); + nodes.resize(nodes.size() - 1); + // relink parent to last + if (_nodes[found].hasParent()) { + next_t parent = _nodes[found].getParent(); + if (_nodes[parent].getLeft() == old) { + _nodes[parent].setLeft(found); + } else { + _nodes[parent].setRight(found); + } + } + } +} + +template< typename Key, typename Value, typename Compare, typename KeyExtract > +size_t +avl<Key, Value, Compare, KeyExtract>::getMemoryConsumption() const +{ + return sizeof(*this) + _nodes.capacity() * sizeof(Node); +} + +#if 0 +template< typename Key, typename Value, typename Compare, typename KeyExtract > +template< typename AltKey, typename AltExtract, typename AltCompare> +typename avl<Key, Value, Compare, KeyExtract>::const_iterator +avl<Key, Value, Compare, KeyExtract>::find(const AltKey & key) const +{ + if (_modulo > 0) { + AltHash altHasher; + next_t h = altHasher(key) % _modulo; + if (_nodes[h].valid()) { + next_t start(h); + AltExtract altExtract; + AltCompare altCompare; + do { + if (altCompare(altExtract(_keyExtractor(_nodes[h].getValue())), key)) { + return const_iterator(this, start, h); + } + h = _nodes[h].getNext(); + } while (h != Node::npos); + } + } + return end(); +} + +template< typename Key, typename Value, typename Compare, typename KeyExtract > +template< typename AltKey, typename AltExtract, typename AltCompare> +typename avl<Key, Value, Hash, Compare, KeyExtract>::iterator +avl<Key, Value, Compare, KeyExtract>::find(const AltKey & key) +{ + if (_modulo > 0) { + AltHash altHasher; + next_t h = altHasher(key) % _modulo; + if (_nodes[h].valid()) { + next_t start(h); + AltExtract altExtract; + AltCompare altCompare; + do { + if (altCompare(altExtract(_keyExtractor(_nodes[h].getValue())), key)) { + return iterator(this, start, h); + } + h = _nodes[h].getNext(); + } while (h != Node::npos); + } + } + return end(); +} +#endif + +} + diff --git a/staging_vespalib/src/vespa/vespalib/stllike/avl_map.h b/staging_vespalib/src/vespa/vespalib/stllike/avl_map.h new file mode 100644 index 00000000000..7eae6aea684 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/stllike/avl_map.h @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/avl.h> + +namespace vespalib { + +template< typename K, typename V, typename EQ = std::equal_to<K> > +class avl_map +{ +public: + typedef std::pair<K, V> value_type; + typedef K key_type; + typedef V mapped_type; +private: + typedef avl< K, value_type, EQ, std::_Select1st< value_type > > Avl; + Avl _avl; +public: + typedef typename Avl::iterator iterator; + typedef typename Avl::const_iterator const_iterator; +public: + avl_map(size_t reserveSize=0) : _avl(reserveSize) { } + iterator begin() { return _avl.begin(); } + iterator end() { return _avl.end(); } + const_iterator begin() const { return _avl.begin(); } + const_iterator end() const { return _avl.end(); } + size_t capacity() const { return _avl.capacity(); } + size_t size() const { return _avl.size(); } + bool empty() const { return _avl.empty(); } + void insert(const value_type & value) { return _avl.insert(value); } + template <typename InputIt> + void insert(InputIt first, InputIt last); + V & operator [] (const K & key) const { return *_avl.find(key)->second; } + V & operator [] (const K & key); + void erase(const K & key) { return _avl.erase(key); } + iterator find(const K & key) { return _avl.find(key); } + const_iterator find(const K & key) const { return _avl.find(key); } + void clear() { _avl.clear(); } + void resize(size_t newSize) { _avl.resize(newSize); } + void swap(avl_map & rhs) { _avl.swap(rhs._avl); } +}; + +template <typename K, typename V, typename H, typename EQ> +template <typename InputIt> +void avl_map<K, V, H, EQ>::insert(InputIt first, InputIt last) { + while (first != last) { + _avl.insert(*first); + ++first; + } +} + +template< typename K, typename V, typename H, typename EQ > +V & avl_map<K, V, H, EQ>::operator [] (const K & key) +{ + iterator found = _avl.find(key); + if (found != _avl.end()) { + return found->second; + } + insert(value_type(key, V())); + return _avl.find(key)->second; +} + +} + + diff --git a/staging_vespalib/src/vespa/vespalib/stllike/cache.h b/staging_vespalib/src/vespa/vespalib/stllike/cache.h new file mode 100644 index 00000000000..ed4eb4350c0 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/stllike/cache.h @@ -0,0 +1,266 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/lrucache_map.h> +#include <vespa/vespalib/util/sync.h> +#include <vespa/vespalib/util/atomic.h> + +namespace vespalib { + +template<typename K, typename V> +class NullStore { +public: + bool read(const K &, V &) const { return false; } + void write(const K &, const V &) { } + void erase(const K &) { } +}; + +/** + * These are the parameters needed for setting up the cache. + * @param P is the set of parameters needed for setting up the underlying lrucache. See @ref LruParam + * @param B is the backing store. That is where the real data is backed up if there are any. + * If there are no backing store or you mix and match yourself, you can give it the @ref NullStore. + * @param SizeK is the method to get the space needed by the key in addition to what you get with sizeof. + * @param SizeV is the method to get the space needed by the value in addition to what you get with sizeof. + */ +template<typename P, typename B, typename sizeK = vespalib::zero<typename P::Key>, typename sizeV = vespalib::zero<typename P::Value> > +struct CacheParam : public P +{ + typedef B BackingStore; + typedef sizeK SizeK; + typedef sizeV SizeV; +}; + +/** + * This is a cache using the underlying lru implementation as the store. It is modelled as a pure cache + * with an backing store underneath it. That backing store is given to the constructor and must of course have + * proper lifetime. The store must implement the same 3 methods as the @ref NullStore above. + * Stuff is evicted from the cache if either number of elements or the accounted size passes the limits given. + * The cache is thread safe by a single lock for accessing the underlying Lru. In addition a striped locking with + * 64 locks chosen by the hash of the key to enable a single fetch for any element required by multiple readers. + */ +template< typename P > +class cache : private lrucache_map<P> +{ +private: + typedef lrucache_map<P> Lru; + typedef typename P::BackingStore BackingStore; + typedef typename P::Hash Hash; + typedef typename P::Key K; + typedef typename P::Value V; + typedef typename P::SizeK SizeK; + typedef typename P::SizeV SizeV; + typedef typename P::value_type value_type; +public: + /** + * Will create a cache that populates on demand from the backing store. + * The cache uses LRU and evicts whne its size in bytes or elements is reached. + * By max elements is initialized to max bytes. + * + * @param backingStore is the store for populating the cache on a cache miss. + * @maxBytes is the maximum limit of bytes the store can hold, before eviction starts. + */ + cache(BackingStore & b, size_t maxBytes); + + /** + * Can be used for controlling max number of elements. + */ + cache & maxElements(size_t elems) { + Lru::maxElements(elems); + return *this; + } + /** + * Can be used for reserving space for elements. + */ + cache & reserveElements(size_t elems) { + Lru::reserve(elems); + return *this; + } + + size_t capacity() const { return Lru::capacity(); } + size_t capacityBytes() const { return _maxBytes; } + size_t size() const { return Lru::size(); } + size_t sizeBytes() const { return _sizeBytes; } + bool empty() const { return Lru::empty(); } + + /** + * This simply erases the object. + * This will also erase from backing store. + */ + void erase(const K & key); + /** + * This simply erases the object from the cache. + */ + void invalidate(const K & key); + + /** + * Return the object with the given key. If it does not exist, the backing store will be consulted. + * and the cache will be updated. + * If none exist an empty one will be created. + * Object is then put at head of LRU list. + */ + V read(const K & key); + + /** + * Update the cache and write through to backing store. + * Object is then put at head of LRU list. + */ + void write(const K & key, const V & value); + + /** + * Tell if an object with given key exists in the cache. + * Does not alter the LRU list. + */ + bool hasKey(const K & key) const; + + size_t getHit() const { return _hit; } + size_t getMiss() const { return _miss; } + size_t getNoneExisting() const { return _noneExisting; } + size_t getRace() const { return _race; } + size_t getInsert() const { return _insert; } + size_t getWrite() const { return _write; } + size_t getErase() const { return _erase; } + size_t getInvalidate() const { return _invalidate; } + size_t getlookup() const { return _lookup; } + +private: + /** + * Called when an object is inserted, to see if the LRU should be removed. + * Default is to obey the maxsize given in constructor. + * The obvious extension is when you are storing pointers and want to cap + * on the real size of the object pointed to. + */ + virtual bool removeOldest(const value_type & v); + size_t calcSize(const K & k, const V & v) const { return sizeof(value_type) + _sizeK(k) + _sizeV(v); } + vespalib::Lock & getLock(const K & k) { + size_t h(_hasher(k)); + return _addLocks[h%(sizeof(_addLocks)/sizeof(_addLocks[0]))]; + } + Hash _hasher; + SizeK _sizeK; + SizeV _sizeV; + size_t _maxBytes; + size_t _sizeBytes; + mutable size_t _hit; + mutable size_t _miss; + mutable size_t _noneExisting; + mutable size_t _race; + mutable size_t _insert; + mutable size_t _write; + mutable size_t _erase; + mutable size_t _invalidate; + mutable size_t _lookup; + BackingStore & _store; + vespalib::Lock _hashLock; + /// Striped locks that can be used for having a locked access to the backing store. + vespalib::Lock _addLocks[113]; +}; + +template< typename P > +cache<P>::cache(BackingStore & b, size_t maxBytes) : + Lru(Lru::UNLIMITED), + _maxBytes(maxBytes), + _sizeBytes(0), + _hit(0), + _miss(0), + _noneExisting(0), + _race(0), + _insert(0), + _write(0), + _erase(0), + _invalidate(0), + _lookup(0), + _store(b) +{ } + +template< typename P > +bool +cache<P>::removeOldest(const value_type & v) { + bool remove(Lru::removeOldest(v) || (sizeBytes() >= capacityBytes())); + if (remove) { + _sizeBytes -= calcSize(v.first, v.second._value); + } + return remove; +} + +template< typename P > +typename P::Value +cache<P>::read(const K & key) +{ + { + vespalib::LockGuard guard(_hashLock); + if (Lru::hasKey(key)) { + _hit++; + return (*this)[key]; + } else { + _miss++; + } + } + + vespalib::LockGuard storeGuard(getLock(key)); + { + vespalib::LockGuard guard(_hashLock); + if (Lru::hasKey(key)) { + // Somebody else just fetched it ahead of me. + _race++; + return (*this)[key]; + } + } + V value; + if (_store.read(key, value)) { + vespalib::LockGuard guard(_hashLock); + Lru::insert(key, value); + _sizeBytes += calcSize(key, value); + _insert++; + } else { + vespalib::Atomic::postInc(&_noneExisting); + } + return value; +} + +template< typename P > +void +cache<P>::write(const K & key, const V & value) +{ + vespalib::LockGuard storeGuard(getLock(key)); + { + vespalib::LockGuard guard(_hashLock); + (*this)[key] = value; + _sizeBytes += calcSize(key, value); + _write++; + } + _store.write(key, value); +} + +template< typename P > +void +cache<P>::erase(const K & key) +{ + vespalib::LockGuard storeGuard(getLock(key)); + invalidate(key); + _store.erase(key); +} + +template< typename P > +void +cache<P>::invalidate(const K & key) +{ + vespalib::LockGuard guard(_hashLock); + if (Lru::hasKey(key)) { + _sizeBytes -= calcSize(key, (*this)[key]); + _invalidate++; + Lru::erase(key); + } +} + +template< typename P > +bool +cache<P>::hasKey(const K & key) const +{ + vespalib::LockGuard guard(_hashLock); + _lookup++; + return Lru::hasKey(key); +} + +} + diff --git a/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.h b/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.h new file mode 100644 index 00000000000..993163a82cd --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.h @@ -0,0 +1,420 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/hashtable.h> + +namespace vespalib { + +struct LinkedValueBase { + static const uint32_t npos = static_cast<uint32_t>(-1); + LinkedValueBase() : _prev(npos), _next(npos) { } + LinkedValueBase(uint32_t prev, uint32_t next) : _prev(prev), _next(next) { } + uint32_t _prev; + uint32_t _next; +}; + +template<typename V> +struct LinkedValue : public LinkedValueBase +{ + LinkedValue() {} + LinkedValue(const V & v) : LinkedValueBase(), _value(v) { } + V _value; +}; + +template<typename K, typename V, typename H = vespalib::hash<K>, typename EQ = std::equal_to<K> > +struct LruParam +{ + typedef LinkedValue<V> LV; + typedef std::pair< K, LV > value_type; + typedef std::_Select1st< value_type > select_key; + typedef K Key; + typedef V Value; + typedef H Hash; + typedef EQ Equal; + typedef hashtable< Key, value_type, Hash, Equal, select_key > HashTable; +}; + +template< typename P > +class lrucache_map : private P::HashTable +{ +private: + typedef typename P::HashTable HashTable; + typedef typename P::Value V; + typedef typename P::Key K; + typedef typename P::value_type value_type; + typedef typename P::LV LV; + typedef typename HashTable::iterator internal_iterator; + typedef typename HashTable::next_t next_t; + typedef typename HashTable::NodeStore NodeStore; +protected: + static constexpr size_t UNLIMITED = std::numeric_limits<size_t>::max(); +public: + typedef typename HashTable::insert_result insert_result; + + class iterator { + public: + iterator(lrucache_map * cache, uint32_t current) : _cache(cache), _current(current) { } + V & operator * () const { return _cache->getByInternalIndex(_current).second._value; } + V * operator -> () const { return & _cache->getByInternalIndex(_current).second._value; } + iterator & operator ++ () { + if (_current != LinkedValueBase::npos) { + _current = _cache->getByInternalIndex(_current).second._next; + } + return *this; + } + iterator operator ++ (int) { + iterator prev = *this; + ++(*this); + return prev; + } + bool operator==(const iterator& rhs) const { return (_current == rhs._current); } + bool operator!=(const iterator& rhs) const { return (_current != rhs._current); } + private: + lrucache_map * _cache; + uint32_t _current; + + friend class lrucache_map; + }; + + /** + * Will create a lrucache with max elements. Use the chained setter + * @ref reserve to control initial size. + * + * @param maxElements in cache unless you override @ref removeOldest. + */ + lrucache_map(size_t maxElems); + + lrucache_map & maxElements(size_t elems) { + _maxElements = elems; + return *this; + } + lrucache_map & reserve(size_t elems) { + HashTable::reserve(elems); + return *this; + } + + + size_t capacity() const { return _maxElements; } + size_t size() const { return HashTable::size(); } + bool empty() const { return HashTable::empty(); } + iterator begin() { return iterator(this, _head); } + iterator end() { return iterator(this, LinkedValueBase::npos); } + + /** + * This fetches the object without modifying the lru list. + */ + const V & get(const K & key) { return HashTable::find(key)->second._value; } + + /** + * This simply erases the object. + */ + void erase(const K & key); + + /** + * Erase object pointed to by iterator. + */ + iterator erase(const iterator & it); + + /** + * Object is inserted in cache with given key. + * Object is then put at head of LRU list. + */ + insert_result insert(const K & key, const V & value) { + return insert(value_type(key, LV(value))); + } + + /** + * Return the object with the given key. If it does not exist an empty one will be created. + * This can be used as an insert. + * Object is then put at head of LRU list. + */ + const V & operator [] (const K & key) const { + return const_cast<lrucache_map<P> *>(this)->findAndRef(key).second._value; + } + + /** + * Return the object with the given key. If it does not exist an empty one will be created. + * This can be used as an insert. + * Object is then put at head of LRU list. + */ + V & operator [] (const K & key); + + /** + * Tell if an object with given key exists in the cache. + * Does not alter the LRU list. + */ + bool hasKey(const K & key) const { return HashTable::find(key) != HashTable::end(); } + + /** + * Called when an object is inserted, to see if the LRU should be removed. + * Default is to obey the maxsize given in constructor. + * The obvious extension is when you are storing pointers and want to cap + * on the real size of the object pointed to. + */ + virtual bool removeOldest(const value_type & v) { + (void) v; + return (size() > capacity()); + } + + /** + * Method for testing that internal consitency is good. + */ + bool verifyInternals(); + + /** + * Implements the move callback from the hashtable + */ + void move(next_t from, next_t to); + + void swap(lrucache_map & rhs); + +private: + typedef std::pair<uint32_t, uint32_t> MoveRecord; + typedef std::vector<MoveRecord> MoveRecords; + /** + * Implements the resize of the hashtable + */ + void move(NodeStore && oldStore) override; + internal_iterator findAndRef(const K & key); + void ref(const internal_iterator & it); + insert_result insert(value_type && value); + void removeOld(); + class RecordMoves : public noncopyable { + public: + RecordMoves(lrucache_map & lru) : + _lru(lru) + { + _lru._moveRecordingEnabled = true; + } + uint32_t movedTo(uint32_t from) { + for (size_t i(0); i < _lru._moved.size(); i++) { + const MoveRecord & mr(_lru._moved[i]); + if (mr.first == from) { + from = mr.second; + } + } + return from; + } + ~RecordMoves() { + _lru._moveRecordingEnabled = false; + _lru._moved.clear(); + } + private: + lrucache_map & _lru; + }; + + size_t _maxElements; + mutable uint32_t _head; + mutable uint32_t _tail; + bool _moveRecordingEnabled; + MoveRecords _moved; +}; + +template< typename P > +lrucache_map<P>::lrucache_map(size_t maxElems) : + HashTable(0), + _maxElements(maxElems), + _head(LinkedValueBase::npos), + _tail(LinkedValueBase::npos), + _moveRecordingEnabled(false), + _moved() +{ +} + +template< typename P > +void +lrucache_map<P>::swap(lrucache_map & rhs) { + std::swap(_maxElements, rhs._maxElements); + std::swap(_head, rhs._head); + std::swap(_tail, rhs._tail); + HashTable::swap(rhs); +} + +template< typename P > +void +lrucache_map<P>::move(next_t from, next_t to) { + (void) from; + if (_moveRecordingEnabled) { + _moved.push_back(std::make_pair(from, to)); + } + value_type & moved = HashTable::getByInternalIndex(to); + if (moved.second._prev != LinkedValueBase::npos) { + HashTable::getByInternalIndex(moved.second._prev).second._next = to; + } else { + _head = to; + } + if (moved.second._next != LinkedValueBase::npos) { + HashTable::getByInternalIndex(moved.second._next).second._prev = to; + } else { + _tail = to; + } +} + +template< typename P > +void +lrucache_map<P>::erase(const K & key) { + internal_iterator it = HashTable::find(key); + if (it != HashTable::end()) { + LV & v = it->second; + if (v._prev != LinkedValueBase::npos) { + HashTable::getByInternalIndex(v._prev).second._next = v._next; + } else { + _head = v._next; + } + if (v._next != LinkedValueBase::npos) { + HashTable::getByInternalIndex(v._next).second._prev = v._prev; + } else { + _tail = v._prev; + } + HashTable::erase(*this, it); + } +} + +template< typename P > +typename lrucache_map<P>::iterator +lrucache_map<P>::erase(const iterator & it) +{ + iterator next(it); + if (it != end()) { + RecordMoves moves(*this); + next++; + const K & key(HashTable::getByInternalIndex(it._current).first); + erase(key); + next = iterator(this, moves.movedTo(next._current)); + } + return next; +} + +template< typename P > +bool +lrucache_map<P>::verifyInternals() +{ + bool retval(true); + assert(_head != LinkedValueBase::npos); + assert(_tail != LinkedValueBase::npos); + assert(HashTable::getByInternalIndex(_head).second._prev == LinkedValueBase::npos); + assert(HashTable::getByInternalIndex(_tail).second._next == LinkedValueBase::npos); + { + size_t i(0); + size_t prev(LinkedValueBase::npos); + size_t c(_head); + for(size_t m(size()); (c != LinkedValueBase::npos) && (i < m); c = HashTable::getByInternalIndex(c).second._next, i++) { + assert((HashTable::getByInternalIndex(c).second._prev == prev)); + prev = c; + } + assert(i == size()); + assert(c == LinkedValueBase::npos); + } + { + size_t i(0); + size_t next(LinkedValueBase::npos); + size_t c(_tail); + for(size_t m(size()); (c != LinkedValueBase::npos) && (i < m); c = HashTable::getByInternalIndex(c).second._prev, i++) { + assert((HashTable::getByInternalIndex(c).second._next == next)); + next = c; + } + assert(i == size()); + assert(c == LinkedValueBase::npos); + } + return retval; +} + +template< typename P > +void +lrucache_map<P>::move(NodeStore && oldStore) +{ + next_t curr(_tail); + _tail = LinkedValueBase::npos; + _head = LinkedValueBase::npos; + + while (curr != LinkedValueBase::npos) { + value_type & v = oldStore[curr].getValue(); + curr = v.second._prev; + v.second._prev = LinkedValueBase::npos; + v.second._next = LinkedValueBase::npos; + insert(std::move(v)); + } +} + +template< typename P > +void +lrucache_map<P>::removeOld() { + if (_tail != LinkedValueBase::npos) { + for (value_type * last(& HashTable::getByInternalIndex(_tail)); + (_tail != _head) && removeOldest(*last); + last = & HashTable::getByInternalIndex(_tail)) + { + _tail = last->second._prev; + HashTable::getByInternalIndex(_tail).second._next = LinkedValueBase::npos; + HashTable::erase(*this, HashTable::find(last->first)); + } + } +} + +template< typename P > +void +lrucache_map<P>::ref(const internal_iterator & it) { + uint32_t me(it.getInternalIndex()); + if (me != _head) { + LV & v = it->second; + LV & oldPrev = HashTable::getByInternalIndex(v._prev).second; + oldPrev._next = v._next; + if (me != _tail) { + LV & oldNext = HashTable::getByInternalIndex(v._next).second; + oldNext._prev = v._prev; + } else { + // I am tail and I am not the only one. + _tail = v._prev; + } + LV & oldHead = HashTable::getByInternalIndex(_head).second; + oldHead._prev = me; + v._next = _head; + v._prev = LinkedValueBase::npos; + _head = me; + } +} + +template< typename P > +typename lrucache_map<P>::insert_result +lrucache_map<P>::insert(value_type && value) { + insert_result res = HashTable::insertInternal(std::forward<value_type>(value)); + uint32_t next(_head); + if ( ! res.second) { + ref(res.first); + } else { + _head = res.first.getInternalIndex(); + HashTable::getByInternalIndex(_head).second._next = next; + if (next != LinkedValueBase::npos) { + HashTable::getByInternalIndex(next).second._prev = _head; + } + if (_tail == LinkedValueBase::npos) { + _tail = _head; + } + removeOld(); + if (_head != res.first.getInternalIndex()) { + res.first.setInternalIndex(_head); + } + } + return res; +} + +template< typename P > +typename P::Value & +lrucache_map<P>::operator [] (const K & key) +{ + return insert(key, V()).first->second._value; +} + +template< typename P > +typename lrucache_map<P>::internal_iterator +lrucache_map<P>::findAndRef(const K & key) +{ + internal_iterator found = HashTable::find(key); + if (found != HashTable::end()) { + ref(found); + } + return found; +} + +} + diff --git a/staging_vespalib/src/vespa/vespalib/stllike/smallvector.cpp b/staging_vespalib/src/vespa/vespalib/stllike/smallvector.cpp new file mode 100644 index 00000000000..59d71217c41 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/stllike/smallvector.cpp @@ -0,0 +1,2 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "smallvector.h" diff --git a/staging_vespalib/src/vespa/vespalib/stllike/smallvector.h b/staging_vespalib/src/vespa/vespalib/stllike/smallvector.h new file mode 100644 index 00000000000..931e8ef433d --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/stllike/smallvector.h @@ -0,0 +1,191 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * A vector type implementation that is optimized for keeping a small amount of + * elements. If a small amount is kept, no malloc will be done within the + * vector implementation. + */ + +#pragma once + +#include <vespa/fastos/fastos.h> +#include <iterator> +#include <memory> +#include <vector> + +namespace vespalib { + +/** + * A generic iterator implementation using size() and operator[] to access + * elements. + */ +template<typename Container, typename T> +class IndexedContainerIterator + : public std::iterator<std::random_access_iterator_tag, T> +{ + Container* _container; + uint64_t _index; + +public: + typedef IndexedContainerIterator<Container, T> Iterator; + typedef typename std::iterator<std::random_access_iterator_tag, T>::difference_type difference_type; + // Required to be possible to default construct iterators + IndexedContainerIterator() : _container(0), _index(-1) {} + IndexedContainerIterator(Container& c, uint64_t index) + : _container(&c), _index(index) {} + + T& operator*() { return (*_container)[_index]; } + T* operator->() { return &(*_container)[_index]; } + + bool operator==(const Iterator& o) const { + return (_index == o._index); + } + bool operator!=(const Iterator& o) const { + return (_index != o._index); + } + bool operator<(const Iterator& o) const { + return (_index < o._index); + } + + Iterator& operator++() { + ++_index; + return *this; + } + Iterator operator++(int) { + return Iterator(*_container, _index++); + } + Iterator& operator--() { + --_index; + return *this; + } + Iterator operator--(int) { + return Iterator(*_container, _index--); + } + + Iterator operator+(const difference_type& v) { + return Iterator(*_container, _index + v); + } + Iterator operator-(const difference_type& v) { + return Iterator(*_container, _index - v); + } + difference_type operator-(const Iterator& o) { + return _index - o._index; + } +}; + +template <typename T, size_t S = 8> +class SmallVector { + size_t _size; + T _smallVector[S]; + std::vector<T> _bigVector; + +public: + typedef IndexedContainerIterator<SmallVector<T, S>, T> iterator; + typedef IndexedContainerIterator<const SmallVector<T, S>, const T> const_iterator; + typedef T value_type; + typedef T& reference; + typedef const T& const_reference; + typedef size_t difference_type; + typedef size_t size_type; + + iterator begin() { return iterator(*this, 0); } + iterator end() { return iterator(*this, _size); } + const_iterator begin() const { return const_iterator(*this, 0); } + const_iterator end() const { return const_iterator(*this, _size); } + + SmallVector() : _size(0) {} + + SmallVector(std::initializer_list<T> elems) + : _size(0) + { + for (auto it=elems.begin(); it != elems.end(); ++it) { + push_back(*it); + } + } + + SmallVector(const SmallVector<T, S>& other) = delete; + SmallVector<T, S>& operator=(const SmallVector<T, S>& other) = delete; + + size_t getEfficientSizeLimit() const { return S; } + + void push_back(const T& t) { + if (_size < S) { + _smallVector[_size] = t; + ++_size; + } else { + if (_size == S) { + populateVector(); + } + _bigVector.push_back(t); + ++_size; + } + } + void pop_back() { + if (_size <= S) { + --_size; + } else { + if (--_size == S) { + _bigVector.clear(); + } else { + _bigVector.pop_back(); + } + } + } + const T& back() const { return operator[](_size - 1); } + T& back() { return operator[](_size - 1); } + const T& front() const { return operator[](0); } + T& front() { return operator[](0); } + void clear() { + _size = 0; + _bigVector.clear(); + } + const T& operator[](size_t i) const { + if (i < S) { + return _smallVector[i]; + } else { + return _bigVector[i]; + } + } + T& operator[](size_t i) { + if (i < S) { + return _smallVector[i]; + } else { + return _bigVector[i]; + } + } + bool empty() const { return (_size == 0); } + size_t size() const { return _size; } + + template<typename O> + bool operator==(const O& o) const { + if (size() != o.size()) return false; + for (size_t i=0; i<_size; ++i) { + if ((*this)[i] != o[i]) return false; + } + return true; + } + template<typename O> + bool operator!=(const O& o) const { + return !(operator==(o)); + } + + void erase(iterator eraseIt) { + SmallVector<T, S> copy; + for (auto it = begin(); it != end(); ++it) { + if (it != eraseIt) { + copy.push_back(*it); + } + } + copy.swap(*this); + } + +private: + void populateVector() { + _bigVector.reserve(S+1); + for (size_t i=0; i<S; ++i) { + _bigVector.push_back(_smallVector[i]); + } + } +}; + + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/trace/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/trace/CMakeLists.txt new file mode 100644 index 00000000000..f2e583a2dfa --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/trace/CMakeLists.txt @@ -0,0 +1,7 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(staging_vespalib_vespalib_trace OBJECT + SOURCES + slime_trace_serializer.cpp + slime_trace_deserializer.cpp + DEPENDS +) diff --git a/staging_vespalib/src/vespa/vespalib/trace/slime_trace_deserializer.cpp b/staging_vespalib/src/vespa/vespalib/trace/slime_trace_deserializer.cpp new file mode 100644 index 00000000000..fc28dfbb40b --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/trace/slime_trace_deserializer.cpp @@ -0,0 +1,68 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/trace/slime_trace_deserializer.h> +#include <vespa/vespalib/trace/slime_trace_serializer.h> +#include <vespa/vespalib/trace/tracenode.h> + +using namespace vespalib::slime; + +namespace vespalib { + +SlimeTraceDeserializer::SlimeTraceDeserializer(const Inspector & inspector) + : _inspector(inspector) +{ +} + +TraceNode +SlimeTraceDeserializer::deserialize() const +{ + return deserialize(_inspector); +} + +TraceNode +SlimeTraceDeserializer::deserialize(const Inspector & inspector) +{ + TraceNode node(deserializeTraceNode(inspector)); + deserializeChildren(inspector[SlimeTraceSerializer::CHILDREN], node); + return node; +} + +TraceNode +SlimeTraceDeserializer::deserializeTraceNode(const Inspector & inspector) +{ + int64_t timestamp(decodeTimestamp(inspector)); + if (hasPayload(inspector)) { + std::string note(decodePayload(inspector)); + return TraceNode(note, timestamp); + } + return TraceNode(timestamp); +} + +bool +SlimeTraceDeserializer::hasPayload(const Inspector & inspector) +{ + return inspector[SlimeTraceSerializer::PAYLOAD].valid(); +} + +vespalib::string +SlimeTraceDeserializer::decodePayload(const Inspector & inspector) +{ + return inspector[SlimeTraceSerializer::PAYLOAD].asString().make_string(); +} + +int64_t +SlimeTraceDeserializer::decodeTimestamp(const Inspector & inspector) +{ + return inspector[SlimeTraceSerializer::TIMESTAMP].asLong(); +} + +void +SlimeTraceDeserializer::deserializeChildren(const Inspector & inspector, TraceNode & node) +{ + for (size_t i(0); i < inspector.children(); i++) { + Inspector & child(inspector[i]); + TraceNode childNode(deserialize(child)); + node.addChild(childNode); + } +} + +} diff --git a/staging_vespalib/src/vespa/vespalib/trace/slime_trace_deserializer.h b/staging_vespalib/src/vespa/vespalib/trace/slime_trace_deserializer.h new file mode 100644 index 00000000000..dc8c6dc5cff --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/trace/slime_trace_deserializer.h @@ -0,0 +1,31 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/stllike/string.h> +#include <stack> + +namespace vespalib { + +class TraceNode; + +/** + * For deserializing a TraceNode and its children. Note that the ordering of nodes + * are NOT guaranteed. + */ +class SlimeTraceDeserializer +{ +public: + SlimeTraceDeserializer(const slime::Inspector & inspector); + TraceNode deserialize() const; +private: + static TraceNode deserialize(const slime::Inspector & inspector); + static TraceNode deserializeTraceNode(const slime::Inspector & inspector); + static void deserializeChildren(const slime::Inspector & inspector, TraceNode & node); + static bool hasPayload(const slime::Inspector & inspector); + static int64_t decodeTimestamp(const slime::Inspector & inspector); + static vespalib::string decodePayload(const slime::Inspector & inspector); + const slime::Inspector & _inspector; +}; + +} diff --git a/staging_vespalib/src/vespa/vespalib/trace/slime_trace_serializer.cpp b/staging_vespalib/src/vespa/vespalib/trace/slime_trace_serializer.cpp new file mode 100644 index 00000000000..4332d6bf045 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/trace/slime_trace_serializer.cpp @@ -0,0 +1,62 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/trace/slime_trace_serializer.h> +#include <vespa/vespalib/trace/trace.h> + +using namespace vespalib::slime; + +namespace vespalib { + +const Memory SlimeTraceSerializer::TIMESTAMP("timestamp"); +const Memory SlimeTraceSerializer::PAYLOAD("payload"); +const Memory SlimeTraceSerializer::CHILDREN("children"); + +SlimeTraceSerializer::SlimeTraceSerializer(Cursor & cursor) + : _cursors() +{ + _cursors.push(&cursor); +} + +void +SlimeTraceSerializer::visit(const TraceNode & node) +{ + assert(!_cursors.empty()); + Cursor * current(_cursors.top()); + assert(current != NULL); + _cursors.pop(); + addTimestamp(*current, node); + addPayload(*current, node); + addChildrenCursors(*current, node); +} + +void +SlimeTraceSerializer::addTimestamp(Cursor & current, const TraceNode & node) +{ + current.setLong(TIMESTAMP, node.getTimestamp()); +} + +void +SlimeTraceSerializer::addPayload(Cursor & current, const TraceNode & node) +{ + if (node.hasNote()) { + current.setString(PAYLOAD, Memory(node.getNote())); + } +} + +void +SlimeTraceSerializer::addChildrenCursors(Cursor & current, const TraceNode & node) +{ + if (node.getNumChildren() > 0) { + addChildrenCursorsToStack(current.setArray(CHILDREN), node); + } +} + +void +SlimeTraceSerializer::addChildrenCursorsToStack(Cursor & childrenArray, const TraceNode & node) +{ + for (uint32_t childIndex(0); childIndex < node.getNumChildren(); childIndex++) { + Cursor & child(childrenArray.addObject()); + _cursors.push(&child); + } +} + +} diff --git a/staging_vespalib/src/vespa/vespalib/trace/slime_trace_serializer.h b/staging_vespalib/src/vespa/vespalib/trace/slime_trace_serializer.h new file mode 100644 index 00000000000..3b50f0878bf --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/trace/slime_trace_serializer.h @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/trace/tracevisitor.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <stack> + +namespace vespalib { + +class TraceNode; + +/** + * A serializer of TraceNodes using the TraceVisitor API. The serialized order of the nodes are NOT + * guaranteed to be in the same order as the original. + */ +class SlimeTraceSerializer : public TraceVisitor +{ +public: + SlimeTraceSerializer(slime::Cursor & cursor); + void visit(const TraceNode & node); + static const slime::Memory TIMESTAMP; + static const slime::Memory PAYLOAD; + static const slime::Memory CHILDREN; +private: + void addTimestamp(slime::Cursor & current, const TraceNode & node); + void addPayload(slime::Cursor & current, const TraceNode & node); + void addChildrenCursors(slime::Cursor & current, const TraceNode & node); + void addChildrenCursorsToStack(slime::Cursor & childrenArray, const TraceNode & node); + std::stack<slime::Cursor *> _cursors; +}; + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/.gitignore b/staging_vespalib/src/vespa/vespalib/util/.gitignore new file mode 100644 index 00000000000..ee8938b6bf4 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/.gitignore @@ -0,0 +1,6 @@ +*.So +*.exe +*.ilk +*.pdb +.depend* +Makefile diff --git a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt new file mode 100644 index 00000000000..8cf7e2239ad --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -0,0 +1,23 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(staging_vespalib_vespalib_util OBJECT + SOURCES + bits.cpp + clock.cpp + cpu.cpp + crc.cpp + doom.cpp + growablebytebuffer.cpp + jsonstream.cpp + jsonwriter.cpp + librarypool.cpp + md5.c + process_memory_stats.cpp + programoptions.cpp + programoptions_testutils.cpp + document_runnable.cpp + rusage.cpp + shutdownguard.cpp + timer.cpp + xmlserializable.cpp + DEPENDS +) diff --git a/staging_vespalib/src/vespa/vespalib/util/allocinarray.h b/staging_vespalib/src/vespa/vespalib/util/allocinarray.h new file mode 100644 index 00000000000..a6a89c4d41a --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/allocinarray.h @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/util/alloc.h> +#include <vespa/vespalib/util/array.h> + +namespace vespalib { + +/** + * This is a class that lets you allocate memory tightly. It uses an vector interface + * for storing objects of the same type efficiently. It also providex vector like access with []. + * New objects are just appended to the backing vector, or if there are holes they are inserted there. + * freed objects are not destructed upon free, but rather when the place is required again. + * That happens either + * - on free if it is the last element in the backing vector. + * - when the AllocInArray goes out of scope. + * - on an explicit clear. +**/ +template <typename T, typename V=vespalib::Array<T, HeapAlloc> > +class AllocInArray { +public: + typedef uint32_t Index; + void reserve(size_t sz) { _v.reserve(sz); } + Index alloc(const T & v); + void free(Index p); + const T & operator [] (Index p) const { return _v[p]; } + T & operator [] (Index p) { return _v[p]; } + void clear(); + size_t size() const { return _v.size() - _free.size(); } +private: + Index last() const { return _v.size() - 1; } + typedef vespalib::Array<Index> FreeList; + V _v; + FreeList _free; +}; + +template <typename T, typename V> +typename AllocInArray<T, V>::Index +AllocInArray<T, V>::alloc(const T & v) +{ + if (_free.empty()) { + _v.push_back(v); + return last(); + } else { + Index p(_free.back()); + _free.pop_back(); + _v[p] = v; + return p; + } +} + +template <typename T, typename V> +void +AllocInArray<T, V>::free(Index p) +{ + if (p == last()) { + _v.pop_back(); + } else if ( p < _v.size()) { + _free.push_back(p); + } +} + +template <typename T, typename V> +void +AllocInArray<T, V>::clear() +{ + _v.clear(); + _free.clear(); +} + +} + diff --git a/staging_vespalib/src/vespa/vespalib/util/bits.cpp b/staging_vespalib/src/vespa/vespalib/util/bits.cpp new file mode 100644 index 00000000000..c60a17ae72d --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/bits.cpp @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/util/bits.h> + +namespace vespalib { + +uint8_t Bits::_reverse[256]; +Bits::ReverseTableInit Bits::_reverseTableInit; + +void * Bits::reverse(void * srcDst, size_t sz) +{ + uint8_t *v(static_cast<uint8_t *>(srcDst)); + size_t i(0); + for(; i < sz/2; i++) { + v[i] = reverse(v[sz-1-i]); + } + return v; +} + +void Bits::forceInitNow() +{ + ReverseTableInit now; +} + +Bits::ReverseTableInit::ReverseTableInit() +{ + if (_reverse[128] == 0) { + for (size_t i(0); i < 256; i++) { + _reverse[i] = reverse(i); + } + } +} + +uint8_t Bits::ReverseTableInit::reverse(uint8_t v) +{ + return ((v >> 7) & 0x01) | + ((v >> 5) & 0x02) | + ((v >> 3) & 0x04) | + ((v >> 1) & 0x08) | + ((v << 1) & 0x10) | + ((v << 3) & 0x20) | + ((v << 5) & 0x40) | + ((v << 7) & 0x80); +} + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/bits.h b/staging_vespalib/src/vespa/vespalib/util/bits.h new file mode 100644 index 00000000000..8926cdfec64 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/bits.h @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/fastos/fastos.h> + +namespace vespalib { + +/** + * @brief Bit fiddling class + * + * This class handles low level bit manipulation operations. + * - Fast bit reversal by table lookup. + **/ +class Bits +{ +public: + static uint8_t reverse(uint8_t v) { return _reverse[v]; } + static uint16_t reverse(uint16_t v) { + union { uint16_t v; uint8_t a[2]; } s, d; + s.v = v; + d.a[0] = _reverse[s.a[1]]; + d.a[1] = _reverse[s.a[0]]; + return d.v; + } + static uint32_t reverse(uint32_t v) { + union { uint32_t v; uint8_t a[4]; } s, d; + s.v = v; + d.a[0] = _reverse[s.a[3]]; + d.a[1] = _reverse[s.a[2]]; + d.a[2] = _reverse[s.a[1]]; + d.a[3] = _reverse[s.a[0]]; + return d.v; + } + static uint64_t reverse(uint64_t v) { + union { uint64_t v; uint8_t a[8]; } s, d; + s.v = v; + d.a[0] = _reverse[s.a[7]]; + d.a[1] = _reverse[s.a[6]]; + d.a[2] = _reverse[s.a[5]]; + d.a[3] = _reverse[s.a[4]]; + d.a[4] = _reverse[s.a[3]]; + d.a[5] = _reverse[s.a[2]]; + d.a[6] = _reverse[s.a[1]]; + d.a[7] = _reverse[s.a[0]]; + return d.v; + } + static void * reverse(void * v, size_t sz); + /** + Utility for other statically contructed objects to force correct init order." + **/ + static void forceInitNow(); +private: + class ReverseTableInit + { + public: + ReverseTableInit(); + static uint8_t reverse(uint8_t v); + }; + static uint8_t _reverse[256]; + static ReverseTableInit _reverseTableInit; +}; + +} + + diff --git a/staging_vespalib/src/vespa/vespalib/util/blockwritermutex.hpp b/staging_vespalib/src/vespa/vespalib/util/blockwritermutex.hpp new file mode 100644 index 00000000000..d238fed18bf --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/blockwritermutex.hpp @@ -0,0 +1,67 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vector> + +#include <boost/noncopyable.hpp> +#include <boost/thread/locks.hpp> +#include <boost/thread/mutex.hpp> +#include <boost/thread/condition_variable.hpp> + + +namespace vespalib { + + +/* Blocks a writer from being called while readers are active and vice versa. + * This is only intended to be used by a single writer. + */ +class BlockWriterMutex : boost::noncopyable { + typedef boost::mutex ReadersMutex; + + ReadersMutex _readersMutex; + int _readers; + + boost::condition_variable _noReaders; + + void lockImpl(int sign) { + boost::unique_lock<ReadersMutex> readersLock(_readersMutex); + while ((sign*_readers) > 0) + _noReaders.wait(readersLock); + _readers -= sign; + } + + void unlockImpl(int sign) { + ReadersMutex::scoped_lock readersLock(_readersMutex); + _readers += sign; + if (_readers == 0) + _noReaders.notify_all(); + } + +public: + typedef boost::shared_lock<BlockWriterMutex> ReaderLock; + typedef boost::unique_lock<BlockWriterMutex> WriterLock; + + BlockWriterMutex() + :_readers(0) + {} + + void lock() { + lockImpl(1); + } + + void unlock() { + unlockImpl(1); + } + + void lock_shared() { + lockImpl(-1); + } + + void unlock_shared() { + unlockImpl(-1); + } +}; + +} //namespace vespalib + + diff --git a/staging_vespalib/src/vespa/vespalib/util/clock.cpp b/staging_vespalib/src/vespa/vespalib/util/clock.cpp new file mode 100644 index 00000000000..951f9797d62 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/clock.cpp @@ -0,0 +1,53 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "clock.h" +#include <vespa/fastos/timestamp.h> + +using namespace fastos; + +namespace vespalib { + + +Clock::Clock(double timePeriod) : + _timeNS(0u), + _timePeriodMS(static_cast<uint32_t>(timePeriod*1000)), + _cond(), + _stop(false), + _running(false) +{ + setTime(); +} + +Clock::~Clock() +{ + assert(!_running); +} + +void Clock::setTime() const +{ + _timeNS = ClockSystem::adjustTick2Sec(ClockSystem::now()); +} + +void Clock::Run(FastOS_ThreadInterface *thread, void *arguments) +{ + (void) arguments; + _running = true; + _cond.Lock(); + while ( ! thread->GetBreakFlag() && !_stop) { + setTime(); + _cond.TimedWait(_timePeriodMS); + } + _cond.Unlock(); + _running = false; +} + +void +Clock::stop(void) +{ + _cond.Lock(); + _stop = true; + _cond.Broadcast(); + _cond.Unlock(); +} + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/clock.h b/staging_vespalib/src/vespa/vespalib/util/clock.h new file mode 100644 index 00000000000..8eb8e147a35 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/clock.h @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/fastos/fastos.h> +#include <vespa/fastos/timestamp.h> + +namespace vespalib { + +/** + * Clock is a clock that updates the time at defined intervals. + * It is intended used where you want to check the time with low cost, but where + * resolution is not that important. + */ + +class Clock : public FastOS_Runnable +{ +private: + Clock(const Clock &); + Clock & operator = (const Clock &); + + mutable fastos::TimeStamp _timeNS; + int _timePeriodMS; + FastOS_Cond _cond; + bool _stop; + bool _running; + + void setTime() const; + + virtual void Run(FastOS_ThreadInterface *thisThread, void *arguments); + +public: + Clock(double timePeriod=0.100); + ~Clock(); + + fastos::TimeStamp getTimeNS(void) const { + if (!_running) { + setTime(); + } + return _timeNS; + } + fastos::TimeStamp getTimeNSAssumeRunning() const { return _timeNS; } + + void stop(void); +}; + +} + diff --git a/staging_vespalib/src/vespa/vespalib/util/cpu.cpp b/staging_vespalib/src/vespa/vespalib/util/cpu.cpp new file mode 100644 index 00000000000..2a65f6ea2ad --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/cpu.cpp @@ -0,0 +1,142 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/util/cpu.h> + +namespace vespalib { + +#if defined(__i386__) || defined(__x86_64) + +X86CpuInfo::CpuFeature X86CpuInfo::_CpuFeatureList[64] = +{ + { 0, "FPU", "Floating-point unit on-Chip", "The processor contains an FPU that supports the Intel387 floating-point instruction set." }, + { 1, "VME", "Virtual Mode Extension", "The processor supports extensions to virtual-8086 mode." }, + { 2, "DE", "Debugging Extension", "The processor supports I/O breakpoints, including the CR4.DE bit for enabling debug extensions and optional trapping of access to the DR4 and DR5 registers." }, + { 3, "PSE", "Page Size Extension", "The processor supports 4-Mbyte pages." }, + { 4, "TSC", "Time Stamp Counter", "The RDTSC instruction is supported including the CR4.TSD bit for access/privilege control." }, + { 5, "MSR", "Model Specific Registers", "Model Specific Registers are implemented with the RDMSR, WRMSR instructions." }, + { 6, "PAE", "Physical Address Extension", "Physical addresses greater than 32 bits are supported." }, + { 7, "MCE", "Machine Check Exception", "Machine Check Exception, Exception 18, and the CR4.MCE enable bit are supported" }, + { 8, "CX8", "CMPXCHG8 Instruction Supported", "The compare and exchange 8 bytes instruction is supported." }, + { 9, "APIC", "On-chip APIC Hardware Supported", "The processor contains a software-accessible Local APIC." }, + { 10, "RES", "Reserved", "Do not count on their value." }, + { 11, "SEP", "Fast System Call", "Indicates whether the processor supports the Fast System Call instructions, SYSENTER and SYSEXIT. NOTE: Refer to Section 3.4 for further information regarding SYSENTER/ SYSEXIT feature and SEP feature bit." }, + { 12, "MTRR", "Memory Type Range Registers", "The Processor supports the Memory Type Range Registers specifically the MTRR_CAP register." }, + { 13, "PGE", "Page Global Enable", "The global bit in the page directory entries (PDEs) and page table entries (PTEs) is supported, indicating TLB entries that are common to different processes and need not be flushed. The CR4.PGE bit controls this feature." }, + { 14, "MCA", "Machine Check Architecture", "The Machine Check Architecture is supported, specifically the MCG_CAP register." }, + { 15, "CMOV", "Conditional Move Instruction Supported", "The processor supports CMOVcc, and if the FPU feature flag (bit 0) is also set, supports the FCMOVCC and FCOMI instructions." }, + { 16, "PAT", "Page Attribute Table", "Indicates whether the processor supports the Page Attribute Table. This feature augments the Memory Type Range Registers (MTRRs), allowing an operating system to specify attributes of memory on 4K granularity through a linear address." }, + { 17, "PSE-36", "36-bit Page Size Extension", "Indicates whether the processor supports 4-Mbyte pages that are capable of addressing physical memory beyond 4GB. This feature indicates that the upper four bits of the physical address of the 4-Mbyte page is encoded by bits 13-16 of the page directory entry." }, + { 18, "PSN", "Processor serial number is present and enabled", "The processor supports the 96-bit processor serial number feature, and the feature is enabled." }, + { 19, "CLFSH", "CLFLUSH Instruction supported", "Indicates that the processor supports the CLFLUSH instruction." }, + { 20, "RES", "Reserved", "Do not count on their value." }, + { 21, "DS", "Debug Store", "Indicates that the processor has the ability to write a history of the branch to and from addresses into a memory buffer." }, + { 22, "ACPI", "Thermal Monitor and Software Controlled Clock Facilities supported", "The processor implements internal MSRs that allow processor temperature to be monitored and processor performance to be modulated in predefined duty cycles under software control." }, + { 23, "MMX", "Intel Architecture MMX technology supported", "The processor supports the MMX technology instruction set extensions to Intel Architecture." }, + { 24, "FXSR", "Fast floating point save and restore", "Indicates whether the processor supports the FXSAVE and FXRSTOR instructions for fast save and restore of the floating point context. Presence of this bit also indicates that CR4.OSFXSR is available for an operating system to indicate that it uses the fast save/restore instructions." }, + { 25, "SSE", "Streaming SIMD Extensions supported", "The processor supports the Streaming SIMD Extensions to the Intel Architecture." }, + { 26, "SSE2", "Streaming SIMD Extensions 2", "Indicates the processor supports the Streaming SIMD Extensions - 2 Instructions." }, + { 27, "SS", "Self-Snoop", "The processor supports the management of conflicting memory types by performing a snoop of its own cache structure for transactions issued to the bus." }, + { 28, "HTT", "Hyper-Threading Technology", "The processor supports Hyper-Threading Technology." }, + { 29, "TM", "Thermal Monitor supported", "The processor implements the Thermal Monitor automatic thermal control circuit (TCC)." }, + { 30, "IA64", "IA64 Capabilities", "The processor is a member of the Intel Itanium processor family and currently operating in IA32 emulation mode." }, + { 31, "PBE", "Pending Break Enable", "The processor supports the use of the FERR#/PBE# pin when th eprocessor is in the stop-clock state(STPCLK# is asserted) to signal the processor that an interrupt is pending and that the processor should return to normal operation to handle the interrupt. Bit 10 (PBE enable) in the IA32_MISc_ENABLE MSR enables this capability." }, + + { 32, "SSE3", "Streaming SIMD Extensions 3", "The processor supports the Streaming SIMD Extensions 3 instructions." }, + { 33, "RES", "Reserved", "Do not count on their value." }, + { 34, "DTES64", "64-Bit Debug Store", "Indicates that the processor has the ability to write a history of the 64-bit branch to and from addresses into a memory buffer." }, + { 35, "MONITOR", "MONITOR/MWAIT", "The processor supports the MONITOR and MWAIT instructions." }, + { 36, "DS-CPL", "CPL Qualified Debug Store", "The processor supports the extensions to the Debug Store feature to allow for branch message storage qualified by CPL." }, + { 37, "VMX", "Virtual Machine Extensions", "The processor supports Intel Virtualization Technology." }, + { 38, "SMX", "Safer Mode Extensions", "The processor supports Intel Trusted Execution Technology." }, + { 39, "EST", "Enhanced Intel SpeedStep", "The processor supports Enhanced Intel SpeedStep Technology and implements the IA32_PERF_STS and IA32_PERF_CTL registers." }, + { 40, "TM2", "Thermal Monitor 2", "The processor implements the Thermal Monitor 2 thermal control circuit (TCC)." }, + { 41, "SSSE3", "Supplemental Streaming SIMD Extensions 3", "The processor supports the Supplemental Streaming SIMD Extensions 3 instructions." }, + { 42, "CID", "L1 Context ID", "The L1 data cache mode can be set to either adaptive mode or shared mode by the BIOS." }, + { 43, "RES", "Reserved", "Do not count on their value." }, + { 44, "RES", "Reserved", "Do not count on their value." }, + { 45, "CX16", "CMPXCHG16B", "This processor supports the CMPXCHG16B instruction." }, + { 46, "xTPR", "Send Task Priority Messages", "The processor supports the ability to disable sending Task Priority messages. When this feature flag is set, Task Priority messages may be disabled. Bit 23 (Echo TPR disable) in the IA32_MISC_ENABLE MSR controls the sending of Task Priority messages." }, + { 47, "PDCM", "Perfmon and Debug Capability", "The processor supports the Performance Capabilities MSR. IA32_PERF_CAPABILITIES register is MSR 345h." }, + { 48, "RES", "Reserved", "Do not count on their value." }, + { 49, "RES", "Reserved", "Do not count on their value." }, + { 50, "DCA", "Direct Cache Access", "The processor supports the ability to prefetch data from a memory mapped device." }, + { 51, "SSS4.1", "Streaming SIMD Extensions 4.1", "The processor supports the Streaming SIMD Extensions 4.1 instructions." }, + { 52, "SSE4.2", "Streaming SIMD Extensions 4.2", "The processor supports the Streaming SIMD Extensions 4.2 instructions." }, + { 53, "x2APIC", "Extended xAPIC Support", "The processor supports x2APIC feature." }, + { 54, "MOVBE", "MOVBE Instruction", "The processor supports MOVBE instruction." }, + { 55, "POPCNT", "POPCNT Instruction", "The processor supports the POPCNT instruction." }, + { 56, "RES", "Reserved", "Do not count on their value." }, + { 57, "RES", "Reserved", "Do not count on their value." }, + { 58, "XSAVE", "XSAVE/XSTOR States", "The processor supports the XSAVE/XRSTOR processor extended states feature, the XSETBV/ XGETBV instructions, and the XFEATURE_ENABLED_MASK register (XCR0)." }, + { 59, "OXSAVE", "OS Enabled XSAVE", "A value of 1 indicates that the OS has enabled XSETBV/XGETBV instructions to access the XFEATURE_ENABLED_MASK register (XCR0), and support for processor extended state management using XSAVE/XRSTOR." }, + { 60, "RES", "Reserved", "Do not count on their value." }, + { 61, "RES", "Reserved", "Do not count on their value." }, + { 62, "RES", "Reserved", "Do not count on their value." }, + { 63, "RES", "Reserved", "Do not count on their value." } + +}; + +// private empty constructor +X86CpuInfo::X86CpuInfo() +{ + memset(this, 0, sizeof(*this)); +} + +X86CpuInfo X86CpuInfo::_CpuInfoSingleton; +X86CpuInfo* X86CpuInfo::_CpuInfoSingletonPtr = 0; + +void X86CpuInfo::fillSingleton() +{ + X86CpuInfo tmp; + + uint32_t b, c, d; + Cpu::cpuid(0, tmp.largestStandardFunction, b, c, d); + memcpy(tmp.cpuName+0, &b, 4); + memcpy(tmp.cpuName+4, &d, 4); + memcpy(tmp.cpuName+8, &c, 4); + + Cpu::cpuid(1, tmp.versionInfo, tmp.apicInfo, tmp.extendedFeatures, tmp.mainFeatures); + uint64_t cf = tmp.extendedFeatures; + cf = (cf << 32) | tmp.mainFeatures; + tmp._cpuFeatures = cf; + + // add more cpuid extraction here + + _CpuInfoSingleton = tmp; +} + +void X86CpuInfo::print(FILE * fp) +{ + uint32_t stepping = (cpuInfo().versionInfo & 0xF); + uint32_t baseModel = (cpuInfo().versionInfo >> 4) & 0xF; + uint32_t baseFamily = (cpuInfo().versionInfo >> 8) & 0xF; + uint32_t extModel = (cpuInfo().versionInfo >> 16) & 0xF; + uint32_t extFamily = (cpuInfo().versionInfo >> 20) & 0xFF; + + fprintf(fp, "cpuFeatures=%0x, cpuExtendedFeatures=%0x, family %d/%d, model %d/%d, stepping=%d\n\n", + cpuInfo().mainFeatures, cpuInfo().extendedFeatures, + baseFamily, extFamily, baseModel, extModel, stepping); + + fprintf(fp, "largestStandardFunction=%d, cpuName=%s\n", + cpuInfo().largestStandardFunction, cpuInfo().cpuName); + + uint64_t cpuFeatures = cpuInfo()._cpuFeatures; + for (unsigned int i = 0; i < sizeof(_CpuFeatureList)/sizeof(_CpuFeatureList[0]); i++) { + const CpuFeature & f = _CpuFeatureList[i]; + if ((cpuFeatures >> i ) & 1) { + fprintf(fp, "Feature #%d = %s\t%s\n\t%s\n", i, f.Name, f.Description, f.Comment); + } + } +} + + +#endif + +} + +#if 0 +int main(int, char **) +{ + vespalib::X86CpuInfo::print(stdout); +} +#endif diff --git a/staging_vespalib/src/vespa/vespalib/util/cpu.h b/staging_vespalib/src/vespa/vespalib/util/cpu.h new file mode 100644 index 00000000000..2288a1fc1e2 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/cpu.h @@ -0,0 +1,96 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright (C) 2003 Fast Search & Transfer ASA +// Copyright (C) 2003 Overture Services Norway AS + +#pragma once + +#include <vespa/fastos/fastos.h> + +namespace vespalib { + +#if defined(__i386__) || defined(__x86_64__) +class X86CpuInfo +{ +public: + bool hasMMX() const { return hasFeature(23); } + bool hasSSE() const { return hasFeature(25); } + bool hasSSE2() const { return hasFeature(26); } + bool hasSSE3() const { return hasFeature(32); } + bool hasCX16() const { return hasFeature(45); } + + bool hasFeature(size_t i) const { return (_cpuFeatures >> i) & 1; } + bool hasFeature(const char *name) const; + static void print(FILE * fp); + + /** + * Use this accessor to get information about the CPU. + * Only the first access will actually extract data, all + * subsequent uses are very cheap. + **/ + static const X86CpuInfo& cpuInfo() { + if (_CpuInfoSingletonPtr == 0) { + fillSingleton(); + _CpuInfoSingletonPtr = &_CpuInfoSingleton; + } + return *_CpuInfoSingletonPtr; + } + +private: + struct CpuFeature + { + size_t BitNo; + const char *Name; + const char *Description; + const char *Comment; + }; + static CpuFeature _CpuFeatureList[]; + + static X86CpuInfo _CpuInfoSingleton; + static X86CpuInfo *_CpuInfoSingletonPtr; + static void fillSingleton(); + + // no outside construction allowed: + X86CpuInfo(); + + uint64_t _cpuFeatures; + +public: + // public data extracted from CPU + uint32_t mainFeatures; + uint32_t extendedFeatures; + uint32_t apicInfo; // brand, CLFLUSH line size, logical / physical, local apic ID + uint32_t versionInfo; // family, model, stepping + + // add more info later: + // uint32_t numCpuInfo; + + char cpuName[13]; + uint32_t largestStandardFunction; + +}; +#endif + +/** + * @brief Atomic instructions class + * + * Here are fast handcoded assembly functions for doing some special + * low level functions that can be carried out very fast by special instructions. + * Currently only implemented for GCC on i386 and x86_64 platforms. + **/ +class Cpu +{ +public: +#if defined(__x86_64__) + static void cpuid(int op, uint32_t & eax, uint32_t & ebx, uint32_t & ecx, uint32_t & edx) { + __asm__("cpuid" + : "=a" (eax), + "=b" (ebx), + "=c" (ecx), + "=d" (edx) + : "0" (op)); + } +#endif +}; + +} + diff --git a/staging_vespalib/src/vespa/vespalib/util/crc.cpp b/staging_vespalib/src/vespa/vespalib/util/crc.cpp new file mode 100644 index 00000000000..88a27f71c7e --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/crc.cpp @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/util/crc.h> + +namespace vespalib { + +uint32_t crc_32_type::_crc[256]; +crc_32_type::CrcTableInit crc_32_type::_crcTableInit; + +uint32_t crc_32_type::crc(const void * src, size_t sz) +{ + const uint8_t *v = static_cast<const uint8_t *>(src); + uint32_t c(uint32_t(-1)); + + for (size_t i(0); i < sz; i++ ) { + c = (c >> 8) ^ _crc[ uint8_t(c ^ v[i]) ]; + } + + return c ^ (uint32_t(-1)); +} + +void crc_32_type::process_bytes(const void *start, size_t sz) +{ + const uint8_t *v = static_cast<const uint8_t *>(start); + uint32_t c(_c); + + for (size_t i(0); i < sz; i++ ) { + c = (c >> 8) ^ _crc[ uint8_t(c ^ v[i]) ]; + } + _c = c; +} + +crc_32_type::CrcTableInit::CrcTableInit() +{ + Bits::forceInitNow(); + uint8_t dividend = 0; + do { + _crc[Bits::reverse(dividend)] = crc(dividend); + } while ( ++dividend ); +} + +uint32_t crc_32_type::CrcTableInit::crc(uint8_t dividend) +{ + const uint32_t fast_hi_bit = 1ul << 31; + const uint8_t byte_hi_bit = 1u << 7; + uint32_t remainder = 0; + + // go through all the dividend's bits + for ( uint8_t mask = byte_hi_bit ; mask ; mask >>= 1 ) { + // check if divisor fits + if ( dividend & mask ) { + remainder ^= fast_hi_bit; + } + + // do polynominal division + if ( remainder & fast_hi_bit ) { + remainder <<= 1; + remainder ^= 0x04C11DB7; + } else { + remainder <<= 1; + } + } + return Bits::reverse( remainder ); +} + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/crc.h b/staging_vespalib/src/vespa/vespalib/util/crc.h new file mode 100644 index 00000000000..6856e07ca64 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/crc.h @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/util/bits.h> + +namespace vespalib { + +/** + * @brief Crc32 class + * + * This has fast Crc32 calculation base on table lookup. + **/ +class crc_32_type +{ +public: + crc_32_type() : _c(uint32_t(-1)) { } + void process_block(const void *start, const void *end) { process_bytes(start, (const uint8_t *)end-(const uint8_t *)start); } + void process_bytes(const void *start, size_t sz); + uint32_t checksum() const { return _c ^ uint32_t(-1); } + static uint32_t crc(const void * v, size_t sz); +private: + uint32_t _c; + class CrcTableInit + { + public: + CrcTableInit(); + static uint32_t crc(uint8_t v); + }; + static uint32_t _crc[256]; + static CrcTableInit _crcTableInit; +}; + +} + diff --git a/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp b/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp new file mode 100644 index 00000000000..1b20b044363 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp @@ -0,0 +1,73 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/util/document_runnable.h> +#include <vespa/vespalib/util/exceptions.h> + +namespace document { + +Runnable::Runnable(FastOS_ThreadPool* pool) + : _stateLock(), + _state(NOT_RUNNING) +{ + if (pool) start(*pool); +} + +bool Runnable::start(FastOS_ThreadPool& pool) +{ + vespalib::MonitorGuard monitor(_stateLock); + while (_state == STOPPING) monitor.wait(); + if (_state != NOT_RUNNING) return false; + _state = STARTING; + if (pool.NewThread(this) == NULL) { + throw vespalib::IllegalStateException("Faled starting a new thread", VESPA_STRLOC); + } + return true; +} + +bool Runnable::stop() +{ + vespalib::MonitorGuard monitor(_stateLock); + if (_state == STOPPING || _state == NOT_RUNNING) return false; + GetThread()->SetBreakFlag(); + _state = STOPPING; + return onStop(); +} + +bool Runnable::onStop() +{ + return true; +} + +bool Runnable::join() const +{ + vespalib::MonitorGuard monitor(_stateLock); + if (_state == STARTING || _state == RUNNING) return false; + while (_state != NOT_RUNNING) monitor.wait(); + return true; +} + +void Runnable::Run(FastOS_ThreadInterface*, void*) +{ + { + vespalib::MonitorGuard monitor(_stateLock); + // Dont set state if its alreadyt at stopping. (And let run() be + // called even though about to stop for consistency) + if (_state == STARTING) { + _state = RUNNING; + } + } + + // By not catching exceptions, they should abort whole application. + // We should thus not need to have a catch all to set state to not + // running. + run(); + + { + vespalib::MonitorGuard monitor(_stateLock); + _state = NOT_RUNNING; + monitor.broadcast(); + } +} + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/document_runnable.h b/staging_vespalib/src/vespa/vespalib/util/document_runnable.h new file mode 100644 index 00000000000..f9a666af507 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/document_runnable.h @@ -0,0 +1,103 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @class document::Runnable + * @ingroup util + * + * @brief Implementation of FastOS_Runnable that implements threadsafe stop. + * + * FastOS_Runnable can easily be used unsafe. If you use the thread pointer for + * anything after your runnable had returned from Run(), it could affect another + * runnable now using that thread. + * + * Using this class should be foolproof to avoid synchronization issues during + * thread starting and stopping :) + * + * @author H�kon Humberset + * @date 2005-09-19 + * @version $Id$ + */ + +#pragma once + +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/util/sync.h> + +namespace document { + +class Runnable : private FastOS_Runnable { +public: + enum State { NOT_RUNNING, STARTING, RUNNING, STOPPING }; + +private: + mutable vespalib::Monitor _stateLock; + State _state; + + void Run(FastOS_ThreadInterface*, void*); + + Runnable(const Runnable&); + Runnable& operator=(const Runnable&); + +public: + /** + * Create a runnable. + * @param pool If set, runnable will be started in constructor. + */ + Runnable(FastOS_ThreadPool* pool = 0); + + /** + * Start this runnable. + * @param pool The threadpool from which a thread is acquired. + * @return True if thread was started, false if thread was already running. + */ + bool start(FastOS_ThreadPool& pool); + + /** + * Stop this runnable. + * @return True if thread was stopped, false if thread was not running. + */ + bool stop(); + + /** + * Called in stop(). Implement, to for instance notify any monitors that + * can be waiting. + */ + virtual bool onStop(); + + /** + * Wait for this thread to finish, if it is in the process of stopping. + * @return True if thread finished (or not running), false if thread is + * running normally and no stop is scheduled. + */ + bool join() const; + + /** + * Implement this to make the runnable actually do something. + */ + virtual void run() = 0; + + /** Get the current state of this runnable. */ + State getState() const { return _state; } + + /** Check if system is in the process of stopping. */ + bool stopping() const + { + State s(getState()); + return (s == STOPPING) || + (s == RUNNING && GetThread()->GetBreakFlag()); + } + + /** + * Checks if runnable is running or not. (Started is considered running) + */ + bool running() const + { + State s(getState()); + // Must check breakflag too, as threadpool will use that to close + // down. + return (s == STARTING || + (s == RUNNING && !GetThread()->GetBreakFlag())); + } +}; + +} + diff --git a/staging_vespalib/src/vespa/vespalib/util/doom.cpp b/staging_vespalib/src/vespa/vespalib/util/doom.cpp new file mode 100644 index 00000000000..801e71b2e7e --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/doom.cpp @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "doom.h" + +namespace vespalib { + +Doom::Doom(const vespalib::Clock &clock, fastos::TimeStamp timeOfDoom) : + _clock(clock), + _timeOfDoom(timeOfDoom) +{ +} + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/util/doom.h b/staging_vespalib/src/vespa/vespalib/util/doom.h new file mode 100644 index 00000000000..8788fa0d7b7 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/doom.h @@ -0,0 +1,24 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/clock.h> + +namespace vespalib { + +class Doom +{ +private: + const vespalib::Clock &_clock; + fastos::TimeStamp _timeOfDoom; + +public: + Doom(const vespalib::Clock &clock, fastos::TimeStamp timeOfDoom); + bool doom() const { + return (_clock.getTimeNSAssumeRunning() > _timeOfDoom); + } + fastos::TimeStamp left() const { return _timeOfDoom - _clock.getTimeNS(); } +}; + +} // namespace vespalib + diff --git a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp new file mode 100644 index 00000000000..96e74ecf007 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp @@ -0,0 +1,88 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/util/growablebytebuffer.h> + +using namespace vespalib; + +GrowableByteBuffer::GrowableByteBuffer(uint32_t initialLen) : + _buffer(initialLen), + _position(0) +{ +} + +char* +GrowableByteBuffer::allocate(uint32_t len) +{ + size_t need(_position + len); + if (need > _buffer.size()) { + uint32_t newSize = vespalib::roundUp2inN(need); + DefaultAlloc newBuf(newSize); + memcpy(newBuf.get(), _buffer.get(), _position); + _buffer.swap(newBuf); + } + + char* pos = static_cast<char *>(_buffer.get()) + _position; + _position += len; + return pos; +} + +void +GrowableByteBuffer::putBytes(const char* buffer, uint32_t length) +{ + char* buf = allocate(length); + memcpy(buf, buffer, length); +} + +void +GrowableByteBuffer::putShort(uint16_t v) +{ + uint16_t val = htons(v); + putBytes(reinterpret_cast<const char*>(&val), sizeof(v)); +} + +void +GrowableByteBuffer::putInt(uint32_t v) +{ + uint32_t val = htonl(v); + putBytes(reinterpret_cast<const char*>(&val), sizeof(v)); +} + +void +GrowableByteBuffer::putReverse(const char* buffer, uint32_t length) +{ + char* buf = allocate(length); + for (uint32_t i = 0; i < length; i++) { + buf[(length - i - 1)] = buffer[i]; + } +} + +void +GrowableByteBuffer::putLong(uint64_t v) +{ + putReverse(reinterpret_cast<const char*>(&v), sizeof(v)); +} + +void +GrowableByteBuffer::putDouble(double v) +{ + putReverse(reinterpret_cast<const char*>(&v), sizeof(v)); +} + +void +GrowableByteBuffer::putString(const vespalib::stringref& v) +{ + putInt(v.size()); + putBytes(v.c_str(), v.size()); +} + +void +GrowableByteBuffer::putByte(uint8_t v) +{ + putBytes(reinterpret_cast<const char*>(&v), sizeof(v)); +} + +void +GrowableByteBuffer::putBoolean(bool v) +{ + putByte(v); +} diff --git a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h new file mode 100644 index 00000000000..61baa250e40 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h @@ -0,0 +1,96 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/fastos/types.h> +#include <vespa/vespalib/util/memory.h> +#include <vespa/vespalib/stllike/string.h> + +namespace vespalib { + +/** +Class that wraps a vector<char> and resizes it as necessary to hold data +put into it. Also has several utility functions commonly used in +data serialization. + +All of the numeric helper methods (putInt/Long/Double etc.) in this class +store the numbers in network byte order. +*/ +class GrowableByteBuffer +{ +public: + /** + Creates a new GrowableByteBuffer with the given initial length and grow factor. + */ + GrowableByteBuffer(uint32_t initialLen = 256); + + /** + If necessary, grows the buffer so it can hold the specified number of bytes, + then moves the position to after that length, and returns a pointer to the previous + position. + + @param len The length to allocate + @return Returns a pointer to a buffer that the user can write data to. + */ + char* allocate(uint32_t len); + + /** + Returns a pointer to the start of the allocated buffer. + */ + const char* getBuffer() const { return static_cast<const char *>(_buffer.get()); } + + /** + Returns the current position. + */ + uint32_t position() const { return _position; } + + /** + Adds the given buffer to this buffer. + */ + void putBytes(const char* buffer, uint32_t length); + + /** + Adds a short to the buffer. + */ + void putShort(uint16_t v); + + /** + Adds an int to the buffer. + */ + void putInt(uint32_t v); + + /** + Adds a long to the buffer. + */ + void putLong(uint64_t v); + + /** + Adds a double to the buffer. + */ + void putDouble(double v); + + /** + Adds a string to the buffer. + */ + void putString(const vespalib::stringref & v); + + /** + Adds a single byte to the buffer. + */ + void putByte(uint8_t v); + + /** + Adds a boolean to the buffer. + */ + void putBoolean(bool v); + +private: + void putReverse(const char* buffer, uint32_t length); + + DefaultAlloc _buffer; + + uint32_t _position; + double _growFactor; +}; + +} + diff --git a/staging_vespalib/src/vespa/vespalib/util/jsonstream.cpp b/staging_vespalib/src/vespa/vespalib/util/jsonstream.cpp new file mode 100644 index 00000000000..c7cf02b1313 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/jsonstream.cpp @@ -0,0 +1,364 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "jsonstream.h" + +#include <iostream> +#include <vespa/vespalib/util/exceptions.h> + +namespace vespalib { + +VESPA_IMPLEMENT_EXCEPTION_SPINE(JsonStreamException); + +const char* +JsonStream::getStateName(const State& s) { + switch (s) { + case State::OBJECT_EXPECTING_KEY: return "ObjectExpectingKey"; + case State::OBJECT_EXPECTING_VALUE: return "ObjectExpectingValue"; + case State::ARRAY: return "ArrayExpectingValue"; + case State::ROOT: return "RootExpectingArrayOrObjectStart"; + } + throw IllegalStateException("Control should not reach this point", + VESPA_STRLOC); +} + +JsonStream::JsonStream(vespalib::asciistream& as, bool createIndents) + : _writer(as) +{ + if (createIndents) _writer.setPretty(); + push({State::ROOT}); +} + +JsonStream& +JsonStream::operator<<(vespalib::stringref value) +{ + if (_state.empty()) { + fail("Stream already finalized. Can't add a string value."); + } + switch (top().state) { + case State::OBJECT_EXPECTING_KEY: { + _writer.appendKey(value); + top() = {State::OBJECT_EXPECTING_VALUE, value}; + break; + } + case State::OBJECT_EXPECTING_VALUE: { + _writer.appendString(value); + top().state = State::OBJECT_EXPECTING_KEY; + break; + } + case State::ARRAY: { + _writer.appendString(value); + ++top().array_index; + break; + } + case State::ROOT: { + _writer.appendString(value); + pop(); + break; + } + } + return *this; +} + +JsonStream& +JsonStream::operator<<(bool value) +{ + if (_state.empty()) { + fail("Stream already finalized. Can't add a bool value."); + } + switch (top().state) { + case State::OBJECT_EXPECTING_KEY: { + fail("A bool value cannot be an object key"); + break; + } + case State::OBJECT_EXPECTING_VALUE: { + _writer.appendBool(value); + top().state = State::OBJECT_EXPECTING_KEY; + break; + } + case State::ARRAY: { + _writer.appendBool(value); + ++top().array_index; + break; + } + case State::ROOT: { + _writer.appendBool(value); + pop(); + break; + } + } + return *this; +} + +JsonStream& +JsonStream::operator<<(double value) +{ + if (_state.empty()) { + fail("Stream already finalized. Can't add a double value."); + } + switch (top().state) { + case State::OBJECT_EXPECTING_KEY: { + fail("A double value cannot be an object key"); + break; + } + case State::OBJECT_EXPECTING_VALUE: { + _writer.appendDouble(value); + top().state = State::OBJECT_EXPECTING_KEY; + break; + } + case State::ARRAY: { + _writer.appendDouble(value); + ++top().array_index; + break; + } + case State::ROOT: { + _writer.appendDouble(value); + pop(); + break; + } + } + return *this; +} + +JsonStream& +JsonStream::operator<<(float value) +{ + if (_state.empty()) { + fail("Stream already finalized. Can't add a float value."); + } + switch (top().state) { + case State::OBJECT_EXPECTING_KEY: { + fail("A float value cannot be an object key"); + break; + } + case State::OBJECT_EXPECTING_VALUE: { + _writer.appendFloat(value); + top().state = State::OBJECT_EXPECTING_KEY; + break; + } + case State::ARRAY: { + _writer.appendFloat(value); + ++top().array_index; + break; + } + case State::ROOT: { + _writer.appendDouble(value); + pop(); + break; + } + } + return *this; +} + +JsonStream& +JsonStream::operator<<(int64_t value) +{ + if (_state.empty()) { + fail("Stream already finalized. Can't add an int64_t value."); + } + switch (top().state) { + case State::OBJECT_EXPECTING_KEY: { + fail("An int64_t value cannot be an object key"); + break; + } + case State::OBJECT_EXPECTING_VALUE: { + _writer.appendInt64(value); + top().state = State::OBJECT_EXPECTING_KEY; + break; + } + case State::ARRAY: { + _writer.appendInt64(value); + ++top().array_index; + break; + } + case State::ROOT: { + _writer.appendInt64(value); + pop(); + break; + } + } + return *this; +} + +JsonStream& +JsonStream::operator<<(uint64_t value) +{ + if (_state.empty()) { + fail("Stream already finalized. Can't add a uint64_t value."); + } + switch (top().state) { + case State::OBJECT_EXPECTING_KEY: { + fail("A uint64_t value cannot be an object key"); + break; + } + case State::OBJECT_EXPECTING_VALUE: { + _writer.appendUInt64(value); + top().state = State::OBJECT_EXPECTING_KEY; + break; + } + case State::ARRAY: { + _writer.appendUInt64(value); + ++top().array_index; + break; + } + case State::ROOT: { + _writer.appendUInt64(value); + pop(); + break; + } + } + return *this; +} + +JsonStream& +JsonStream::operator<<(const Object&) +{ + if (_state.empty()) { + fail("Stream already finalized. Can't start a new object."); + } + switch (top().state) { + case State::OBJECT_EXPECTING_KEY: { + fail("An object value cannot be an object key"); + break; + } + case State::OBJECT_EXPECTING_VALUE: { + _writer.beginObject(); + top().state = State::OBJECT_EXPECTING_KEY; + push({State::OBJECT_EXPECTING_KEY, ""}); + break; + } + case State::ARRAY: { + _writer.beginObject(); + push({State::OBJECT_EXPECTING_KEY, ""}); + break; + } + case State::ROOT: { + _writer.beginObject(); + top() = {State::OBJECT_EXPECTING_KEY, ""}; + break; + } + } + return *this; +} + +JsonStream& +JsonStream::operator<<(const Array&) +{ + if (_state.empty()) { + fail("Stream already finalized. Can't start a new array."); + } + switch (top().state) { + case State::OBJECT_EXPECTING_KEY: { + fail("An array value cannot be an object key"); + break; + } + case State::OBJECT_EXPECTING_VALUE: { + _writer.beginArray(); + top().state = State::OBJECT_EXPECTING_KEY; + push({State::ARRAY}); + break; + } + case State::ARRAY: { + _writer.beginArray(); + push({State::ARRAY}); + break; + } + case State::ROOT: { + _writer.beginArray(); + top() = {State::ARRAY}; + break; + } + } + return *this; +} + +JsonStream& +JsonStream::operator<<(const End&) +{ + if (_state.empty()) { + fail("Stream already finalized. Can't end it."); + } + switch (top().state) { + case State::OBJECT_EXPECTING_KEY: { + _writer.endObject(); + pop(); + break; + } + case State::OBJECT_EXPECTING_VALUE: { + fail("Object got key but not value. Cannot end it now"); + break; + } + case State::ARRAY: { + _writer.endArray(); + pop(); + break; + } + case State::ROOT: { + fail("No tag to end. At root"); + break; + } + } + if (!_state.empty() && top().state == State::ARRAY) { + ++top().array_index; + } + return *this; +} + +JsonStream& +JsonStream::finalize() +{ + while (!_state.empty()) { + operator<<(End()); + } + return *this; +} + +string +JsonStream::getStateString() const +{ + vespalib::asciistream as; + for (auto it(_state.begin()), mt(_state.end()); it != mt; it++) { + switch (it->state) { + case State::OBJECT_EXPECTING_KEY: + case State::OBJECT_EXPECTING_VALUE: { + as << "{" << it->object_key << "}"; + break; + } + case State::ARRAY: { + as << "["; + if (it->array_index != 0) { + as << (it->array_index - 1); + } + as << "]"; + break; + } + case State::ROOT: { + break; + } + } + } + if (_state.empty()) { + as << "Finalized"; + } else { + as << "(" << getStateName(_state.back().state) << ")"; + } + return as.str(); +} + +vespalib::string +JsonStream::getJsonStreamState() const +{ + asciistream report; + report << "Current: " << getStateString(); + return report.str(); +} + +void +JsonStream::fail(stringref error) const +{ + asciistream report; + report << "Invalid state on call: " << error + << " (" << getStateString() << ")"; + throw JsonStreamException(report.str(), "", VESPA_STRLOC); +} + +} // vespalib diff --git a/staging_vespalib/src/vespa/vespalib/util/jsonstream.h b/staging_vespalib/src/vespa/vespalib/util/jsonstream.h new file mode 100644 index 00000000000..adff9e505b9 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/jsonstream.h @@ -0,0 +1,113 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +/** + * An overbuild of the json writer, making the code writing the json looking + * neater. Also allows templates to use it with unknown (but supported) types, + * letting the compiler take care of calling the correct function rather than + * having to resort to template specialization. + */ + +#include <stack> +#include <vespa/vespalib/util/exception.h> +#include <vespa/vespalib/util/jsonwriter.h> + +namespace vespalib { + +class JsonStreamException : public Exception { + vespalib::string _reason; +public: + JsonStreamException(vespalib::stringref reason, + vespalib::stringref history, + vespalib::stringref location, int skipStack = 0) + : Exception(reason + (history.empty() ? "" : "\nHistory:\n" + history), + location, skipStack + 1), + _reason(reason) + { + } + stringref getReason() const { return _reason; } + VESPA_DEFINE_EXCEPTION_SPINE(JsonStreamException); + virtual ~JsonStreamException() throw () { } +}; + +// Inherit to refer to types without namespace prefix in header file. +struct JsonStreamTypes { + class Object {}; + class Array {}; + class End {}; +}; +// Use namespace in function to avoid prefixing namespace everywhere. +namespace jsonstream { + typedef JsonStreamTypes::Object Object; + typedef JsonStreamTypes::Array Array; + typedef JsonStreamTypes::End End; +} + +// We can disable this if it ends up being a performance issue. +// Really useful to explain what code bits have tried to write invalid json +// though. +#define TRACK_JSON_CREATION_TO_CREATE_EASY_TO_DEBUG_ERROR_MESSAGES 1 + +class JsonStream : public JsonStreamTypes { + JSONWriter _writer; + enum class State { + ROOT, + OBJECT_EXPECTING_KEY, + OBJECT_EXPECTING_VALUE, + ARRAY + }; + static const char* getStateName(const State&); + struct StateEntry { + State state; + string object_key; + size_t array_index; + + StateEntry() + : state(State::ROOT), object_key(""), array_index(size_t(0)) {} + StateEntry(State s) + : state(s), object_key(""), array_index(size_t(0)) {} + StateEntry(State s, stringref key) + : state(s), object_key(key), array_index(size_t(0)) {} + }; + std::vector<StateEntry> _state; + + JsonStream(const JsonStream&) = delete; + JsonStream& operator=(const JsonStream&) = delete; + + StateEntry & top() { return _state.back(); } + const StateEntry & top() const { return _state.back(); } + void pop() { _state.resize(_state.size() - 1); } + void push(const StateEntry & e) { _state.push_back(e); } +public: + JsonStream(asciistream&, bool createIndents = false); + + JsonStream& operator<<(stringref); + JsonStream& operator<<(bool); + JsonStream& operator<<(double); + JsonStream& operator<<(float); // Less precision that double + JsonStream& operator<<(int64_t); + JsonStream& operator<<(uint64_t); + JsonStream& operator<<(const Object&); + JsonStream& operator<<(const Array&); + JsonStream& operator<<(const End&); + + // Additional functions provided to let compiler work out correct + // function without requiring user to cast their value + JsonStream& operator<<(uint32_t v) + { return operator<<(static_cast<int64_t>(v)); } + JsonStream& operator<<(int32_t v) + { return operator<<(static_cast<int64_t>(v)); } + JsonStream& operator<<(const char* c) + { return operator<<(stringref(c)); } + + JsonStream& finalize(); + + vespalib::string getJsonStreamState() const; + +private: + string getStateString() const; + void fail(stringref error) const; +}; + +} // vespalib + diff --git a/staging_vespalib/src/vespa/vespalib/util/jsonwriter.cpp b/staging_vespalib/src/vespa/vespalib/util/jsonwriter.cpp new file mode 100644 index 00000000000..0fe6727a416 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/jsonwriter.cpp @@ -0,0 +1,278 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(".search.util.jsonwriter"); + +#include "jsonwriter.h" +#include <iostream> +#include <cmath> + +namespace vespalib { + +void +JSONWriter::push(State next) +{ + _stack.push_back(next); +} + +void +JSONWriter::pop(State expected) +{ + State actual = _stack.back(); + assert(actual == expected); + (void) actual; + (void) expected; + _stack.pop_back(); +} + +void +JSONWriter::considerComma() +{ + if (_comma) { + (*_os) << ','; + } +} + +void +JSONWriter::updateCommaState() +{ + if (_stack.back() == ARRAY || _stack.back() == OBJECT) { + _comma = true; + } else { + _comma = false; + } +} + +void +JSONWriter::quote(const char * str, size_t len) +{ + std::vector<char> v((len+1)*2 + 1); + v[0] = '\"'; + size_t j(1); + for (size_t i = 0; i < len; ++i) { + switch (str[i]) { + case '\b': v[j++] = '\\'; v[j++] = 'b'; break; + case '\f': v[j++] = '\\'; v[j++] = 'f'; break; + case '\n': v[j++] = '\\'; v[j++] = 'n'; break; + case '\r': v[j++] = '\\'; v[j++] = 'r'; break; + case '\t': v[j++] = '\\'; v[j++] = 't'; break; + case '\"': + case '\\': + v[j++] = '\\'; + default: + v[j++] = str[i]; + } + } + v[j++] = '\"'; + v[j++] = 0; + (*_os) << &v[0]; +} + +JSONWriter::JSONWriter() : + _os(NULL), + _stack(), + _comma(false), + _pretty(false), + _indent(0) +{ + clear(); +} + +JSONWriter::JSONWriter(vespalib::asciistream & output) : + _os(&output), + _stack(), + _comma(false), + _pretty(false), + _indent(0) +{ + clear(); + (*_os) << vespalib::asciistream::Precision(16) << vespalib::forcedot; +} + +JSONWriter& +JSONWriter::setOutputStream(vespalib::asciistream & output) { + _os = &output; + (*_os) << vespalib::asciistream::Precision(16) << vespalib::forcedot; + return *this; +} + +void +JSONWriter::indent() +{ + if (_pretty) { + (*_os) << "\n"; + for (uint32_t i = 0; i < _indent; ++i) { + (*_os) << " "; + } + } +} + +JSONWriter & +JSONWriter::clear() +{ + _stack.clear(); + _stack.push_back(INIT); + _comma = false; + return *this; +} + +JSONWriter & +JSONWriter::beginObject() +{ + push(OBJECT); + considerComma(); + indent(); + (*_os) << '{'; + _indent++; + _comma = false; + return *this; +} + +JSONWriter & +JSONWriter::endObject() +{ + pop(OBJECT); + _indent--; + indent(); + (*_os) << '}'; + updateCommaState(); + return *this; +} + +JSONWriter & +JSONWriter::beginArray() +{ + push(ARRAY); + considerComma(); + indent(); + (*_os) << '['; + _indent++; + _comma = false; + return *this; +} + +JSONWriter & +JSONWriter::endArray() +{ + pop(ARRAY); + _indent--; + indent(); + (*_os) << ']'; + updateCommaState(); + return *this; +} + +JSONWriter & +JSONWriter::appendNull() +{ + considerComma(); + (*_os) << "null"; + updateCommaState(); + return *this; +} + +JSONWriter & +JSONWriter::appendKey(const vespalib::stringref & str) +{ + considerComma(); + indent(); + quote(str.c_str(), str.size()); + (*_os) << ':'; + _comma = false; + return *this; +} + +JSONWriter & +JSONWriter::appendBool(bool v) +{ + considerComma(); + (*_os) << (v ? "true" : "false"); + updateCommaState(); + return *this; +} + +JSONWriter & +JSONWriter::appendDouble(double v) +{ + considerComma(); + if (!std::isinf(v) && !std::isnan(v)) { + // Doubles can represent all whole numbers up to 2^53, which has + // 16 digits. A precision of 16 should allow thus fit. + (*_os) << vespalib::asciistream::Precision(16) + << vespalib::automatic << v; + } else { + (*_os) << "null"; + } + updateCommaState(); + return *this; +} + +JSONWriter & +JSONWriter::appendFloat(float v) +{ + considerComma(); + if (!std::isinf(v) && !std::isnan(v)) { + // Floats can represent all whole numbers up to 2^24, which has + // 8 digits. A precision of 8 should allow thus fit. + (*_os) << vespalib::asciistream::Precision(8) + << vespalib::automatic << v; + } else { + (*_os) << "null"; + } + updateCommaState(); + return *this; +} + +JSONWriter & +JSONWriter::appendInt64(int64_t v) +{ + considerComma(); + (*_os) << v; + updateCommaState(); + return *this; +} + +JSONWriter & +JSONWriter::appendUInt64(uint64_t v) +{ + considerComma(); + (*_os) << v; + updateCommaState(); + return *this; +} + +JSONWriter & +JSONWriter::appendString(const vespalib::stringref & str) +{ + considerComma(); + quote(str.c_str(), str.size()); + updateCommaState(); + return *this; +} + +JSONWriter & +JSONWriter::appendJSON(const vespalib::stringref & json) +{ + considerComma(); + (*_os) << json; + updateCommaState(); + return *this; +} + +JSONStringer::JSONStringer() : + JSONWriter(), + _oss() +{ + setOutputStream(_oss); +} + +JSONStringer & +JSONStringer::clear() +{ + JSONWriter::clear(); + // clear the string stream as well + _oss.clear(); + return *this; +} + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/jsonwriter.h b/staging_vespalib/src/vespa/vespalib/util/jsonwriter.h new file mode 100644 index 00000000000..f63580c45d3 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/jsonwriter.h @@ -0,0 +1,101 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vector> +#include <vespa/vespalib/stllike/asciistream.h> + +namespace vespalib { + +/** + * If you want a simpler interface to write JSON with, look at the JsonStream + * class in the same directory as this class, this uses this class to make a + * more user friendly writer. + */ +class JSONWriter { +private: + enum State { + INIT = 0, + OBJECT, + ARRAY + }; + vespalib::asciistream * _os; + std::vector<State> _stack; + bool _comma; + bool _pretty; + uint32_t _indent; + + void push(State next); + void pop(State expected); + void considerComma(); + void updateCommaState(); + void quote(const char * str, size_t len); + void indent(); + +public: + JSONWriter(); + JSONWriter(vespalib::asciistream & output); + + JSONWriter & setOutputStream(vespalib::asciistream & output); + JSONWriter & clear(); + JSONWriter & beginObject(); + JSONWriter & endObject(); + JSONWriter & beginArray(); + JSONWriter & endArray(); + JSONWriter & appendNull(); + JSONWriter & appendKey(const vespalib::stringref & str); + JSONWriter & appendBool(bool v); + JSONWriter & appendDouble(double v); + JSONWriter & appendFloat(float v); + JSONWriter & appendInt64(int64_t v); + JSONWriter & appendUInt64(uint64_t v); + JSONWriter & appendString(const vespalib::stringref & str); + JSONWriter & appendJSON(const vespalib::stringref & json); + + void setPretty() { _pretty = true; }; +}; + +class JSONStringer : public JSONWriter { +private: + vespalib::asciistream _oss; + +public: + JSONStringer(); + JSONStringer & clear(); + vespalib::stringref toString() { return _oss.str(); } +}; + +template<typename T> +struct JSONPrinter +{ + static void printJSON(vespalib::JSONWriter& w, T v) { + w.appendInt64(v); + } +}; + +template<> +struct JSONPrinter<uint64_t> +{ + static void printJSON(vespalib::JSONWriter& w, uint64_t v) { + w.appendUInt64(v); + } +}; + +template<> +struct JSONPrinter<float> +{ + static void printJSON(vespalib::JSONWriter& w, float v) { + w.appendDouble(v); + } +}; + +template<> +struct JSONPrinter<double> +{ + static void printJSON(vespalib::JSONWriter& w, double v) { + w.appendDouble(v); + } +}; + + +} + diff --git a/staging_vespalib/src/vespa/vespalib/util/librarypool.cpp b/staging_vespalib/src/vespa/vespalib/util/librarypool.cpp new file mode 100644 index 00000000000..8dd6995a6c1 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/librarypool.cpp @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/vespalib/util/librarypool.h> +#include <vespa/vespalib/util/exceptions.h> + +namespace vespalib { + +LibraryPool::LibraryPool() : + _libraries(), + _lock() +{ +} + +LibraryPool::~LibraryPool() +{ + LockGuard guard(_lock); + _libraries.clear(); +} + +void +LibraryPool::loadLibrary(const vespalib::stringref & libName) +{ + LockGuard guard(_lock); + if (_libraries.find(libName) == _libraries.end()) { + DynamicLibrarySP lib(new FastOS_DynamicLibrary); + vespalib::string file(libName); + if (!lib->Open(file.c_str())) { + vespalib::string error = lib->GetLastErrorString(); + throw IllegalArgumentException(make_string("Failed loading dynamic library '%s' due to '%s'.", + file.c_str(), error.c_str())); + } else { + _libraries[libName] = lib; + } + } +} + +FastOS_DynamicLibrary * +LibraryPool::get(const vespalib::stringref & name) +{ + LockGuard guard(_lock); + LibraryMap::const_iterator found(_libraries.find(name)); + return (found != _libraries.end()) + ? found->second.get() + : NULL; +} + +const FastOS_DynamicLibrary * +LibraryPool::get(const vespalib::stringref & name) const +{ + LockGuard guard(_lock); + LibraryMap::const_iterator found(_libraries.find(name)); + return (found != _libraries.end()) + ? found->second.get() + : NULL; +} + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/librarypool.h b/staging_vespalib/src/vespa/vespalib/util/librarypool.h new file mode 100644 index 00000000000..95491865a3f --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/librarypool.h @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/util/sync.h> +#include <vespa/vespalib/stllike/string.h> +#include <map> + +namespace vespalib { + +class LibraryPool +{ +public: + LibraryPool(); + ~LibraryPool(); + /** + * This will load the library with the name given. + * - It will verify linkage at load time. + * - Symbols will be private. + * @param name The name of the library to load. That is without the 'lib' prefix and the '.so' extension. + * @throws IllegalArgumentException if there are any errors. + */ + void loadLibrary(const vespalib::stringref & name); + /** + * Will return the library requested. NULL if not found. + * @param name The name of the library as given in the @ref loadLibrary call. + * @return The library that has already been loaded. NULL if not found. + */ + FastOS_DynamicLibrary *get(const vespalib::stringref & name); + const FastOS_DynamicLibrary *get(const vespalib::stringref & name) const; +private: + typedef std::shared_ptr<FastOS_DynamicLibrary> DynamicLibrarySP; + typedef std::map<vespalib::string, DynamicLibrarySP> LibraryMap; + LibraryMap _libraries; + Lock _lock; +}; + +} + diff --git a/staging_vespalib/src/vespa/vespalib/util/md5.c b/staging_vespalib/src/vespa/vespalib/util/md5.c new file mode 100644 index 00000000000..0d287fb3c45 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/md5.c @@ -0,0 +1,334 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/* $Id$ + * + * Generic hash functions to ensure the same hash values are used everywhere. + */ + +/* + * The following copyright notice applies to the code between the + * "--- START OF MD5 CODE ---" and "--- END OF MD5 CODE ---" markers: + * + * MD5C.C - RSA Data Security, Inc., MD5 message-digest algorithm + * + * Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All + * rights reserved. + * + * License to copy and use this software is granted provided that it + * is identified as the "RSA Data Security, Inc. MD5 Message-Digest + * Algorithm" in all material mentioning or referencing this software + * or this function. + * + * License is also granted to make and use derivative works provided + * that such works are identified as "derived from the RSA Data + * Security, Inc. MD5 Message-Digest Algorithm" in all material + * mentioning or referencing the derived work. + * + * RSA Data Security, Inc. makes no representations concerning either + * the merchantability of this software or the suitability of this + * software for any particular purpose. It is provided "as is" + * without express or implied warranty of any kind. + * + * These notices must be retained in any copies of any part of this + * documentation and/or software. + * + */ + +#include <sys/types.h> +#include <string.h> +#include <stdio.h> +#include <stdlib.h> + +/* +#include <debugmalloc.h> +#include <hash.h> +#include <fileutil.h> +#include <poolmalloc.h> +#include <workarounds.h> +*/ + +#include <vespa/vespalib/util/md5.h> + +/* --- START OF MD5 CODE --- */ + +#define S11 7 +#define S12 12 +#define S13 17 +#define S14 22 +#define S21 5 +#define S22 9 +#define S23 14 +#define S24 20 +#define S31 4 +#define S32 11 +#define S33 16 +#define S34 23 +#define S41 6 +#define S42 10 +#define S43 15 +#define S44 21 + +/* MD5 context. */ +typedef struct +{ + uint state[4]; /* state (ABCD) */ + uint count[2]; /* number of bits, modulo 2^64 (lsb first) */ + unsigned char buffer[64]; /* input buffer */ +} MD5_CTX; + +static void MD5Init (MD5_CTX *); +static void MD5Update (MD5_CTX *, const unsigned char *, uint); +static void MD5Final (unsigned char [16], MD5_CTX *); +static void MD5Transform (uint [4], const unsigned char [64]); +static void Encode (unsigned char *, uint *, uint); +static void Decode (uint *, const unsigned char *, uint); + +static unsigned char PADDING[64] = { + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +/* F, G, H and I are basic MD5 functions. + */ +#define F(x, y, z) (((x) & (y)) | ((~x) & (z))) +#define G(x, y, z) (((x) & (z)) | ((y) & (~z))) +#define H(x, y, z) ((x) ^ (y) ^ (z)) +#define I(x, y, z) ((y) ^ ((x) | (~z))) + +/* ROTATE_LEFT rotates x left n bits. + */ +#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32-(n)))) + +/* FF, GG, HH, and II transformations for rounds 1, 2, 3, and 4. +Rotation is separate from addition to prevent recomputation. + */ +#define FF(a, b, c, d, x, s, ac) { \ + (a) += F ((b), (c), (d)) + (x) + (uint)(ac); \ + (a) = ROTATE_LEFT ((a), (s)); \ + (a) += (b); \ +} +#define GG(a, b, c, d, x, s, ac) { \ + (a) += G ((b), (c), (d)) + (x) + (uint)(ac); \ + (a) = ROTATE_LEFT ((a), (s)); \ + (a) += (b); \ +} +#define HH(a, b, c, d, x, s, ac) { \ + (a) += H ((b), (c), (d)) + (x) + (uint)(ac); \ + (a) = ROTATE_LEFT ((a), (s)); \ + (a) += (b); \ +} +#define II(a, b, c, d, x, s, ac) { \ + (a) += I ((b), (c), (d)) + (x) + (uint)(ac); \ + (a) = ROTATE_LEFT ((a), (s)); \ + (a) += (b); \ +} + +/* MD5 initialization. Begins an MD5 operation, writing a new context. + */ +static void +MD5Init (MD5_CTX *context) +{ + context->count[0] = context->count[1] = 0; + /* Load magic initialization constants. + */ + context->state[0] = 0x67452301ul; + context->state[1] = 0xefcdab89ul; + context->state[2] = 0x98badcfeul; + context->state[3] = 0x10325476ul; +} + +/* MD5 block update operation. Continues an MD5 message-digest + operation, processing another message block, and updating the + context. + */ +static void +MD5Update (MD5_CTX *context, const unsigned char *input, uint inputLen) +{ + uint i, idx, partLen; + + /* Compute number of bytes mod 64 */ + idx = (uint)((context->count[0] >> 3) & 0x3F); + + /* Update number of bits */ + if ((context->count[0] += ((uint)inputLen << 3)) < ((uint)inputLen << 3)) { + context->count[1]++; + } + context->count[1] += ((uint)inputLen >> 29); + + partLen = 64 - idx; + + /* Transform as many times as possible. + */ + if (inputLen >= partLen) { + memcpy((unsigned char *)&context->buffer[idx], (const unsigned char *)input, partLen); + MD5Transform(context->state, context->buffer); + + for (i = partLen; i + 63 < inputLen; i += 64) { + MD5Transform (context->state, &input[i]); + } + idx = 0; + } else { + i = 0; + } + + /* Buffer remaining input */ + memcpy((unsigned char *)&context->buffer[idx], (const unsigned char *)&input[i], inputLen-i); +} + +/* MD5 finalization. Ends an MD5 message-digest operation, writing the + the message digest and zeroizing the context. + */ +static void +MD5Final (unsigned char digest[16], MD5_CTX *context) +{ + unsigned char bits[8]; + uint idx, padLen; + + /* Save number of bits */ + Encode (bits, context->count, 8); + + /* Pad out to 56 mod 64. + */ + idx = (uint)((context->count[0] >> 3) & 0x3f); + padLen = (idx < 56) ? (56 - idx) : (120 - idx); + MD5Update (context, PADDING, padLen); + + /* Append length (before padding) */ + MD5Update (context, bits, 8); + /* Store state in digest */ + Encode (digest, context->state, 16); + + /* Zeroize sensitive information. + */ + memset ((unsigned char *)context, 0, sizeof (*context)); +} + +/* MD5 basic transformation. Transforms state based on block. + */ +static void +MD5Transform (uint state[4], const unsigned char dblock[64]) +{ + uint a = state[0], b = state[1], c = state[2], d = state[3], x[16]; + + Decode (x, dblock, 64); + + /* Round 1 */ + FF (a, b, c, d, x[ 0], S11, 0xd76aa478ul); /* 1 */ + FF (d, a, b, c, x[ 1], S12, 0xe8c7b756ul); /* 2 */ + FF (c, d, a, b, x[ 2], S13, 0x242070dbul); /* 3 */ + FF (b, c, d, a, x[ 3], S14, 0xc1bdceeeul); /* 4 */ + FF (a, b, c, d, x[ 4], S11, 0xf57c0faful); /* 5 */ + FF (d, a, b, c, x[ 5], S12, 0x4787c62aul); /* 6 */ + FF (c, d, a, b, x[ 6], S13, 0xa8304613ul); /* 7 */ + FF (b, c, d, a, x[ 7], S14, 0xfd469501ul); /* 8 */ + FF (a, b, c, d, x[ 8], S11, 0x698098d8ul); /* 9 */ + FF (d, a, b, c, x[ 9], S12, 0x8b44f7aful); /* 10 */ + FF (c, d, a, b, x[10], S13, 0xffff5bb1ul); /* 11 */ + FF (b, c, d, a, x[11], S14, 0x895cd7beul); /* 12 */ + FF (a, b, c, d, x[12], S11, 0x6b901122ul); /* 13 */ + FF (d, a, b, c, x[13], S12, 0xfd987193ul); /* 14 */ + FF (c, d, a, b, x[14], S13, 0xa679438eul); /* 15 */ + FF (b, c, d, a, x[15], S14, 0x49b40821ul); /* 16 */ + + /* Round 2 */ + GG (a, b, c, d, x[ 1], S21, 0xf61e2562ul); /* 17 */ + GG (d, a, b, c, x[ 6], S22, 0xc040b340ul); /* 18 */ + GG (c, d, a, b, x[11], S23, 0x265e5a51ul); /* 19 */ + GG (b, c, d, a, x[ 0], S24, 0xe9b6c7aaul); /* 20 */ + GG (a, b, c, d, x[ 5], S21, 0xd62f105dul); /* 21 */ + GG (d, a, b, c, x[10], S22, 0x2441453ul); /* 22 */ + GG (c, d, a, b, x[15], S23, 0xd8a1e681ul); /* 23 */ + GG (b, c, d, a, x[ 4], S24, 0xe7d3fbc8ul); /* 24 */ + GG (a, b, c, d, x[ 9], S21, 0x21e1cde6ul); /* 25 */ + GG (d, a, b, c, x[14], S22, 0xc33707d6ul); /* 26 */ + GG (c, d, a, b, x[ 3], S23, 0xf4d50d87ul); /* 27 */ + GG (b, c, d, a, x[ 8], S24, 0x455a14edul); /* 28 */ + GG (a, b, c, d, x[13], S21, 0xa9e3e905ul); /* 29 */ + GG (d, a, b, c, x[ 2], S22, 0xfcefa3f8ul); /* 30 */ + GG (c, d, a, b, x[ 7], S23, 0x676f02d9ul); /* 31 */ + GG (b, c, d, a, x[12], S24, 0x8d2a4c8aul); /* 32 */ + + /* Round 3 */ + HH (a, b, c, d, x[ 5], S31, 0xfffa3942ul); /* 33 */ + HH (d, a, b, c, x[ 8], S32, 0x8771f681ul); /* 34 */ + HH (c, d, a, b, x[11], S33, 0x6d9d6122ul); /* 35 */ + HH (b, c, d, a, x[14], S34, 0xfde5380cul); /* 36 */ + HH (a, b, c, d, x[ 1], S31, 0xa4beea44ul); /* 37 */ + HH (d, a, b, c, x[ 4], S32, 0x4bdecfa9ul); /* 38 */ + HH (c, d, a, b, x[ 7], S33, 0xf6bb4b60ul); /* 39 */ + HH (b, c, d, a, x[10], S34, 0xbebfbc70ul); /* 40 */ + HH (a, b, c, d, x[13], S31, 0x289b7ec6ul); /* 41 */ + HH (d, a, b, c, x[ 0], S32, 0xeaa127faul); /* 42 */ + HH (c, d, a, b, x[ 3], S33, 0xd4ef3085ul); /* 43 */ + HH (b, c, d, a, x[ 6], S34, 0x4881d05ul); /* 44 */ + HH (a, b, c, d, x[ 9], S31, 0xd9d4d039ul); /* 45 */ + HH (d, a, b, c, x[12], S32, 0xe6db99e5ul); /* 46 */ + HH (c, d, a, b, x[15], S33, 0x1fa27cf8ul); /* 47 */ + HH (b, c, d, a, x[ 2], S34, 0xc4ac5665ul); /* 48 */ + + /* Round 4 */ + II (a, b, c, d, x[ 0], S41, 0xf4292244ul); /* 49 */ + II (d, a, b, c, x[ 7], S42, 0x432aff97ul); /* 50 */ + II (c, d, a, b, x[14], S43, 0xab9423a7ul); /* 51 */ + II (b, c, d, a, x[ 5], S44, 0xfc93a039ul); /* 52 */ + II (a, b, c, d, x[12], S41, 0x655b59c3ul); /* 53 */ + II (d, a, b, c, x[ 3], S42, 0x8f0ccc92ul); /* 54 */ + II (c, d, a, b, x[10], S43, 0xffeff47dul); /* 55 */ + II (b, c, d, a, x[ 1], S44, 0x85845dd1ul); /* 56 */ + II (a, b, c, d, x[ 8], S41, 0x6fa87e4ful); /* 57 */ + II (d, a, b, c, x[15], S42, 0xfe2ce6e0ul); /* 58 */ + II (c, d, a, b, x[ 6], S43, 0xa3014314ul); /* 59 */ + II (b, c, d, a, x[13], S44, 0x4e0811a1ul); /* 60 */ + II (a, b, c, d, x[ 4], S41, 0xf7537e82ul); /* 61 */ + II (d, a, b, c, x[11], S42, 0xbd3af235ul); /* 62 */ + II (c, d, a, b, x[ 2], S43, 0x2ad7d2bbul); /* 63 */ + II (b, c, d, a, x[ 9], S44, 0xeb86d391ul); /* 64 */ + + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + + /* Zeroize sensitive information. + */ + memset((unsigned char *)x, 0, sizeof (x)); +} + +static void +Encode (unsigned char *output, uint *input, uint len) +{ + uint i, j; + + for (i = 0, j = 0; j < len; i++, j += 4) { + output[j] = (unsigned char)(input[i] & 0xff); + output[j+1] = (unsigned char)((input[i] >> 8) & 0xff); + output[j+2] = (unsigned char)((input[i] >> 16) & 0xff); + output[j+3] = (unsigned char)((input[i] >> 24) & 0xff); + } +} + +/* Decodes input (unsigned char) into output (uint). Assumes len is + a multiple of 4. + */ + +static void +Decode (uint *output, const unsigned char *input, uint len) +{ + uint i, j; + + for (i = 0, j = 0; j < len; i++, j += 4) + output[i] = ((uint)input[j]) | (((uint)input[j+1]) << 8) | + (((uint)input[j+2]) << 16) | (((uint)input[j+3]) << 24); +} + + +/* --- END OF MD5 CODE --- */ + +void fastc_md5sum(const void *s, size_t len, unsigned char *key) +{ + MD5_CTX m5; + MD5Init(&m5); + MD5Update(&m5, (const unsigned char *)s, len); + MD5Final(key, &m5); +} + diff --git a/staging_vespalib/src/vespa/vespalib/util/md5.h b/staging_vespalib/src/vespa/vespalib/util/md5.h new file mode 100644 index 00000000000..edb54d56d6f --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/md5.h @@ -0,0 +1,19 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + @author Thomas F. Gundersen + @version $Id$ + @date 2004-03-15 +*/ + +#pragma once + +/* Returns pointer to a key in compacted format, ie 16 unsigned chars */ + +#ifdef __cplusplus +extern "C" { +void fastc_md5sum(const void *s, size_t len, unsigned char *key); +} +#else +void fastc_md5sum(const void *s, size_t len, unsigned char *key); +#endif + diff --git a/staging_vespalib/src/vespa/vespalib/util/overview.h b/staging_vespalib/src/vespa/vespalib/util/overview.h new file mode 100644 index 00000000000..302ef16256d --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/overview.h @@ -0,0 +1,49 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/*! \mainpage Vespalib - C++ utility library for Vespa components + * + * \section intro_sec Introduction + * + * vespalib is a collection of simple utility classes shared + * between most Vespa components that are written in C++. + * Most of these aren't Vespa specific in any way, except that + * they reflect the Vespa code standard and conventions. + * + * \section install_sec Installation + * + * install vespa_vespalib_dev + * + * \section overview_sec Overview + * + * Most of the classes in vespalib can be used by themselves, + * for a full list see the alphabetical class list. + * Here are the major groups of classes: + * + * Generation counter + * + * vespalib::GenCnt + * + * Reference counting and atomic operations + * + * <BR> vespalib::ReferenceCounter + * + * Simple hashmap + * + * \ref vespalib::HashMap<T> + * + * Scope guards for easier exception-safety + * + * vespalib::CounterGuard + * <BR> vespalib::DirPointer + * <BR> vespalib::FileDescriptor + * <BR> vespalib::FilePointer + * <BR> \ref vespalib::MaxValueGuard<T> + * <BR> \ref vespalib::ValueGuard<T> + * + * General utility macros + * <BR> \ref VESPA_STRLOC + * <BR> \ref VESPA_STRINGIZE(str) + * + * Standalone testing framework + * + * vespalib::TestApp + */ diff --git a/staging_vespalib/src/vespa/vespalib/util/polymorphicarrays.h b/staging_vespalib/src/vespa/vespalib/util/polymorphicarrays.h new file mode 100644 index 00000000000..f57cd84c91a --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/polymorphicarrays.h @@ -0,0 +1,135 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// +#pragma once + +#include <vespa/vespalib/util/memory.h> + +namespace vespalib { + +/** + * Describes an interface an array of polymorphic types. + * The intention is to allow efficient implementations when that is possible + * while still enjoying the flexibility of the polymorph interface. + * It is not a full feldged Array implementation as std::vector. It contains just + * the minimum required to allow for efficient implementations for document::ArrayFieldValue. + * + * You specify the base type the interface shall provide. This base type must define + * virtual void assign(const B & rhs); + * For use with ComplexArrayT your type also need + * virtual T * clone() const; + */ +template<typename B> +class IArrayT +{ +public: + class iterator { + public: + iterator(IArrayT & a, size_t i) : _a(&a), _i(i) { } + iterator operator+(size_t diff) const { return iterator(*_a, _i + diff); } + iterator& operator++() { ++_i; return *this; } + iterator operator++(int) { iterator other(*this); ++_i; return other; } + bool operator==(const iterator & other) const { return (_a == other._a) && (_i == other._i); } + bool operator!=(const iterator & other) const { return (_i != other._i) || (_a != other._a); } + B & operator*() { return (*_a)[_i]; } + B * operator->() { return &(*_a)[_i]; } + friend ssize_t operator - (const iterator & a, const iterator & b) { return a._i - b._i; } + private: + IArrayT * _a; + size_t _i; + }; + class const_iterator { + public: + const_iterator(const IArrayT & a, size_t i) : _a(&a), _i(i) { } + const_iterator operator+(size_t diff) const { return const_iterator(*_a, _i + diff); } + const_iterator& operator++() { ++_i; return *this; } + const_iterator operator++(int) { const_iterator other(*this); ++_i; return other; } + bool operator==(const const_iterator & other) const { return (_a == other._a) && (_i == other._i); } + bool operator!=(const const_iterator & other) const { return (_i != other._i) || (_a != other._a); } + const B & operator*() const { return (*_a)[_i]; } + const B * operator->() const { return &(*_a)[_i]; } + size_t operator - (const const_iterator & b) const { return _i - b._i; } + private: + const IArrayT * _a; + size_t _i; + }; + typedef std::unique_ptr<IArrayT> UP; + + virtual ~IArrayT() { } + virtual const B & operator [] (size_t i) const = 0; + virtual B & operator [] (size_t i) = 0; + virtual void resize(size_t sz) = 0; + virtual void reserve(size_t sz) = 0; + virtual void clear() = 0; + virtual IArrayT * clone() const = 0; + virtual size_t size() const = 0; + virtual iterator erase(iterator it) = 0; + virtual const_iterator begin() const { return const_iterator(*this, 0); } + virtual const_iterator end() const { return const_iterator(*this, size()); } + virtual iterator begin() { return iterator(*this, 0); } + virtual iterator end() { return iterator(*this, size()); } + bool empty() const { return size() == 0; } + virtual void push_back(const B & v) = 0; +}; + +template <typename T, typename B> +class PrimitiveArrayT : public IArrayT<B> +{ + using typename IArrayT<B>::iterator; +public: + PrimitiveArrayT() : _array() { } + virtual ~PrimitiveArrayT() { } + virtual const T & operator [] (size_t i) const { return _array[i]; } + virtual T & operator [] (size_t i) { return _array[i]; } + virtual void resize(size_t sz) { _array.resize(sz); } + virtual void reserve(size_t sz) { _array.reserve(sz); } + virtual void clear() { _array.clear(); } + virtual IArrayT<B> * clone() const { return new PrimitiveArrayT<T, B>(*this); } + virtual size_t size() const { return _array.size(); } + virtual iterator erase(iterator it) { _array.erase(_array.begin() + (it - this->begin())); return it; } + virtual void push_back(const B & v) { + size_t sz(_array.size()); + _array.resize(sz + 1); + _array[sz].assign(v); + } +private: + std::vector<T> _array; +}; + +template <typename B> +class ComplexArrayT : public IArrayT<B> +{ + using typename IArrayT<B>::iterator; +public: + class Factory { + public: + typedef std::unique_ptr<Factory> UP; + typedef vespalib::CloneablePtr<Factory> CP; + virtual B * create() = 0; + virtual Factory * clone() const = 0; + virtual ~Factory() { } + }; + explicit ComplexArrayT(typename Factory::UP factory) : _array(), _factory(factory.release()) { } + virtual ~ComplexArrayT() { } + virtual const B & operator [] (size_t i) const { return *_array[i]; } + virtual B & operator [] (size_t i) { return *_array[i]; } + virtual void resize(size_t sz) { + _array.resize(sz); + for (auto & cp : _array) { + if ( cp.get() == nullptr) { + cp.reset(_factory->create()); + } + } + } + virtual void reserve(size_t sz) { _array.reserve(sz); } + virtual void clear() { _array.clear(); } + virtual IArrayT<B> * clone() const { return new ComplexArrayT<B>(*this); } + virtual size_t size() const { return _array.size(); } + virtual iterator erase(iterator it) { _array.erase(_array.begin() + (it - this->begin())); return it; } + virtual void push_back(const B & v) { _array.push_back(v.clone()); } +private: + typedef vespalib::CloneablePtr<B> CP; + std::vector<CP> _array; + typename Factory::CP _factory; +}; + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp b/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp new file mode 100644 index 00000000000..18385f13914 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp @@ -0,0 +1,149 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include "process_memory_stats.h" +#include <fstream> +#include <sstream> + +namespace vespalib +{ + +namespace { + +/* + * Check if line specifies an address range. + * + * address perms offset dev inode pathname + * + * 00400000-00420000 r-xp 00000000 fd:04 16545041 /usr/bin/less + */ + +bool isRange(const std::string &line) { + for (char c : line) { + if (c == ' ') { + return true; + } + if (c == ':') { + return false; + } + } + return false; +} + + +/* + * Check if address range is anonymous, e.g. not mapped from file. + * inode number is 0 in that case. + * + * address perms offset dev inode pathname + * + * 00400000-00420000 r-xp 00000000 fd:04 16545041 /usr/bin/less + * 00625000-00628000 rw-p 00000000 00:00 0 + * + * The range starting at 00400000 is not anonymous. + * The range starting at 00625000 is anonymous. + */ + +bool isAnonymous(const std::string &line) { + int delims = 0; + for (char c : line) { + if (delims >= 4) { + return (c == '0'); + } + if (c == ' ') { + ++delims; + } + } + return true; +} + + +/* + * Lines not containing an address range contains a header and a + * value, e.g. + * + * Size: 128 kB + * Rss: 96 kB + * Anonymous: 0 kB + * + * The lines with header Anonymous are ignored, thus anonymous pages + * caused by mmap() of a file with MAP_PRIVATE flags are counted as + * mapped pages. + */ + +std::string getLineHeader(const std::string &line) +{ + size_t len = 0; + for (char c : line) { + if (c == ':') { + return line.substr(0, len); + } + ++len; + } + abort(); +} + +} + + +ProcessMemoryStats::ProcessMemoryStats() + : _mapped_virt(0), + _mapped_rss(0), + _anonymous_virt(0), + _anonymous_rss(0) +{ +} + + +ProcessMemoryStats::ProcessMemoryStats(uint64_t mapped_virt, + uint64_t mapped_rss, + uint64_t anonymous_virt, + uint64_t anonymous_rss) + : _mapped_virt(mapped_virt), + _mapped_rss(mapped_rss), + _anonymous_virt(anonymous_virt), + _anonymous_rss(anonymous_rss) +{ +} + + +ProcessMemoryStats +ProcessMemoryStats::create() +{ + std::ifstream smaps("/proc/self/smaps"); + std::string line; + std::string lineHeader; + bool anonymous = true; + uint64_t mapped_virt = 0; + uint64_t mapped_rss = 0; + uint64_t anonymous_virt = 0; + uint64_t anonymous_rss = 0; + uint64_t lineVal = 0; + while (smaps.good()) { + std::getline(smaps, line); + if (isRange(line)) { + anonymous = isAnonymous(line); + } else if (!line.empty()) { + lineHeader = getLineHeader(line); + std::istringstream is(line.substr(lineHeader.size() + 1)); + is >> lineVal; + if (lineHeader == "Size") { + if (anonymous) { + anonymous_virt += lineVal * 1024; + } else { + mapped_virt += lineVal * 1024; + } + } else if (lineHeader == "Rss") { + if (anonymous) { + anonymous_rss += lineVal * 1024; + } else { + mapped_rss += lineVal * 1024; + } + } + } + } + return ProcessMemoryStats(mapped_virt, mapped_rss, + anonymous_virt, anonymous_rss); +} + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.h b/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.h new file mode 100644 index 00000000000..2a35f7d87a7 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.h @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace vespalib +{ + +/* + * Class for linux specific way to get memory stats for current process. + */ +class ProcessMemoryStats +{ + uint64_t _mapped_virt; // virtual size + uint64_t _mapped_rss; // resident size + uint64_t _anonymous_virt; // virtual size + uint64_t _anonymous_rss; // resident size + +public: + ProcessMemoryStats(); + ProcessMemoryStats(uint64_t mapped_virt, uint64_t mapped_rss, + uint64_t anonymous_virt, uint64_t anonymous_rss); + static ProcessMemoryStats create(); // based on /proc/self/smaps + uint64_t getMappedVirt() const { return _mapped_virt; } + uint64_t getMappedRss() const { return _mapped_rss; } + uint64_t getAnonymousVirt() const { return _anonymous_virt; } + uint64_t getAnonymousRss() const { return _anonymous_rss; } +}; + +} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/util/programoptions.cpp b/staging_vespalib/src/vespa/vespalib/util/programoptions.cpp new file mode 100644 index 00000000000..a4b120ab492 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/programoptions.cpp @@ -0,0 +1,683 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/util/programoptions.h> + +#include <iostream> +#include <vespa/log/log.h> +#include <vespa/vespalib/util/exceptions.h> + +LOG_SETUP(".programoptions"); + +namespace vespalib { + +VESPA_IMPLEMENT_EXCEPTION(InvalidCommandLineArgumentsException, Exception); + +namespace { + + std::string UNSET_TOKEN = "-_-/#UNSET#\\-_-"; + + // Use tokenizer instead when moved from document + std::vector<std::string> splitString(const std::string& source, char split) + { + std::vector<std::string> target; + std::string::size_type start = 0; + std::string::size_type stop = source.find(split); + while (stop != std::string::npos) { + target.push_back(source.substr(start, stop - start)); + start = stop + 1; + stop = source.find(split, start); + } + target.push_back(source.substr(start)); + return target; + } + +} + +template<typename Number> +std::string ProgramOptions::NumberOptionParser<Number>::getStringValue(Number n) +{ + std::ostringstream ost; + ost << n; + return ost.str(); +} + +template<typename Number> +void ProgramOptions::NumberOptionParser<Number>::set(const std::vector<std::string>& arguments) +{ + try{ + _number = boost::lexical_cast<Number>(arguments[0]); + } catch (const boost::bad_lexical_cast& e) { + std::ostringstream ost; + ost << "The argument '" << arguments[0] + << "' can not be interpreted as a number of type " + << getTypeName<Number>() << "."; + throw InvalidCommandLineArgumentsException( + ost.str(), VESPA_STRLOC); + } +} + +ProgramOptions::OptionParser::OptionParser( + const std::string& nameList, uint32_t argCount, const std::string& desc) + : _names(splitString(nameList, ' ')), + _hiddenNames(), + _argCount(argCount), + _argTypes(argCount), + _hasDefault(false), + _invalidDefault(false), + _defaultString(), + _description(desc) +{ + if (nameList == "") _names.clear(); +} + +ProgramOptions::OptionParser::OptionParser( + const std::string& nameList, uint32_t argCount, + const std::string& defString, const std::string& desc) + : _names(splitString(nameList, ' ')), + _hiddenNames(), + _argCount(argCount), + _argTypes(argCount), + _hasDefault(true), + _invalidDefault(false), + _defaultString(defString), + _description(desc) +{ +} + +void +ProgramOptions::OptionParser::setInvalidDefault() +{ + _invalidDefault = true; +} + +std::string ProgramOptions::OptionParser::getOptSyntaxString() const +{ + std::ostringstream ost; + for (uint32_t i=0; i<_names.size(); ++i) { + ost << (_names[i].size() == 1 ? " -" : " --"); + ost << _names[i]; + } + for (uint32_t i=0; i<_argCount; ++i) { + std::string type = (_argTypes[i] != "" ? _argTypes[i] : getArgType(i)); + ost << " <" << type << ">"; + } + return ost.str(); +} + +ProgramOptions::ProgramOptions() + : _argc(0), + _argv(0), + _options(), + _optionMap(), + _setOptions(), + _syntaxMessage(), + _maxLeftColumnSize(30), + _defaultsSet(false) +{ +} + +ProgramOptions::ProgramOptions(int argc, const char* const* argv) + : _argc(argc), + _argv(argv), + _options(), + _optionMap(), + _setOptions(), + _syntaxMessage(), + _maxLeftColumnSize(30), + _defaultsSet(false) +{ +} + +void +ProgramOptions::setCommandLineArguments(int argc, const char* const* argv) +{ + _argc = argc; + _argv = argv; +} + +void +ProgramOptions::setSyntaxMessage(const std::string& msg) +{ + _syntaxMessage = msg; +} + +void +ProgramOptions::addHiddenIdentifiers(const std::string& optionNameList) +{ + if (_options.size() == 0) { + throw InvalidCommandLineArgumentsException( + "Cannot add hidden identifier to last " + "option as no option has been added yet.", VESPA_STRLOC); + } + OptionParser::SP opt = _options.back(); + if (opt->isHeader()) { + throw InvalidCommandLineArgumentsException( + "Cannot add option arguments to option header.", VESPA_STRLOC); + } + std::vector<std::string> newIds(splitString(optionNameList, ' ')); + for (uint32_t i=0; i<newIds.size(); ++i) { + std::map<std::string, OptionParser::SP>::const_iterator it( + _optionMap.find(newIds[i])); + if (it != _optionMap.end()) { + throw InvalidCommandLineArgumentsException( + "Option '" + newIds[i] + "' is already registered.", + VESPA_STRLOC); + } + } + for (uint32_t i=0; i<newIds.size(); ++i) { + _optionMap[newIds[i]] = opt; + opt->_hiddenNames.push_back(newIds[i]); + } +} + +void +ProgramOptions::setArgumentTypeName(const std::string& name, uint32_t index) +{ + if (_options.size() == 0) { + throw InvalidCommandLineArgumentsException( + "Cannot add hidden identifier to last " + "option as no option has been added yet.", VESPA_STRLOC); + } + OptionParser::SP opt = _options.back(); + if (opt->isHeader()) { + throw InvalidCommandLineArgumentsException( + "Cannot add option arguments to option header.", VESPA_STRLOC); + } + opt->_argTypes[index] = name; +} + +void +ProgramOptions::addOptionHeader(const std::string& description) +{ + _options.push_back(OptionParser::SP(new OptionHeader(description))); +} + +namespace { + bool isNumber(const std::string& arg) { + if (arg.size() > 1 && arg[0] == '-' + && arg[1] >= '0' && arg[1] <= '9') + { + return true; + } + return false; + } +} + +void +ProgramOptions::parse() +{ + try{ + std::ostringstream ost; + ost << "Parsing options:\n"; + for (int i=0; i<_argc; ++i) { + ost << " " << i << ": '" << _argv[i] << "'\n"; + } + LOG(debug, "%s", ost.str().c_str()); + uint32_t argPos = 0; + uint32_t optPos = 1; + for (; optPos < static_cast<uint32_t>(_argc); ++optPos) { + std::string s(_argv[optPos]); + // Skip arguments + if (s.size() < 2 || s[0] != '-' || s == "--" || isNumber(s)) { + if (argPos <= optPos) { // No more options to parse + break; + } else { // This has already been consumed as argument + continue; + } + } + if (argPos <= optPos) { argPos = optPos + 1; } + if (s.substr(0, 2) == "--") { + std::string id(s.substr(2)); + LOG(debug, "Parsing long option %s at pos %u, arg pos is now " + "%u", id.c_str(), optPos, argPos); + std::map<std::string, OptionParser::SP>::const_iterator it( + _optionMap.find(id)); + if (it == _optionMap.end()) { + throw InvalidCommandLineArgumentsException( + "Invalid option '" + id + "'.", VESPA_STRLOC); + } + parseOption(id, *it->second, argPos); + _setOptions.insert(it->second); + } else { + LOG(debug, "Parsing short options %s at pos %u, arg pos is now " + "%u.", s.c_str() + 1, optPos, argPos); + for (uint32_t shortPos = 1; shortPos < s.size(); ++shortPos) { + std::string id(s.substr(shortPos, 1)); + LOG(debug, "Parsing short option %s, arg pos is %u.", + id.c_str(), argPos); + std::map<std::string, OptionParser::SP>::const_iterator it( + _optionMap.find(id)); + if (it == _optionMap.end()) { + throw InvalidCommandLineArgumentsException( + "Invalid option '" + id + "'.", VESPA_STRLOC); + } + parseOption(id, *it->second, argPos); + _setOptions.insert(it->second); + } + } + } + if (!_defaultsSet) setDefaults(true); + for (uint32_t i=0; i<_arguments.size(); ++i) { + OptionParser& opt(*_arguments[i]); + if (opt._argCount == 0) { + LOG(debug, "Parsing list argument %s. Pos is %u.", + opt.getArgName().c_str(), optPos); + std::vector<std::string> arguments; + for (uint32_t j=optPos; j<static_cast<uint32_t>(_argc); ++j) { + arguments.push_back(_argv[j]); + } + opt.set(arguments); + optPos = _argc; + LOG(debug, "Done. Pos is now %u.", optPos); + } else if (optPos + opt._argCount > static_cast<uint32_t>(_argc)) { + if (!opt.isRequired()) { + LOG(debug, "Setting default for argument %u.", i); + opt.setDefault(); + } else { + throw InvalidCommandLineArgumentsException( + "Insufficient data is given to set required argument '" + + opt.getArgName() + "'.", VESPA_STRLOC); + } + } else { + parseArgument(opt, optPos); + } + } + } catch (const vespalib::Exception& e) { + throw; + } + for (uint32_t i=0; i<_configurables.size(); ++i) { + _configurables[i]->finalizeOptions(); + } +} + +void +ProgramOptions::setDefaults(bool failUnsetRequired) +{ + for (uint32_t i=0; i<_options.size(); ++i) { + OptionParser::SP opt(_options[i]); + if (opt->isHeader()) { + continue; + } + std::set<OptionParser::SP>::const_iterator it(_setOptions.find(opt)); + if (it == _setOptions.end()) { + if (opt->_hasDefault) { + opt->setDefault(); + } else if (failUnsetRequired) { + throw InvalidCommandLineArgumentsException( + "Option '" + opt->_names[0] + + "' has no default and must be set.", VESPA_STRLOC); + } + } + } + _defaultsSet = true; +} + +void +ProgramOptions::parseOption( + const std::string& id, OptionParser& opt, uint32_t& argPos) +{ + LOG(debug, "Parsing option %s. Argpos is %u.", id.c_str(), argPos); + std::vector<std::string> arguments; + for (; arguments.size() != opt._argCount; ++argPos) { + if (argPos >= static_cast<uint32_t>(_argc)) { + throw InvalidCommandLineArgumentsException( + vespalib::make_string( + "Option '%s' needs %u arguments. Only %u available.", + id.c_str(), opt._argCount, (uint32_t) arguments.size()), + VESPA_STRLOC); + } + if (strlen(_argv[argPos]) >= 2 && _argv[argPos][0] == '-' + && !isNumber(_argv[argPos])) + { + continue; + } + arguments.push_back(_argv[argPos]); + } + opt.set(arguments); + LOG(debug, "Done. Argpos is now %u.", argPos); +} + +void +ProgramOptions::parseArgument(OptionParser& opt, uint32_t& pos) +{ + LOG(debug, "Parsing argument %s. Pos is %u.", + opt.getArgName().c_str(), pos); + std::vector<std::string> arguments; + for (; arguments.size() != opt._argCount; ++pos) { + assert(pos < static_cast<uint32_t>(_argc)); + arguments.push_back(_argv[pos]); + } + opt.set(arguments); + LOG(debug, "Done. Pos is now %u.", pos); +} + +namespace { + std::vector<std::string> breakText(const std::vector<std::string>& source, + uint32_t maxLen, + int preserveWordSpaceLimit = -1) + { + if (preserveWordSpaceLimit < 0) { + preserveWordSpaceLimit = maxLen / 5; + } + std::vector<std::string> result; + for (uint32_t i=0; i<source.size(); ++i) { + std::vector<std::string> split(splitString(source[i], '\n')); + for (uint32_t j=0; j<split.size(); ++j) { + // Process each line of input here to possible break + // it. + std::string line = split[j]; + while (true) { + // If the line is already short enough, we're done. + if (line.size() <= maxLen) { + result.push_back(line); + break; + } + // Otherwise, find the last space before max len + std::string::size_type pos( + line.rfind(" ", maxLen)); + if (pos != std::string::npos + && pos > maxLen - preserveWordSpaceLimit) + { + // If the space comes late enough, add the line up + // to that point, and let the remainder go to a new + // line. + result.push_back(line.substr(0, pos)); + line = line.substr(pos+1); + } else { + // If the space is not late enough, force break + // inside a word and add a dash. + result.push_back(line.substr(0, maxLen - 1) + "-"); + line = line.substr(maxLen - 1); + } + } + } + } + return result; + } + + std::vector<std::string> breakText(const std::string& source, + uint32_t maxLen, + int preserveWordSpaceLimit = -1) + { + std::vector<std::string> v(1); + v[0] = source; + return breakText(v, maxLen, preserveWordSpaceLimit); + } +} + +void +ProgramOptions::writeSyntaxPage(std::ostream& out, bool showDefaults) +{ + bool hasOptions = false; + for(uint32_t i=0; i<_options.size(); ++i) { + if (!_options[i]->isHeader() && !_options[i]->hideFromSyntaxPage()) { + hasOptions = true; + } + } + if (!_syntaxMessage.empty()) { + out << "\n"; + std::vector<std::string> text(breakText(_syntaxMessage, 80)); + for (uint32_t i=0; i<text.size(); ++i) { + out << text[i] << "\n"; + } + } + if (_argc > 0) { + std::string progName = _argv[0]; + out << "\nUsage: "; + std::string::size_type pos = progName.rfind("/"); + if (pos != std::string::npos) { + out << progName.substr(pos+1); + } else { + out << progName; + } + if (hasOptions) { + out << " [options]"; + } + for (uint32_t i=0; i<_arguments.size(); ++i) { + OptionParser& opt(*_arguments[i]); + out << (opt.isRequired() ? " <" : " [") + << opt.getArgName(); + if (opt._argCount == 0) out << "..."; + out << (opt.isRequired() ? ">" : "]"); + } + out << "\n"; + } + if (_arguments.size() > 0) { + out << "\nArguments:\n"; + // To reuse option parser objects, argument names get split on + // whitespace. Concatenating to show nice text + std::vector<std::string> argNames(_arguments.size()); + uint32_t argSize = 10; + for(uint32_t i=0; i<_arguments.size(); ++i) { + OptionParser& opt(*_arguments[i]); + argNames[i] = opt.getArgName() + + " (" + opt.getArgType(0) + ")"; + if (argNames[i].size() <= _maxLeftColumnSize) { + argSize = std::max(argSize, + static_cast<uint32_t>(argNames[i].size())); + } + } + // Calculate indent used for extra lines + std::string indent = " "; // 1 space indent + " : " in between. + for (uint32_t i=0; i<argSize; ++i) indent += ' '; + + for(uint32_t i=0; i<_arguments.size(); ++i) { + OptionParser& opt(*_arguments[i]); + out << " " << argNames[i]; + if (argNames[i].size() > _maxLeftColumnSize) { + out << "\n "; + for (uint32_t j=0; j<argSize; ++j) { + out << " "; + } + } else { + for (uint32_t j=argNames[i].size(); j<argSize; ++j) { + out << " "; + } + } + std::vector<std::string> message( + breakText(opt._description, 80 - indent.size())); + if (opt._hasDefault) { + if (message.back().size() + indent.size() + 11 <= 80) { + message.back() += " (optional)"; + } else { + message.push_back("(optional)"); + } + } + for (uint32_t j=0; j<message.size(); ++j) { + out << (j == 0 ? " : " : indent) << message[j] << "\n"; + } + } + } + if (hasOptions) { + if (!_options[0]->isHeader()) { + out << "\nOptions:\n"; + } + uint32_t argSize = 10; + for(uint32_t i=0; i<_options.size(); ++i) { + OptionParser& opt(*_options[i]); + if (opt.isHeader() || opt.hideFromSyntaxPage()) continue; + argSize = std::max(argSize, static_cast<uint32_t>( + opt.getOptSyntaxString().size())); + } + // If too much space is used in first colo, calculate inset again, + // such that we don't need to push to max just because one is + // too long + if (argSize > _maxLeftColumnSize) { + argSize = 10; + for(uint32_t i=0; i<_options.size(); ++i) { + OptionParser& opt(*_options[i]); + if (opt.isHeader() || opt.hideFromSyntaxPage()) continue; + uint32_t size = static_cast<uint32_t>( + opt.getOptSyntaxString().size()); + if (size <= _maxLeftColumnSize) { + argSize = std::max(argSize, size); + } + } + } + std::string indent = " "; + for (uint32_t i=0; i<argSize; ++i) indent += ' '; + for(uint32_t i=0; i<_options.size(); ++i) { + OptionParser& opt(*_options[i]); + if (opt.isHeader()) { + out << "\n" << opt._description << ":\n"; + continue; + } + if (opt.hideFromSyntaxPage()) continue; + std::string optStr = opt.getOptSyntaxString(); + out << optStr; + for (uint32_t j=optStr.size(); j<argSize; ++j) { + out << " "; + } + std::vector<std::string> message( + breakText(opt._description, 80 - indent.size())); + if (showDefaults) { + std::string s; + if (!opt._hasDefault) { + s = "(required)"; + } else if (!opt._invalidDefault + && opt._defaultString != UNSET_TOKEN) + { + s = "(default " + opt._defaultString + ")"; + } + if (s.size() > 0) { + if (message.back().size() + indent.size() + + 1 + s.size() <= 80) + { + message.back() += " " + s; + } else { + message.push_back(s); + } + } + } + for (uint32_t j=0; j<message.size(); ++j) { + out << (j == 0 ? " : " : indent) << message[j] << "\n"; + } + } + } +} + +ProgramOptions::OptionParser& +ProgramOptions::addOption(OptionParser::SP opt) +{ + for (uint32_t i=0; i<opt->_names.size(); ++i) { + std::map<std::string, OptionParser::SP>::const_iterator it( + _optionMap.find(opt->_names[i])); + if (it != _optionMap.end()) { + throw InvalidCommandLineArgumentsException( + "Option '" + opt->_names[i] + "' is already registered.", + VESPA_STRLOC); + } + } + _options.push_back(opt); + for (uint32_t i=0; i<opt->_names.size(); ++i) { + _optionMap[opt->_names[i]] = opt; + } + return *opt; +} + +ProgramOptions::OptionParser& +ProgramOptions::addArgument(std::shared_ptr<OptionParser> arg) +{ + if (!_arguments.empty() && !_arguments.back()->isRequired()) { + if (arg->isRequired()) { + throw InvalidCommandLineArgumentsException( + "Argument '" + arg->_names[0] + "' is required and cannot " + "follow an optional argument.", VESPA_STRLOC); + } + } + if (!_arguments.empty() && _arguments.back()->_argCount == 0) { + throw InvalidCommandLineArgumentsException( + "Argument '" + arg->_names[0] + "' cannot follow a list " + "argument that will consume all remaining arguments.", + VESPA_STRLOC); + + } + _arguments.push_back(arg); + return *arg; +} + +ProgramOptions::OptionParser& +ProgramOptions::getOptionParser(const std::string& id) +{ + std::map<std::string, OptionParser::SP>::const_iterator it( + _optionMap.find(id)); + if (it == _optionMap.end()) { + throw InvalidCommandLineArgumentsException( + "No option registered with id '" + id + "'.", VESPA_STRLOC); + } + return *it->second; +} + +ProgramOptions::OptionParser& +ProgramOptions::getArgumentParser(uint32_t argIndex) +{ + if (argIndex >= _arguments.size()) { + std::ostringstream ost; + ost << "Only " << _arguments.size() + << " arguments registered. Thus argument " << argIndex + << " does not exist."; + throw InvalidCommandLineArgumentsException(ost.str(), VESPA_STRLOC); + } + return *_arguments[argIndex]; +} + +ProgramOptions::BoolOptionParser::BoolOptionParser( + const std::string& nameList, bool& value, const std::string& desc) + : OptionParser(nameList, 0, UNSET_TOKEN, desc), + _value(value), + _defaultValue(false) +{ +} + +ProgramOptions::FlagOptionParser::FlagOptionParser( + const std::string& nameList, bool& value, const std::string& desc) + : OptionParser(nameList, 0, UNSET_TOKEN, desc), + _value(value), + _unsetValue(false) +{ + _invalidDefault = true; +} + +ProgramOptions::FlagOptionParser::FlagOptionParser( + const std::string& nameList, bool& value, const bool& unsetValue, + const std::string& desc) + : OptionParser(nameList, 0, unsetValue ? "true" : "false", desc), + _value(value), + _unsetValue(unsetValue) +{ + _invalidDefault = true; +} + +ProgramOptions::StringOptionParser::StringOptionParser( + const std::string& nameList, std::string& val, const std::string& desc) + : OptionParser(nameList, 1, desc), + _value(val), + _defaultValue() +{ +} + +ProgramOptions::StringOptionParser::StringOptionParser( + const std::string& nameList, std::string& value, + const std::string& defValue, const std::string& desc) + : OptionParser(nameList, 1, '"' + defValue + '"', desc), + _value(value), + _defaultValue(defValue) +{ +} + +ProgramOptions::MapOptionParser::MapOptionParser( + const std::string& nameList, std::map<std::string, std::string>& value, + const std::string& description) + : OptionParser(nameList, 2, "empty", description), + _value(value) +{ +} + +template class ProgramOptions::NumberOptionParser<int32_t>; +template class ProgramOptions::NumberOptionParser<uint32_t>; +template class ProgramOptions::NumberOptionParser<int64_t>; +template class ProgramOptions::NumberOptionParser<uint64_t>; +template class ProgramOptions::NumberOptionParser<float>; +template class ProgramOptions::NumberOptionParser<double>; + +} // vespalib diff --git a/staging_vespalib/src/vespa/vespalib/util/programoptions.h b/staging_vespalib/src/vespa/vespalib/util/programoptions.h new file mode 100644 index 00000000000..cbba95ae150 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/programoptions.h @@ -0,0 +1,494 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * \class vespalib::ProgramOptions + * \ingroup util + * + * \brief Utility class for easy parsing of program options. + * + * This class makes it easy to parse program options, and to write a decent + * syntax page. + * + * Just call addOption to register options, and call parseOptions to do the + * parsing. There's also a function for writing the syntax page, such that this + * is automatically updated. + * + * Stuff to come later: + * + * Support for arguments (So you can do stuff like ./myprog file.txt, and not + * only stuff like ./myprog -f file.txt) + * Setting min and max values for numbers. + * Support for multiargument options. + * Automatic man page writing. + */ + +#pragma once + +#include <boost/lexical_cast.hpp> +#include <map> +#include <set> +#include <string> +#include <vector> +#include <stdint.h> +#include <vespa/vespalib/util/exception.h> + +namespace vespalib { + +VESPA_DEFINE_EXCEPTION(InvalidCommandLineArgumentsException, Exception); + + + +struct ProgramOptions { + /** Utility class used by command line configurable utility. */ + class LifetimeToken { + ProgramOptions& o; + public: + typedef std::unique_ptr<LifetimeToken> UP; + LifetimeToken(ProgramOptions& op) : o(op) {} + ~LifetimeToken() { o.clear(); } + }; + /** + * Utility class used to deletage stuff to be configured into multiple + * units. + */ + struct Configurable { + virtual ~Configurable() {} + /** + * Called on configurables to have it register its options. + * Unit must hang onto lifetimetoken until command line parsing have + * completed. Lifetimetoken should be deleted before stuff registered + * to be configured is deleted. + */ + virtual void registerCommandLineOptions(ProgramOptions&, LifetimeToken::UP) = 0; + + /** + * Called after command line parsing is complete, in order for + * components to ensure validity of options and throw exceptionse on + * failures. + */ + virtual void finalizeOptions() = 0; + }; + + class OptionParser; + + int _argc; + const char* const* _argv; + std::vector<std::shared_ptr<OptionParser> > _options; + std::map<std::string, std::shared_ptr<OptionParser> > _optionMap; + std::set<std::shared_ptr<OptionParser> > _setOptions; + std::vector<std::shared_ptr<OptionParser> > _arguments; + std::string _syntaxMessage; + uint32_t _maxLeftColumnSize; + bool _defaultsSet; + std::vector<Configurable*> _configurables; + + ProgramOptions(const ProgramOptions&); + ProgramOptions& operator=(const ProgramOptions&); + +public: + /** + * If using empty constructor, setCommandLineArguments() must be called + * before parse() and writeSyntaxPage(). + */ + ProgramOptions(); + ProgramOptions(int argc, const char* const* argv); + + void addConfigurable(Configurable& c) { + _configurables.push_back(&c); + c.registerCommandLineOptions( + *this, LifetimeToken::UP(new LifetimeToken(*this))); + } + + void setCommandLineArguments(int argc, const char* const* argv); + + /** + * In bool case, add an optional option that will be true if used. + * In all other cases, adds a required option as there are no default. + * Parsing will fail if required option is not set. + */ + template<typename Type> + OptionParser& addOption(const std::string& optionNameList, + Type& value, + const std::string& description); + + /** Add an optional option. Default value will be used if not set. */ + template<typename Type> + OptionParser& addOption(const std::string& optionNameList, + Type& value, + const Type& defaultValue, + const std::string& description); + + template<typename Type> + OptionParser& addArgument(const std::string& optionNameList, + Type& value, + const std::string& description); + + template<typename Type> + OptionParser& addArgument(const std::string& optionNameList, + Type& value, + const Type& defaultValue, + const std::string& description); + + template<typename Type> + OptionParser& addListArgument(const std::string& optionNameList, + std::vector<Type>& value, + const std::string& description); + + OptionParser& getOptionParser(const std::string& id); + OptionParser& getArgumentParser(uint32_t argIndex); + + void addHiddenIdentifiers(const std::string& optionNameList); + void setArgumentTypeName(const std::string& type, uint32_t index = 0); + + void addOptionHeader(const std::string& description); + + void setSyntaxPageMaxLeftColumnSize(uint32_t cols) + { _maxLeftColumnSize = cols; } + + /** + * Parses the command line arguments. Enable vespa debug logging if you want + * to see details. + * + * @throws InvalidCommandLineArgumentsException on any failures. + */ + void parse(); + + /** Writes a syntax page to fit an 80 column screen. */ + void writeSyntaxPage(std::ostream& out, bool showDefaults = true); + + /** Sets some textual description added to syntax page. */ + void setSyntaxMessage(const std::string& msg); + + /** + * Can be used after having added all the options to initialize all the + * parameters to default values. Useful if you want to set defaults first, + * override defaults with config of some kind, and then parse, such that + * command line parameters override config. + */ + void setDefaults() { setDefaults(false); } + + /** + * Useful to clear out all options before shutdown if this class outlives + * a class defining options. + */ + void clear() { + _configurables.clear(); + _options.clear(); + _optionMap.clear(); + _setOptions.clear(); + _arguments.clear(); + } + +private: + void parseOption(const std::string& id, OptionParser&, uint32_t& argPos); + void parseArgument(OptionParser& opt, uint32_t& pos); + OptionParser& addOption(std::shared_ptr<OptionParser> opt); + OptionParser& addArgument(std::shared_ptr<OptionParser> arg); + void setDefaults(bool failUnsetRequired); + + struct OptionHeader; + template<typename Number> struct NumberOptionParser; + struct BoolOptionParser; + struct FlagOptionParser; + struct StringOptionParser; + struct MapOptionParser; + template<typename T> struct ListOptionParser; + +}; + +// ---------------------------------------------------------------------------- + +// Implementation of templates and inner classes +// Not a part of the public interface + +template<typename T> const char* getTypeName(); +template<> inline const char* getTypeName<int32_t>() { return "int"; } +template<> inline const char* getTypeName<uint32_t>() { return "uint"; } +template<> inline const char* getTypeName<int64_t>() { return "long"; } +template<> inline const char* getTypeName<uint64_t>() { return "ulong"; } +template<> inline const char* getTypeName<float>() { return "float"; } +template<> inline const char* getTypeName<double>() { return "double"; } + +struct ProgramOptions::OptionParser { + typedef std::unique_ptr<OptionParser> UP; + typedef std::shared_ptr<OptionParser> SP; + + std::vector<std::string> _names; + std::vector<std::string> _hiddenNames; + uint32_t _argCount; + std::vector<std::string> _argTypes; + bool _hasDefault; + bool _invalidDefault; + std::string _defaultString; + std::string _description; + + OptionParser(const std::string& nameList, uint32_t argCount, + const std::string& desc); + OptionParser(const std::string& nameList, uint32_t argCount, + const std::string& defString, const std::string& desc); + virtual ~OptionParser() {} + + virtual bool isRequired() const { return !_hasDefault; } + virtual void set(const std::vector<std::string>& arguments) = 0; + virtual void setDefault() = 0; + virtual void setInvalidDefault(); + virtual std::string getArgType(uint32_t /* index */) const { return "val"; } + std::string getOptSyntaxString() const; + std::string getArgName() const { + std::string name = _names[0]; + for (uint32_t i=1; i<_names.size(); ++i) name += " " + _names[i]; + return name; + } + virtual bool isHeader() const { return false; } + virtual bool hideFromSyntaxPage() const + { return !isHeader() && _names.empty(); } +}; + +struct ProgramOptions::OptionHeader : public OptionParser { + OptionHeader(const std::string& desc) : OptionParser("", 0, desc) {} + virtual void set(const std::vector<std::string>&) {} + virtual void setDefault() {} + virtual bool isHeader() const { return true; } +}; + +template<typename Number> +struct ProgramOptions::NumberOptionParser : public OptionParser { + Number& _number; + Number _defaultValue; + + std::string getStringValue(Number n); + + NumberOptionParser(const std::string& nameList, Number& number, + const std::string& description) + : OptionParser(nameList, 1, description), + _number(number), + _defaultValue(number) + { + } + + NumberOptionParser(const std::string& nameList, Number& number, + const Number& defValue, const std::string& desc) + : OptionParser(nameList, 1, getStringValue(defValue), desc), + _number(number), + _defaultValue(defValue) + { + } + + virtual void set(const std::vector<std::string>& arguments); + + virtual void setDefault() { _number = _defaultValue; } + + virtual std::string getArgType(uint32_t /* index */) const + { return getTypeName<Number>(); } + +}; + +struct ProgramOptions::BoolOptionParser : public OptionParser { + bool& _value; + bool _defaultValue; + + BoolOptionParser(const std::string& nameList, bool& value, + const std::string& description); + + virtual void set(const std::vector<std::string>&) { _value = true; } + + virtual void setDefault() { _value = false; } +}; + +struct ProgramOptions::FlagOptionParser : public OptionParser { + bool& _value; + bool _unsetValue; + + FlagOptionParser(const std::string& nameList, bool& value, + const std::string& description); + FlagOptionParser(const std::string& nameList, bool& value, + const bool& unsetValue, const std::string& description); + + virtual void set(const std::vector<std::string>&) { _value = !_unsetValue; } + + virtual void setDefault() { _value = _unsetValue; } +}; + +struct ProgramOptions::StringOptionParser : public OptionParser { + std::string& _value; + std::string _defaultValue; + + StringOptionParser(const std::string& nameList, std::string& value, + const std::string& description); + + StringOptionParser(const std::string& nameList, std::string& value, + const std::string& defVal, const std::string& desc); + + virtual void set(const std::vector<std::string>& arguments) + { _value = arguments[0]; } + + virtual void setDefault() { _value = _defaultValue; } + + virtual std::string getArgType(uint32_t /* index */) const + { return "string"; } +}; + +struct ProgramOptions::MapOptionParser : public OptionParser { + typedef std::map<std::string, std::string> MapType; + std::map<std::string, std::string>& _value; + + MapOptionParser(const std::string& nameList, + std::map<std::string, std::string>& value, + const std::string& description); + + virtual void set(const std::vector<std::string>& arguments) + { _value[arguments[0]] = arguments[1]; } + + virtual std::string getArgType(uint32_t /* index */) const + { return "string"; } + + // Default of map is just an empty map. + virtual void setDefault() { _value.clear(); } +}; + +template<typename T> +struct ProgramOptions::ListOptionParser : public OptionParser { + std::vector<T>& _value; + T _singleValue; + OptionParser::UP _entryParser; + + ListOptionParser(const std::string& nameList, + std::vector<T>& value, + const std::string& description) + : OptionParser(nameList, 0, description), + _value(value) + { + } + + T& getSingleValue() { return _singleValue; } + + void setEntryParser(OptionParser::UP entryParser) { + _entryParser = std::move(entryParser); + } + + virtual bool isRequired() const { return false; } + virtual void set(const std::vector<std::string>& arguments) { + for (uint32_t i=0; i<arguments.size(); ++i) { + std::vector<std::string> v; + v.push_back(arguments[i]); + _entryParser->set(v); + _value.push_back(_singleValue); + } + } + virtual void setDefault() { + _value.clear(); + } + virtual std::string getArgType(uint32_t index) const { + return _entryParser->getArgType(index) + "[]"; + } +}; + +#define VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDOPTION(type, parsertype) \ +template<> \ +inline \ +ProgramOptions::OptionParser& \ +ProgramOptions::addOption(const std::string& optionNameList, \ + type& value, const std::string& desc) \ +{ \ + return addOption(OptionParser::SP( \ + new parsertype(optionNameList, value, desc))); \ +} + +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDOPTION(bool, FlagOptionParser); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDOPTION(std::string, StringOptionParser); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDOPTION(int32_t, NumberOptionParser<int32_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDOPTION(uint32_t, NumberOptionParser<uint32_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDOPTION(int64_t, NumberOptionParser<int64_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDOPTION(uint64_t, NumberOptionParser<uint64_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDOPTION(float, NumberOptionParser<float>); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDOPTION(double, NumberOptionParser<double>); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDOPTION(MapOptionParser::MapType, + MapOptionParser); + +#define VESPALIB_PROGRAMOPTIONS_IMPL_ADDOPTION(type, parsertype) \ +template<> \ +inline \ +ProgramOptions::OptionParser& \ +ProgramOptions::addOption(const std::string& optionNameList, \ + type& value, const type& defVal, \ + const std::string& desc) \ +{ \ + return addOption(OptionParser::SP( \ + new parsertype(optionNameList, value, defVal, desc))); \ +} + +VESPALIB_PROGRAMOPTIONS_IMPL_ADDOPTION(bool, FlagOptionParser); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDOPTION(std::string, StringOptionParser); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDOPTION(int32_t, NumberOptionParser<int32_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDOPTION(uint32_t, NumberOptionParser<uint32_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDOPTION(int64_t, NumberOptionParser<int64_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDOPTION(uint64_t, NumberOptionParser<uint64_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDOPTION(float, NumberOptionParser<float>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDOPTION(double, NumberOptionParser<double>); + +#define VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDARGUMENT(type, parsertype) \ +template<> \ +inline \ +ProgramOptions::OptionParser& \ +ProgramOptions::addArgument(const std::string& name, \ + type& value, \ + const std::string& desc) \ +{ \ + return addArgument(OptionParser::SP( \ + new parsertype(name, value, desc))); \ +} + +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDARGUMENT(bool, BoolOptionParser); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDARGUMENT(std::string, StringOptionParser); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDARGUMENT(int32_t, NumberOptionParser<int32_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDARGUMENT(uint32_t, NumberOptionParser<uint32_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDARGUMENT(int64_t, NumberOptionParser<int64_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDARGUMENT(uint64_t, NumberOptionParser<uint64_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDARGUMENT(float, NumberOptionParser<float>); +VESPALIB_PROGRAMOPTIONS_IMPL_NODEF_ADDARGUMENT(double, NumberOptionParser<double>); + +#define VESPALIB_PROGRAMOPTIONS_IMPL_ADDARGUMENT(type, parsertype) \ +template<> \ +inline \ +ProgramOptions::OptionParser& \ +ProgramOptions::addArgument(const std::string& name, \ + type& value, const type& defVal, \ + const std::string& desc) \ +{ \ + return addArgument(OptionParser::SP( \ + new parsertype(name, value, defVal, desc))); \ +} + +VESPALIB_PROGRAMOPTIONS_IMPL_ADDARGUMENT(std::string, StringOptionParser); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDARGUMENT(int32_t, NumberOptionParser<int32_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDARGUMENT(uint32_t, NumberOptionParser<uint32_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDARGUMENT(int64_t, NumberOptionParser<int64_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDARGUMENT(uint64_t, NumberOptionParser<uint64_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDARGUMENT(float, NumberOptionParser<float>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDARGUMENT(double, NumberOptionParser<double>); + +#define VESPALIB_PROGRAMOPTIONS_IMPL_ADDLISTARGUMENT(type, parsertype) \ +template<> \ +inline \ +ProgramOptions::OptionParser& \ +ProgramOptions::addListArgument(const std::string& name, \ + std::vector<type>& value, \ + const std::string& desc) \ +{ \ + ListOptionParser<type>* listParser( \ + new ListOptionParser<type>(name, value, desc)); \ + OptionParser::UP entryParser( \ + new parsertype(name, listParser->getSingleValue(), desc)); \ + listParser->setEntryParser(std::move(entryParser)); \ + return addArgument(OptionParser::SP(listParser)); \ +} + +VESPALIB_PROGRAMOPTIONS_IMPL_ADDLISTARGUMENT(std::string, StringOptionParser); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDLISTARGUMENT(int32_t, NumberOptionParser<int32_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDLISTARGUMENT(uint32_t, NumberOptionParser<uint32_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDLISTARGUMENT(int64_t, NumberOptionParser<int64_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDLISTARGUMENT(uint64_t, NumberOptionParser<uint64_t>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDLISTARGUMENT(float, NumberOptionParser<float>); +VESPALIB_PROGRAMOPTIONS_IMPL_ADDLISTARGUMENT(double, NumberOptionParser<double>); + +} // vespalib + diff --git a/staging_vespalib/src/vespa/vespalib/util/programoptions_testutils.cpp b/staging_vespalib/src/vespa/vespalib/util/programoptions_testutils.cpp new file mode 100644 index 00000000000..16c1767c421 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/programoptions_testutils.cpp @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/util/programoptions_testutils.h> + +namespace vespalib { + +namespace { + std::vector<std::string> splitString(const std::string& source) { + std::vector<std::string> target; + std::string::size_type start = 0; + std::string::size_type stop = source.find(' '); + while (stop != std::string::npos) { + target.push_back(source.substr(start, stop - start)); + start = stop + 1; + stop = source.find(' ', start); + } + target.push_back(source.substr(start)); + return target; + } +} // anonymous + +AppOptions::AppOptions(const std::string& optString) + : _argc(0), _argv(0), _source() +{ + _source = splitString(optString); + _argc = _source.size(); + _argv = new const char*[_source.size()]; + for (int i=0; i<_argc; ++i) { + if (_source[i].size() > 1 + && _source[i][0] == _source[i][_source[i].size() - 1] + && (_source[i][0] == '\'' || _source[i][0] == '"')) + { + if (_source[i].size() == 2) { + _source[i] = ""; + } else { + _source[i] = _source[i].substr(1, _source.size() - 2); + } + } + _argv[i] = _source[i].c_str(); + } +} + +AppOptions::~AppOptions() +{ + delete[] _argv; +} + +} // vespalib diff --git a/staging_vespalib/src/vespa/vespalib/util/programoptions_testutils.h b/staging_vespalib/src/vespa/vespalib/util/programoptions_testutils.h new file mode 100644 index 00000000000..5798a873169 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/programoptions_testutils.h @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * This class contains some test utilities, to create argc/argv inputs for + * application tests. + */ + +#pragma once + +#include <string> +#include <vector> + +namespace vespalib { + +class AppOptions { + int _argc; + const char** _argv; + std::vector<std::string> _source; + + AppOptions(const AppOptions&); + AppOptions& operator=(const AppOptions&); + +public: + AppOptions(const std::string& optString); + ~AppOptions(); + + int getArgCount() const { return _argc; } + const char* const* getArguments() const { return _argv; } + +}; + +} // vespalib + diff --git a/staging_vespalib/src/vespa/vespalib/util/rand48.h b/staging_vespalib/src/vespa/vespalib/util/rand48.h new file mode 100644 index 00000000000..91fcf1b03e9 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/rand48.h @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright (C) 2003 Fast Search & Transfer ASA +// Copyright (C) 2003 Overture Services Norway AS + +#pragma once + +#include <vespa/fastos/fastos.h> + +namespace search { + +/* + * Simple random generator based on lrand48() spec. + */ +class Rand48 +{ +private: + uint64_t _state; +public: + void + srand48(long seed) + { + _state = ((static_cast<uint64_t>(seed & 0xffffffffu)) << 16) + 0x330e; + } + + Rand48(void) + : _state(0) + { + srand48(0x1234abcd); + }; + void iterate(void) { + _state = (UINT64_C(0x5DEECE66D) * _state + 0xb) & + UINT64_C(0xFFFFFFFFFFFF); + } + /* + * Return value from 0 to 2^31 - 1 + */ + long lrand48(void) { + iterate(); + return static_cast<long>(_state >> 17); + } +}; + +} // namespace search + diff --git a/staging_vespalib/src/vespa/vespalib/util/rusage.cpp b/staging_vespalib/src/vespa/vespalib/util/rusage.cpp new file mode 100644 index 00000000000..084cec7cca0 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/rusage.cpp @@ -0,0 +1,130 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "rusage.h" +#include <stdexcept> +#include <vespa/vespalib/util/vstringfmt.h> + +namespace vespalib { + +RUsage::RUsage() : + rusage(), + _time(0) +{ + ru_utime.tv_sec = 0; + ru_utime.tv_usec = 0; + ru_stime.tv_sec = 0; + ru_stime.tv_usec = 0; + ru_maxrss = 0; + ru_ixrss = 0; + ru_idrss = 0; + ru_isrss = 0; + ru_minflt = 0; + ru_majflt = 0; + ru_nswap = 0; + ru_inblock = 0; + ru_oublock = 0; + ru_msgsnd = 0; + ru_msgrcv = 0; + ru_nsignals = 0; + ru_nvcsw = 0; + ru_nivcsw = 0; +} + +RUsage RUsage::createSelf() +{ + return createSelf(0); +} + +RUsage RUsage::createChildren() +{ + return createChildren(0); +} + +RUsage RUsage::createSelf(const fastos::TimeStamp & since) +{ + RUsage r; + r._time = fastos::TimeStamp(fastos::ClockSystem::now()) - since; + if (getrusage(RUSAGE_SELF, &r) != 0) { + throw std::runtime_error(vespalib::make_vespa_string("getrusage failed with errno = %d", errno).c_str()); + } + return r; +} + +RUsage RUsage::createChildren(const fastos::TimeStamp & since) +{ + RUsage r; + r._time = fastos::TimeStamp(fastos::ClockSystem::now()) - since; + if (getrusage(RUSAGE_CHILDREN, &r) != 0) { + throw std::runtime_error(vespalib::make_vespa_string("getrusage failed with errno = %d", errno).c_str()); + } + return r; +} + +vespalib::string RUsage::toString() +{ + vespalib::string s; + if (_time.sec() != 0.0) s += make_vespa_string("duration = %1.6f\n", _time.sec()); + if (fastos::TimeStamp(ru_utime).sec() != 0.0) s += make_vespa_string("user time = %1.6f\n", fastos::TimeStamp(ru_utime).sec()); + if (fastos::TimeStamp(ru_stime).sec() != 0.0) s += make_vespa_string("system time = %1.6f\n", fastos::TimeStamp(ru_stime).sec()); + if (ru_maxrss != 0) s += make_vespa_string("ru_maxrss = %ld\n", ru_maxrss); + if (ru_ixrss != 0) s += make_vespa_string("ru_ixrss = %ld\n", ru_ixrss); + if (ru_idrss != 0) s += make_vespa_string("ru_idrss = %ld\n", ru_idrss); + if (ru_isrss != 0) s += make_vespa_string("ru_isrss = %ld\n", ru_isrss); + if (ru_minflt != 0) s += make_vespa_string("ru_minflt = %ld\n", ru_minflt); + if (ru_majflt != 0) s += make_vespa_string("ru_majflt = %ld\n", ru_majflt); + if (ru_nswap != 0) s += make_vespa_string("ru_nswap = %ld\n", ru_nswap); + if (ru_inblock != 0) s += make_vespa_string("ru_inblock = %ld\n", ru_inblock); + if (ru_oublock != 0) s += make_vespa_string("ru_oublock = %ld\n", ru_oublock); + if (ru_msgsnd != 0) s += make_vespa_string("ru_msgsnd = %ld\n", ru_msgsnd); + if (ru_msgrcv != 0) s += make_vespa_string("ru_msgrcv = %ld\n", ru_msgrcv); + if (ru_nsignals != 0) s += make_vespa_string("ru_nsignals = %ld\n", ru_nsignals); + if (ru_nvcsw != 0) s += make_vespa_string("ru_nvcsw = %ld\n", ru_nvcsw); + if (ru_nivcsw != 0) s += make_vespa_string("ru_nivcsw = %ld", ru_nivcsw); + return s; +} + +RUsage & +RUsage::operator -= (const RUsage & b) +{ + _time -= b._time; + ru_utime = ru_utime - b.ru_utime; + ru_stime = ru_stime - b.ru_stime; + ru_maxrss -= b.ru_maxrss; + ru_ixrss -= b.ru_ixrss; + ru_idrss -= b.ru_idrss; + ru_isrss -= b.ru_isrss; + ru_minflt -= b.ru_minflt; + ru_majflt -= b.ru_majflt; + ru_nswap -= b.ru_nswap; + ru_inblock -= b.ru_inblock; + ru_oublock -= b.ru_oublock; + ru_msgsnd -= b.ru_msgsnd; + ru_msgrcv -= b.ru_msgrcv; + ru_nsignals -= b.ru_nsignals; + ru_nvcsw -= b.ru_nvcsw; + ru_nivcsw -= b.ru_nivcsw; + return *this; +} + +timeval +operator - (const timeval & a, const timeval & b) +{ + timeval tmp; + if (a.tv_usec >= b.tv_usec) { + tmp.tv_usec = a.tv_usec - b.tv_usec; + tmp.tv_sec = a.tv_sec - b.tv_sec; + } else { + tmp.tv_usec = (a.tv_usec + 1000000) - b.tv_usec; + tmp.tv_sec = a.tv_sec - 1 - b.tv_sec; + } + return tmp; +} + +RUsage operator -(const RUsage & a, const RUsage & b) +{ + RUsage tmp(a); + tmp -= b; + return tmp; +} + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/rusage.h b/staging_vespalib/src/vespa/vespalib/util/rusage.h new file mode 100644 index 00000000000..d41b7961097 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/rusage.h @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <vespa/fastos/timestamp.h> + +namespace vespalib { + +class RUsage : private rusage { +public: + /** + * Create an rusage with all member set to zero. + **/ + RUsage(); + /** + * Will create an RUsage and initialize member with RUSAGE_SELF + **/ + static RUsage createSelf(); + static RUsage createSelf(const fastos::TimeStamp & since); + /** + * Will create an RUsage and initialize member with RUSAGE_CHILDREN + **/ + static RUsage createChildren(); + static RUsage createChildren(const fastos::TimeStamp & since); + /** + * Will create an RUsage and initialize member with RUSAGE_CHILDREN + **/ + vespalib::string toString(); + RUsage & operator -= (const RUsage & rhs); +private: + fastos::TimeStamp _time; +}; + +RUsage operator -(const RUsage & a, const RUsage & b); +timeval operator -(const timeval & a, const timeval & b); + +} + diff --git a/staging_vespalib/src/vespa/vespalib/util/shutdownguard.cpp b/staging_vespalib/src/vespa/vespalib/util/shutdownguard.cpp new file mode 100644 index 00000000000..d0d6667556f --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/shutdownguard.cpp @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "shutdownguard.h" +#include <vespa/log/log.h> +LOG_SETUP(".vespalib.shutdownguard"); + +namespace vespalib { + +void ShutdownGuard::Run(FastOS_ThreadInterface *, void *) +{ + while (_dieAtTime > getTimeInMillis()) { + FastOS_Thread::Sleep(5); + } + if (_dieAtTime != 0) { + LOG(warning, "ShutdownGuard is now forcing an exit of the process."); + _exit(EXIT_FAILURE); + } +} + +ShutdownGuard::ShutdownGuard(uint64_t millis) : + FastOS_Runnable(), + _pool(STACK_SIZE, 1), + _dieAtTime(getTimeInMillis() + millis) +{ + _pool.NewThread(this); +} + +ShutdownGuard::~ShutdownGuard() +{ + _dieAtTime = 0; + _pool.Close(); +} + + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/shutdownguard.h b/staging_vespalib/src/vespa/vespalib/util/shutdownguard.h new file mode 100644 index 00000000000..7c94127d903 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/shutdownguard.h @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/fastos/fastos.h> + +namespace vespalib { + +/** + * Class to ensure that the current process finishes within a given time. + * Construct with the number of milliseconds before triggering _exit(); + * destruct the ShutdownGuard object to dismiss the automatic process + * termination. + * A separate shutdown thread will perform the actual _exit() call. + **/ +class ShutdownGuard : public FastOS_Runnable +{ + enum { STACK_SIZE = (1u << 16) }; + + static uint64_t getTimeInMillis() { + struct timeval mytime; + gettimeofday(&mytime, 0); + uint64_t mult = 1000; + return (mytime.tv_sec * mult) + (mytime.tv_usec / mult); + } + + FastOS_ThreadPool _pool; + volatile uint64_t _dieAtTime; + + virtual void Run(FastOS_ThreadInterface *, void *); + +public: + /** + * Construct a shutdown guard with a given lifetime. + * @arg millis the number of milliseconds before process automatically exits + **/ + ShutdownGuard(uint64_t millis); + + /** + * Destructor that dismisses the guard and collects the shutdown thread. + **/ + ~ShutdownGuard(); + +}; + +} // namespace vespalib + diff --git a/staging_vespalib/src/vespa/vespalib/util/sort.h b/staging_vespalib/src/vespa/vespalib/util/sort.h new file mode 100644 index 00000000000..e3ff9a1522b --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/sort.h @@ -0,0 +1,260 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/objects/nbostream.h> +#include <functional> +#include <limits> +#include <algorithm> + +namespace vespalib { + +template<typename T, bool asc=true> +class convertForSort +{ +}; + +template<> +class convertForSort<float, true> +{ +public: + typedef float InputType; + typedef int32_t IntType; + typedef uint32_t UIntType; + typedef std::less<InputType> Compare; + static inline UIntType convert(float value) + { + union { float f; UIntType u; } val; + val.f=value; + return (static_cast<IntType>(val.u) >= 0) + ? (val.u ^ (std::numeric_limits<IntType>::max() + 1)) + : (val.u ^ std::numeric_limits<UIntType>::max()); + } +}; + +template<> +class convertForSort<float, false> +{ +public: + typedef float InputType; + typedef int32_t IntType; + typedef uint32_t UIntType; + typedef std::greater<InputType> Compare; + static inline UIntType convert(float value) + { + union { float f; UIntType u; } val; + val.f=value; + return (static_cast<IntType>(val.u) >= 0) + ? (val.u ^ std::numeric_limits<IntType>::max()) + : val.u; + } +}; + + +template<> +class convertForSort<double, true> +{ +public: + typedef double InputType; + typedef int64_t IntType; + typedef uint64_t UIntType; + typedef std::less<InputType> Compare; + static inline UIntType convert(double value) + { + union { double f; UIntType u; } val; + val.f=value; + return (static_cast<IntType>(val.u) >= 0) + ? (val.u ^ (std::numeric_limits<IntType>::max() + 1)) + : (val.u ^ std::numeric_limits<UIntType>::max()); + } +}; + +template<> +class convertForSort<double, false> +{ +public: + typedef double InputType; + typedef int64_t IntType; + typedef uint64_t UIntType; + typedef std::greater<InputType> Compare; + static inline UIntType convert(double value) + { + union { double f; UIntType u; } val; + val.f=value; + return (static_cast<IntType>(val.u) >= 0) + ? (val.u ^ std::numeric_limits<IntType>::max()) + : val.u; + } +}; + +template<> +class convertForSort<uint8_t, true> +{ +public: + typedef uint8_t InputType; + typedef int8_t IntType; + typedef uint8_t UIntType; + typedef std::less<InputType> Compare; + static inline UIntType convert(UIntType value) { return value; } +}; + +template<> +class convertForSort<uint8_t, false> +{ +public: + typedef uint8_t InputType; + typedef int8_t IntType; + typedef uint8_t UIntType; + typedef std::greater<InputType> Compare; + static inline UIntType convert(UIntType value) { return ~value; } +}; +template<> +class convertForSort<uint16_t, true> +{ +public: + typedef uint16_t InputType; + typedef int16_t IntType; + typedef uint16_t UIntType; + typedef std::less<InputType> Compare; + static inline UIntType convert(UIntType value) { return value; } +}; +template<> +class convertForSort<uint16_t, false> +{ +public: + typedef uint16_t InputType; + typedef int16_t IntType; + typedef uint16_t UIntType; + typedef std::greater<InputType> Compare; + static inline UIntType convert(UIntType value) { return ~value; } +}; +template<> +class convertForSort<uint32_t, true> +{ +public: + typedef uint32_t InputType; + typedef int32_t IntType; + typedef uint32_t UIntType; + typedef std::less<InputType> Compare; + static inline UIntType convert(UIntType value) { return value; } +}; +template<> +class convertForSort<uint32_t, false> +{ +public: + typedef uint32_t InputType; + typedef int32_t IntType; + typedef uint32_t UIntType; + typedef std::greater<InputType> Compare; + static inline UIntType convert(UIntType value) { return ~value; } +}; +template<> +class convertForSort<uint64_t, true> +{ +public: + typedef uint64_t InputType; + typedef int64_t IntType; + typedef uint64_t UIntType; + typedef std::less<InputType> Compare; + static inline UIntType convert(UIntType value) { return value; } +}; +template<> +class convertForSort<uint64_t, false> +{ +public: + typedef uint64_t InputType; + typedef int64_t IntType; + typedef uint64_t UIntType; + typedef std::greater<InputType> Compare; + static inline UIntType convert(UIntType value) { return ~value; } +}; + +template<> +class convertForSort<int8_t, true> +{ +public: + typedef int8_t InputType; + typedef int8_t IntType; + typedef uint8_t UIntType; + typedef std::less<InputType> Compare; + static inline UIntType convert(IntType value) { return value ^ (std::numeric_limits<IntType>::max() + 1); } +}; +template<> +class convertForSort<int8_t, false> +{ +public: + typedef int8_t InputType; + typedef int8_t IntType; + typedef uint8_t UIntType; + typedef std::greater<InputType> Compare; + static inline UIntType convert(IntType value) { return value ^ std::numeric_limits<IntType>::max(); } +}; +template<> +class convertForSort<int16_t, true> +{ +public: + typedef int16_t InputType; + typedef int16_t IntType; + typedef uint16_t UIntType; + typedef std::less<InputType> Compare; + static inline UIntType convert(IntType value) { return value ^ (std::numeric_limits<IntType>::max() + 1); } +}; +template<> +class convertForSort<int16_t, false> +{ +public: + typedef int16_t InputType; + typedef int16_t IntType; + typedef uint16_t UIntType; + typedef std::greater<InputType> Compare; + static inline UIntType convert(IntType value) { return value ^ std::numeric_limits<IntType>::max(); } +}; +template<> +class convertForSort<int32_t, true> +{ +public: + typedef int32_t InputType; + typedef int32_t IntType; + typedef uint32_t UIntType; + typedef std::less<InputType> Compare; + static inline UIntType convert(IntType value) { return value ^ (std::numeric_limits<IntType>::max() + 1); } +}; +template<> +class convertForSort<int32_t, false> +{ +public: + typedef int32_t InputType; + typedef int32_t IntType; + typedef uint32_t UIntType; + typedef std::greater<InputType> Compare; + static inline UIntType convert(IntType value) { return value ^ std::numeric_limits<IntType>::max(); } +}; +template<> +class convertForSort<int64_t, true> +{ +public: + typedef int64_t InputType; + typedef int64_t IntType; + typedef uint64_t UIntType; + typedef std::less<InputType> Compare; + static inline UIntType convert(IntType value) { return value ^ (std::numeric_limits<IntType>::max() + 1); } +}; +template<> +class convertForSort<int64_t, false> +{ +public: + typedef int64_t InputType; + typedef int64_t IntType; + typedef uint64_t UIntType; + typedef std::greater<InputType> Compare; + static inline UIntType convert(IntType value) { return value ^ std::numeric_limits<IntType>::max(); } +}; + +template<typename C> +uint32_t serializeForSort(typename C::InputType v, void * dst) { + typename C::UIntType nbo(vespalib::nbostream::n2h(C::convert(v))); + memcpy(dst, &nbo, sizeof(nbo)); + return sizeof(nbo); +} + +} // namespace vespalib + diff --git a/staging_vespalib/src/vespa/vespalib/util/timer.cpp b/staging_vespalib/src/vespa/vespalib/util/timer.cpp new file mode 100644 index 00000000000..2e19de231e3 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/timer.cpp @@ -0,0 +1,74 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include "timer.h" + +namespace vespalib { + +typedef vespalib::Executor::Task Task; + +class TimerTask : public FNET_Task +{ +private: + TimerTask(const TimerTask &); + TimerTask&operator=(const TimerTask &); + + FNET_Scheduler *_scheduler; + Task::UP _task; + double _interval; +public: + TimerTask(FNET_Scheduler *scheduler, Task::UP task, double interval) + : FNET_Task(scheduler), + _task(std::move(task)), + _interval(interval) + { } + + ~TimerTask(void) + { + Kill(); + } + + void PerformTask() + { + _task->run(); + Schedule(_interval); + } +}; + +Timer::Timer() + : _threadPool(128 * 1024), + _transport(new FNET_Transport()), + _lock(), + _taskList() +{ + _transport->Start(&_threadPool); +} + +Timer::~Timer() +{ + vespalib::LockGuard guard(_lock); + _transport->ShutDown(true); + _threadPool.Close(); + _taskList.clear(); +} + + +void +Timer::scheduleAtFixedRate(vespalib::Executor::Task::UP task, double delay, double interval) +{ + vespalib::LockGuard guard(_lock); + TimerTaskPtr tTask(new TimerTask(_transport->GetScheduler(), std::move(task), interval)); + _taskList.push_back(std::move(tTask)); + _taskList.back()->Schedule(delay); +} + +void +Timer::reset() +{ + vespalib::LockGuard guard(_lock); + _transport->ShutDown(true); + _taskList.clear(); + _transport.reset(new FNET_Transport()); + _transport->Start(&_threadPool); +} + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/timer.h b/staging_vespalib/src/vespa/vespalib/util/timer.h new file mode 100644 index 00000000000..eeb5f6b4c5d --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/timer.h @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/util/linkedptr.h> +#include <vespa/vespalib/util/executor.h> +#include <vespa/vespalib/util/sync.h> +#include <vespa/fnet/fnet.h> + +namespace vespalib { + +class TimerTask; + +/** + * Timer is a class capable of running Tasks at a regular + * interval. The timer can be reset to clear all tasks currently being + * scheduled. + */ +class Timer +{ +private: + typedef std::unique_ptr<TimerTask> TimerTaskPtr; + typedef std::vector<TimerTaskPtr> TaskList; + FastOS_ThreadPool _threadPool; + std::unique_ptr<FNET_Transport> _transport; + vespalib::Lock _lock; + TaskList _taskList; + +public: + /** + * Create a new timer, capable of scheduling tasks at fixed intervals. + */ + Timer(); + + /** + * Destroys this timer, finishing the current task executing and then + * finishing. + */ + ~Timer(); + + /** + * Schedule new task to be executed at specified intervals. + * + * @param task The task to schedule. + * @param delay The delay to wait before first execution. + * @param interval The interval in seconds. + */ + void scheduleAtFixedRate(vespalib::Executor::Task::UP task, double delay, double interval); + + /** + * Reset timer, clearing the list of task to execute. + */ + void reset(); +}; + +} + diff --git a/staging_vespalib/src/vespa/vespalib/util/varholder.h b/staging_vespalib/src/vespa/vespalib/util/varholder.h new file mode 100644 index 00000000000..465ae868253 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/varholder.h @@ -0,0 +1,63 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/noncopyable.hpp> +#include <vespa/vespalib/util/sync.h> + +namespace vespalib +{ + + +template <typename T> +class VarHolder : public noncopyable +{ + + T _v; + vespalib::Lock _lock; + +public: + VarHolder(void) + : _v(), + _lock() + { + } + + explicit + VarHolder(const T &v) + : _v(v), + _lock() + { + } + + ~VarHolder(void) + { + } + + void + set(const T &v) + { + T old; + { + vespalib::LockGuard guard(_lock); + old = _v; + _v = v; + } + } + + void + clear(void) + { + set(T()); + } + + T + get(void) const + { + vespalib::LockGuard guard(_lock); + return _v; + } +}; + +} // namespace vespalib + diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp b/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp new file mode 100644 index 00000000000..1020f714b81 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp @@ -0,0 +1,442 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/util/xmlserializable.h> + +#include <sstream> +#include <vector> +#include <vespa/vespalib/encoding/base64.h> +#include <vespa/vespalib/util/exceptions.h> + +namespace vespalib { +namespace xml { + +namespace { + + std::vector<bool> getLegalIdentifierFirstCharacters() { + std::vector<bool> vec(256, false); + for (uint32_t i='a'; i<='z'; ++i) vec[i] = true; + for (uint32_t i='A'; i<='Z'; ++i) vec[i] = true; + vec[':'] = true; + vec['_'] = true; + return vec; + } + + std::vector<bool> getLegalIdentifierCharacters() { + std::vector<bool> vec(getLegalIdentifierFirstCharacters()); + vec['-'] = true; + vec['.'] = true; + for (uint32_t i='0'; i<='9'; ++i) { + vec[i] = true; + } + return vec; + } + + std::vector<bool> getBinaryCharacters() { + std::vector<bool> vec(256, false); + for (uint32_t i=0; i<32; ++i) { + vec[i] = true; + } + vec['\t'] = false; + vec['\n'] = false; + vec['\r'] = false; + vec['\f'] = false; + return vec; + } + + std::vector<bool> getEscapedXmlCharacters() { + std::vector<bool> vec(256, false); + for (uint32_t i=0; i<32; ++i) { + vec[i] = true; + } + vec['\n'] = false; + vec['<'] = true; + vec['>'] = true; + vec['&'] = true; + return vec; + } + + std::vector<bool> legalIdentifierFirstChar( + getLegalIdentifierFirstCharacters()); + std::vector<bool> legalIdentifierChars = getLegalIdentifierCharacters(); + std::vector<bool> binaryChars = getBinaryCharacters(); + std::vector<bool> escapedXmlChars = getEscapedXmlCharacters(); + + bool containsBinaryCharacters(const std::string& s) { + for (int i=0, n=s.size(); i<n; ++i) { + if (binaryChars[static_cast<uint8_t>(s[i])]) return true; + } + return false; + } + + const std::string xmlAttributeEscape(const std::string& s) { + std::ostringstream ost; + for (uint32_t i=0, n=s.size(); i<n; ++i) { + if (s[i] == '"' || s[i] == '\n' + || escapedXmlChars[static_cast<uint8_t>(s[i])]) + { + if (s[i] == '<') ost << "<"; + else if (s[i] == '>') ost << ">"; + else if (s[i] == '&') ost << "&"; + else if (s[i] == '"') ost << """; + else { + ost << "&#" << (int) s[i] << ";"; + } + } else { + ost << s[i]; + } + } + return ost.str(); + } + + void writeEscaped(std::ostream& out, const std::string& s) { + for (uint32_t i=0, n=s.size(); i<n; ++i) { + if (escapedXmlChars[static_cast<uint8_t>(s[i])]) { + if (s[i] == '<') out << "<"; + else if (s[i] == '>') out << ">"; + else if (s[i] == '&') out << "&"; + else { + out << "&#" << (int) s[i] << ";"; + } + } else { + out << s[i]; + } + } + } + + void writeBase64Encoded(std::ostream& out, const std::string& s) { + out << vespalib::Base64::encode(&s[0], s.size()); + } +} + +bool isLegalName(const std::string& name) { + if (name.size() == 0) return false; + if (!legalIdentifierFirstChar[static_cast<uint8_t>(name[0])]) return false; + for (int i=1, n=name.size(); i<n; ++i) { + if (!legalIdentifierChars[static_cast<uint8_t>(name[i])]) return false; + } + return true; +} + +void convertToLegalName(std::string& name) { + if (name.size() == 0) { + name == "__no_name__"; + } else { + if (!legalIdentifierFirstChar[static_cast<uint8_t>(name[0])]) { + name[0] = '_'; + } + for (int i=1, n=name.size(); i<n; ++i) { + if (!legalIdentifierChars[static_cast<uint8_t>(name[i])]) { + name[i] = '_'; + } + } + } +} + +XmlOutputStream::XmlOutputStream(std::ostream& ostream, + const std::string& indent) + : _indent(indent), + _wrappedStream(ostream), + _tagStack(), + _cachedTag(), + _cachedAttributes(), + _cachedContent() +{ +} + +XmlAttribute::~XmlAttribute() +{ +} + +XmlContent::~XmlContent() +{ +} + +XmlOutputStream::~XmlOutputStream() +{ +} + +XmlOutputStream& +XmlOutputStream::operator<<(const XmlTag& tag) +{ + //std::cerr << "Trying to add tag " << tag.getName() << ". cached tag is " + // << (void*) _cachedTag.get() << "\n"; + if (_cachedTag.get() != 0) flush(false); + _cachedTag.reset(new XmlTag(tag)); + _cachedContentType = XmlContent::AUTO; + //std::cerr << "Added tag " << _cachedTag->getName() << "\n"; + return *this; +} + +XmlOutputStream& +XmlOutputStream::operator<<(const XmlAttribute& attribute) +{ + //std::cerr << "Adding attribute\n"; + if (_cachedTag.get() == 0) { + throw vespalib::IllegalStateException("Cannot add attribute " + + attribute.getName() + ", as no tag is open"); + } + _cachedAttributes.push_back(attribute); + return *this; +} + +XmlOutputStream& +XmlOutputStream::operator<<(const XmlEndTag&) +{ + //std::cerr << "Adding endtag\n"; + if (_cachedTag.get()) { + flush(true); + _cachedContentType = XmlContent::ESCAPED; + } else if (_tagStack.empty()) { + throw vespalib::IllegalStateException("No open tags left to end"); + } else { + for (uint32_t i=1; i<_tagStack.size(); ++i) { + _wrappedStream << _indent; + } + _wrappedStream << "</" << _tagStack.back() << ">"; + _tagStack.pop_back(); + if (!_tagStack.empty()) _wrappedStream << '\n'; + _cachedContentType = XmlContent::ESCAPED; + } + return *this; +} + +XmlOutputStream& +XmlOutputStream::operator<<(const XmlContent& content) +{ + //std::cerr << "Adding content\n"; + if (_cachedTag.get() == 0 && _tagStack.empty()) { + throw vespalib::IllegalStateException( + "No open tag to write content in"); + } + if (_cachedTag.get() != 0) { + //std::cerr << "Content is '" << content.getContent() << "'\n"; + if (content.getType() == XmlContent::AUTO) { // Do nothing.. Always ok + } else if (_cachedContentType == XmlContent::AUTO) { + _cachedContentType = content.getType(); + } else if (_cachedContentType != content.getType()) { + throw vespalib::IllegalStateException( + "Have already added content of different type"); + } + _cachedContent.push_back(content); + } else { + if (content.getType() == XmlContent::BASE64) { + throw vespalib::IllegalStateException( + "Cannot add Base64 encoded content after tag content"); + } + for (uint32_t i=0; i<_tagStack.size(); ++i) { + _wrappedStream << _indent; + } + _wrappedStream << content.getContent() << '\n'; + } + return *this; +} + +XmlOutputStream& +XmlOutputStream::operator<<(const XmlSerializable& serializable) +{ + //std::cerr << "Adding serializable\n"; + serializable.printXml(*this); + return *this; +} + +XmlOutputStream& +XmlOutputStream::operator<<(const std::string& content) +{ + //std::cerr << "Adding content string\n"; + return *this << XmlContent(content); +} + +XmlOutputStream& +XmlOutputStream::operator<<(char c) +{ + return *this << XmlContent(std::string(&c, 1)); +} + +XmlOutputStream& +XmlOutputStream::operator<<(int32_t i) +{ + return *this << XmlContent(vespalib::make_string("%d", i)); +} + +XmlOutputStream& +XmlOutputStream::operator<<(int64_t i) +{ + return *this << XmlContent(vespalib::make_string("%" PRId64, i)); +} + +XmlOutputStream& +XmlOutputStream::operator<<(float f) +{ + return *this << XmlContent(vespalib::make_string("%g", f)); +} + +XmlOutputStream& +XmlOutputStream::operator<<(double d) +{ + return *this << XmlContent(vespalib::make_string("%g", d)); +} + +void +XmlOutputStream::flush(bool endTag) +{ + //std::cerr << "Flushing\n"; + if (_cachedTag.get() == 0) { + throw vespalib::IllegalStateException("Cannot write non-existing tag"); + } + for (uint32_t i=0; i<_tagStack.size(); ++i) { + _wrappedStream << _indent; + } + _wrappedStream << '<' << _cachedTag->getName(); + for (std::list<XmlAttribute>::const_iterator it = _cachedAttributes.begin(); + it != _cachedAttributes.end(); ++it) + { + _wrappedStream << ' ' << it->getName() << "=\"" + << xmlAttributeEscape(it->getValue()) << '"'; + } + _cachedAttributes.clear(); + if (_cachedContent.empty() && endTag) { + _wrappedStream << "/>\n"; + } else if (_cachedContent.empty()) { + _wrappedStream << ">\n"; + _tagStack.push_back(_cachedTag->getName()); + } else { + if (_cachedContentType == XmlContent::AUTO) { + _cachedContentType = XmlContent::ESCAPED; + for (std::list<XmlContent>::const_iterator it + = _cachedContent.begin(); it != _cachedContent.end(); ++it) + { + if (containsBinaryCharacters(it->getContent())) { + _cachedContentType = XmlContent::BASE64; + break; + } + } + } + if (_cachedContentType == XmlContent::BASE64) { + _wrappedStream << " binaryencoding=\"base64\""; + } + _wrappedStream << '>'; + for (std::list<XmlContent>::const_iterator it = _cachedContent.begin(); + it != _cachedContent.end(); ++it) + { + if (!endTag) { + _wrappedStream << '\n'; + for (uint32_t i=0; i<=_tagStack.size(); ++i) { + _wrappedStream << _indent; + } + } + switch (_cachedContentType) { + case XmlContent::ESCAPED: { + writeEscaped(_wrappedStream, it->getContent()); + break; + } + case XmlContent::BASE64: { + writeBase64Encoded(_wrappedStream, it->getContent()); + break; + } + default: assert(false); + } + } + _cachedContent.clear(); + if (endTag) { + _wrappedStream << "</" << _cachedTag->getName() << ">\n"; + } else { + _wrappedStream << '\n'; + _tagStack.push_back(_cachedTag->getName()); + } + } + _cachedTag.reset(0); +} + +XmlTag::XmlTag(const XmlTag& tag) + : _name(tag._name), + _attributes(), + _content(), + _flags(tag._flags) +{ +} + +XmlTag::XmlTag(const std::string& name, XmlTagFlags flags) + : _name(name), + _attributes(), + _content(), + _flags(flags) +{ + if (_flags == CONVERT_ILLEGAL_CHARACTERS) { + convertToLegalName(_name); + } + if (!isLegalName(_name)) { + throw vespalib::IllegalArgumentException("Name '" + _name + "' contains " + "illegal XML characters and cannot be used as tag name"); + } +} + +XmlAttribute::XmlAttribute(const XmlAttribute& attribute) + : _name(attribute._name), + _value(attribute._value), + _next() +{ +} + +XmlEndTag::XmlEndTag() +{ +} + +XmlContent::XmlContent(Type type) + : _type(type), + _content(), + _nextContent(), + _nextTag() +{ +} + +XmlContent::XmlContent() + : _type(AUTO), + _content(), + _nextContent(), + _nextTag() +{ +} + +XmlContent::XmlContent(const XmlContent& content) + : _type(content._type), + _content(content._content), + _nextContent(), + _nextTag() +{ +} + +XmlContent::XmlContent(const std::string& value) + : _type(AUTO), + _content(value), + _nextContent(), + _nextTag() +{ +} + +XmlContentWrapper::XmlContentWrapper(const XmlContentWrapper& wrapper) + : XmlContent(wrapper) +{ +} + +XmlContentWrapper::XmlContentWrapper(const char* value) + : XmlContent(std::string(value)) +{ +} + +XmlContentWrapper::XmlContentWrapper(const char* value, uint32_t size) + : XmlContent(std::string(value, size)) +{ +} + +std::string +XmlSerializable::toXml(const std::string& indent) const +{ + std::ostringstream ost; + XmlOutputStream xos(ost, indent); + printXml(xos); + return ost.str(); +} + +} // xml +} // vespalib diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.h b/staging_vespalib/src/vespa/vespalib/util/xmlserializable.h new file mode 100644 index 00000000000..83162024580 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/xmlserializable.h @@ -0,0 +1,257 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @file xmlserializable.h + * @ingroup util + * + * @brief Interfaces to be used for XML serialization. + * + * This file contains XML utility classes, to make XML serialization simple. + * Rather than users writing their own XML, these tools let you define a tree + * structure, and this library builds the XML for you. This ensures that you + * write legal XML and that stuff that needs to be escaped is. + * <p> + * It defines a superclass for XML serializable classes, called XmlSerializable. + * This is what classes that should be XML serializable will inherit. + * <p> + * When implementing the printXml() function in XmlSerializable, one will + * use the various XML helper classes defined here to build a tree structure + * creating the XML. These are: XmlTag, XmlEndTag, XmlAttribute, and XmlContent. + * Some subclasses exist of XmlContent to facilitate various types of content. + * <p> + * The XmlOutputStream wraps a regular std::ostream. You write XML objects to it + * and it is responsible for writing all the XML code. This way, the XML + * serialization is done without interfering with regular output operators. + * <p> + * For example usage, refer to the unit test: + * vespalib/tests/xmlserializable/xmlserializabletest.cpp + * + */ + +#pragma once + +#include <iostream> +#include <list> +#include <memory> +#include <sstream> +#include <vespa/vespalib/util/exception.h> +#include <vespa/vespalib/objects/cloneable.h> +#include <vespa/vespalib/objects/identifiable.h> + +namespace vespalib { +namespace xml { + +class XmlAttribute; +class XmlContent; +class XmlOutputStream; + +bool isLegalName(const std::string& name); + +enum XmlTagFlags { NONE = 0, CONVERT_ILLEGAL_CHARACTERS = 1 }; + +/** + * @class document::XmlTag + * + * @brief Start a new tag with given name. + */ +class XmlTag { + std::string _name; + std::unique_ptr<XmlAttribute> _attributes; + std::unique_ptr<XmlContent> _content; + XmlTagFlags _flags; +public: + XmlTag(const XmlTag&); + XmlTag(const std::string& name, XmlTagFlags = NONE); + + const std::string& getName() const { return _name; } +}; + +/** + * @class document::XmlEndTag + * + * @brief Indicates that current tag is closed. + */ +class XmlEndTag { +public: + XmlEndTag(); +}; + +/** + * @class document::XmlAttribute + * + * @brief Defined a single attribute within an XML tag. + * + * When adding an XML to an XML stream, the attribute will be added to the last + * tag added. This can not be called after the last tag opened in the stream is + * closed, so add all attributes before starting to add new XML child tags. + */ +class XmlAttribute { + std::string _name; + std::string _value; + std::unique_ptr<XmlAttribute> _next; +public: + enum Flag { NONE = 0x0, HEX = 0x1 }; + XmlAttribute(const XmlAttribute&); + /** Add any value that can be written to an ostringstream. */ + template<typename T> + XmlAttribute(const std::string& name, const T& value, + uint32_t flags = NONE); + ~XmlAttribute(); + + const std::string& getName() const { return _name; } + const std::string& getValue() const { return _value; } +}; + +template<typename T> +XmlAttribute::XmlAttribute(const std::string& name, const T& value, + uint32_t flags) + : _name(name), + _value(), + _next() +{ + std::ostringstream ost; + if (flags & HEX) ost << std::hex << "0x"; + ost << value; + _value = ost.str(); + if (!isLegalName(name)) { + throw vespalib::IllegalArgumentException("Name '" + name + "' contains " + "illegal XML characters and cannot be used as attribute name"); + } +} + + +/** + * @class document::XmlContent + * + * XML content to be written to stream. By default it will autodetect whether to + * escape or base64 encode content. XmlOutputStream functions taking primitives + * will generate XmlContent instances. + */ +class XmlContent { +public: + enum Type { AUTO, ESCAPED, BASE64 }; +protected: + XmlContent(Type type); +private: + Type _type; + std::string _content; + std::unique_ptr<XmlContent> _nextContent; + std::unique_ptr<XmlTag> _nextTag; + +public: + XmlContent(); + XmlContent(const XmlContent&); + XmlContent(const std::string& value); + ~XmlContent(); + + Type getType() const { return _type; } + const std::string& getContent() const { return _content; } +}; + +/** + * @class document::XmlEscapedContent + * + * Token used to tell that this content field should only be XML escaped. + */ +class XmlEscapedContent : public XmlContent { +public: + XmlEscapedContent() : XmlContent(ESCAPED) {} +}; + +/** + * @class document::XmlBase64Content + * + * Token used to tell that this content field should always be base64 encoded. + */ +class XmlBase64Content : public XmlContent { +public: + XmlBase64Content() : XmlContent(BASE64) {} +}; + +/** + * @class document::XmlContentWrapper + * + * A wrapper class for content that one doesn't want to copy or release + * ownership of. This wrapper merely takes pointer to data, and assumes it + * will stay alive as long as needed. + */ +class XmlContentWrapper : public XmlContent { +public: + XmlContentWrapper(const XmlContentWrapper&); + XmlContentWrapper(const char* value); + XmlContentWrapper(const char* value, uint32_t size); +}; + +/** + * @class document::XmlSerializable + * + * Base class for classes that can be converted into XML. + */ +class XmlSerializable +{ +public: + XmlSerializable() {} + virtual ~XmlSerializable() {} + + virtual void printXml(XmlOutputStream& out) const = 0; + + /** Utility function, using printXml() to create a string. */ + virtual std::string toXml(const std::string& indent = "") const; +}; + +/** + * @class document::XmlOutputStream + * + * @brief std::ostream wrapper, only accepting data that will become XML. + * + * After XmlEndTag() has been sent to the stream, the tag is guarantueed to have + * been written. Call isFinalized() to ensure that you have closed all the tags + * that have been opened. Within a tag, the stream will cache some information, + * as more information might be required before knowing what to print. + */ +class XmlOutputStream { + const std::string _indent; + std::ostream& _wrappedStream; + std::list<std::string> _tagStack; + std::unique_ptr<XmlTag> _cachedTag; + std::list<XmlAttribute> _cachedAttributes; + std::list<XmlContent> _cachedContent; + XmlContent::Type _cachedContentType; + + void flush(bool endTag); + +public: + + XmlOutputStream(std::ostream& ostream, const std::string& indent = ""); + ~XmlOutputStream(); + + bool isFinalized() const + { return (_tagStack.empty() && _cachedTag.get() == 0); } + + std::ostream& getWrappedStream() { return _wrappedStream; } + + XmlOutputStream& operator<<(const XmlTag& tag); + XmlOutputStream& operator<<(const XmlAttribute& attribute); + XmlOutputStream& operator<<(const XmlEndTag& endtag); + XmlOutputStream& operator<<(const XmlContent& content); + XmlOutputStream& operator<<(const XmlSerializable& serializable); + + XmlOutputStream& operator<<(const std::string& content); + XmlOutputStream& operator<<(char c); + XmlOutputStream& operator<<(int32_t i); + XmlOutputStream& operator<<(int64_t i); + XmlOutputStream& operator<<(float f); + XmlOutputStream& operator<<(double d); +}; + +} // xml + +// The XmlSerializable and XmlOutputStream is often used in header files +// and is thus available in the vespalib namespace. To not pollute the +// vespalib namespace with all the other classes, use +// "using namespace vespalib::xml" within your printXml functions + +typedef vespalib::xml::XmlOutputStream XmlOutputStream; +typedef vespalib::xml::XmlSerializable XmlSerializable; + +} // vespalib + diff --git a/staging_vespalib/testrun/.gitignore b/staging_vespalib/testrun/.gitignore new file mode 100644 index 00000000000..faed45bc94a --- /dev/null +++ b/staging_vespalib/testrun/.gitignore @@ -0,0 +1,10 @@ +test-report.html +test-report.html.* +test.*.*.desc +test.*.*.file.* +test.*.*.files.html +test.*.*.log +tmp.* +xsync.log +/test.*.*.result +Makefile |