diff options
author | Tor Egge <Tor.Egge@broadpark.no> | 2019-08-20 14:01:04 +0200 |
---|---|---|
committer | Tor Egge <Tor.Egge@broadpark.no> | 2019-08-20 14:20:57 +0200 |
commit | d1fa0d15e07d7d6dd10dbc22dd650a313d065074 (patch) | |
tree | 67bf7db6fd2ecd748897d8dee53825939128c1a3 /vespalib | |
parent | e95f9418f8dd5cf163fbbe4b802f8e4480f9a4c7 (diff) |
Add handling of NaN (not a number) in unique store.
Diffstat (limited to 'vespalib')
6 files changed, 136 insertions, 2 deletions
diff --git a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp index ba2c98a6d54..0c82bcbc6ed 100644 --- a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp +++ b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp @@ -135,6 +135,7 @@ struct TestBase : public ::testing::Test { using NumberUniqueStore = UniqueStore<uint32_t>; using StringUniqueStore = UniqueStore<std::string>; using CStringUniqueStore = UniqueStore<const char *, EntryRefT<22>, UniqueStoreStringComparator<EntryRefT<22>>, UniqueStoreStringAllocator<EntryRefT<22>>>; +using DoubleUniqueStore = UniqueStore<double>; using SmallOffsetNumberUniqueStore = UniqueStore<uint32_t, EntryRefT<10,10>>; template <> @@ -143,6 +144,8 @@ template <> std::vector<std::string> TestBase<StringUniqueStore>::values{ "aa", "bbb", "ccc", "aa" }; template <> std::vector<const char *> TestBase<CStringUniqueStore>::values{ "aa", "bbb", "ccc", "aa" }; +template <> +std::vector<double> TestBase<DoubleUniqueStore>::values{ 10.0, 20.0, 30.0, 10.0 }; using UniqueStoreTestTypes = ::testing::Types<NumberUniqueStore, StringUniqueStore, CStringUniqueStore>; TYPED_TEST_CASE(TestBase, UniqueStoreTestTypes); @@ -156,6 +159,7 @@ TYPED_TEST_CASE(TestBase, UniqueStoreTestTypes); using NumberTest = TestBase<NumberUniqueStore>; using StringTest = TestBase<StringUniqueStore>; using CStringTest = TestBase<CStringUniqueStore>; +using DoubleTest = TestBase<DoubleUniqueStore>; using SmallOffsetNumberTest = TestBase<SmallOffsetNumberUniqueStore>; TEST(UniqueStoreTest, trivial_and_non_trivial_types_are_tested) @@ -291,4 +295,33 @@ TYPED_TEST(TestBase, store_can_be_saved) #pragma GCC diagnostic pop +TEST_F(DoubleTest, nan_is_handled) +{ + std::vector<double> myvalues = { + std::numeric_limits<double>::quiet_NaN(), + std::numeric_limits<double>::infinity(), + -std::numeric_limits<double>::infinity(), + 10.0, + -std::numeric_limits<double>::quiet_NaN(), + std::numeric_limits<double>::infinity(), + -std::numeric_limits<double>::infinity() + }; + std::vector<EntryRef> refs; + refs.push_back(EntryRef()); + for (auto &value : myvalues) { + refs.emplace_back(add(value)); + } + trimHoldLists(); + EXPECT_TRUE(std::isnan(store.get(refs[1]))); + EXPECT_TRUE(std::signbit(store.get(refs[1]))); + auto saver = getSaver(); + saver.enumerateValues(); + std::vector<uint32_t> enumerated; + for (auto &ref : refs) { + enumerated.push_back(saver.mapEntryRefToEnumValue(ref)); + } + std::vector<uint32_t> exp_enumerated = { 0, 1, 4, 2, 3, 1, 4, 2 }; + EXPECT_EQ(exp_enumerated, enumerated); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp index 7a170507096..83772224713 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp @@ -3,6 +3,7 @@ #pragma once #include "unique_store_allocator.h" +#include "unique_store_value_filter.h" #include "datastore.hpp" namespace search::datastore { @@ -33,7 +34,7 @@ template <typename EntryT, typename RefT> EntryRef UniqueStoreAllocator<EntryT, RefT>::allocate(const EntryType& value) { - return _store.template freeListAllocator<WrappedEntryType, UniqueStoreEntryReclaimer<WrappedEntryType>>(0).alloc(value).ref; + return _store.template freeListAllocator<WrappedEntryType, UniqueStoreEntryReclaimer<WrappedEntryType>>(0).alloc(UniqueStoreValueFilter<EntryType>::filter(value)).ref; } template <typename EntryT, typename RefT> diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_comparator.h b/vespalib/src/vespa/vespalib/datastore/unique_store_comparator.h index 12c798fcd52..acd4f531e2c 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_comparator.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_comparator.h @@ -5,10 +5,57 @@ #include "entry_comparator.h" #include "unique_store_entry.h" #include "datastore.h" +#include <cmath> namespace search::datastore { /* + * Helper class for comparing elements in unique store. + */ +template <typename EntryT> +class UniqueStoreComparatorHelper { +public: + static bool less(const EntryT& lhs, const EntryT& rhs) { + return lhs < rhs; + } +}; + +/* + * Helper class for comparing floating point elements in unique store with + * special handling of NAN. + */ +template <typename EntryT> +class UniqueStoreFloatingPointComparatorHelper +{ +public: + static bool less(EntryT lhs, const EntryT rhs) { + if (std::isnan(lhs)) { + return !std::isnan(rhs); + } else if (std::isnan(rhs)) { + return false; + } else { + return (lhs < rhs); + } + } +}; + +/* + * Specialized helper class for comparing float elements in unique store with + * special handling of NAN. + */ +template <> +class UniqueStoreComparatorHelper<float> : public UniqueStoreFloatingPointComparatorHelper<float> { +}; + +/* + * Specialized helper class for comparing double elements in unique store with + * special handling of NAN. + */ +template <> +class UniqueStoreComparatorHelper<double> : public UniqueStoreFloatingPointComparatorHelper<double> { +}; + +/* * Compare two entries based on entry refs. Valid entry ref is mapped * to an entry in a data store. Invalid entry ref is mapped to a * temporary entry referenced by comparator instance. @@ -39,7 +86,7 @@ public: { const EntryType &lhsValue = get(lhs); const EntryType &rhsValue = get(rhs); - return lhsValue < rhsValue; + return UniqueStoreComparatorHelper<EntryT>::less(lhsValue, rhsValue); } }; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_entry.h b/vespalib/src/vespa/vespalib/datastore/unique_store_entry.h index b9157901b12..87c8e860244 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_entry.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_entry.h @@ -4,6 +4,7 @@ #include "unique_store_entry_base.h" #include <cassert> +#include <utility> namespace search::datastore { diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_entry_base.h b/vespalib/src/vespa/vespalib/datastore/unique_store_entry_base.h index cda2ea8337e..2b5bff45d79 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_entry_base.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_entry_base.h @@ -3,6 +3,7 @@ #pragma once #include <cstring> +#include <cstdint> namespace search::datastore { diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_value_filter.h b/vespalib/src/vespa/vespalib/datastore/unique_store_value_filter.h new file mode 100644 index 00000000000..3694936c550 --- /dev/null +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_value_filter.h @@ -0,0 +1,51 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace search::datastore { + +/* + * Helper class for normalizing values inserted into unique store. + */ +template <typename EntryT> +class UniqueStoreValueFilter { +public: + static const EntryT &filter(const EntryT &value) { + return value; + } +}; + +/* + * Specialized helper class for normalizing floating point values + * inserted into unique store. Any type of NAN is normalized to a + * specific one. + */ +template <typename EntryT> +class UniqueStoreFloatingPointValueFilter { + static const EntryT normalized_nan; +public: + static const EntryT &filter(const EntryT &value) { + return std::isnan(value) ? normalized_nan : value; + } +}; + +template <typename EntryT> +const EntryT UniqueStoreFloatingPointValueFilter<EntryT>::normalized_nan = -std::numeric_limits<EntryT>::quiet_NaN(); + +/* + * Specialized helper class for normalizing float values inserted into unique store. + * Any type of NAN is normalized to a specific one. + */ +template <> +class UniqueStoreValueFilter<float> : public UniqueStoreFloatingPointValueFilter<float> { +}; + +/* + * Specialized helper class for normalizing double values inserted into unique store. + * Any type of NAN is normalized to a specific one. + */ +template <> +class UniqueStoreValueFilter<double> : public UniqueStoreFloatingPointValueFilter<double> { +}; + +} |