Add handling of NaN (not a number) in unique store.

6 files changed, 136 insertions, 2 deletions

diff --git a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp
index ba2c98a6d54..0c82bcbc6ed 100644
--- a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp
+++ b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp
@@ -135,6 +135,7 @@ struct TestBase : public ::testing::Test {
 using NumberUniqueStore  = UniqueStore<uint32_t>;
 using StringUniqueStore  = UniqueStore<std::string>;
 using CStringUniqueStore = UniqueStore<const char *, EntryRefT<22>, UniqueStoreStringComparator<EntryRefT<22>>, UniqueStoreStringAllocator<EntryRefT<22>>>;
+using DoubleUniqueStore  = UniqueStore<double>;
 using SmallOffsetNumberUniqueStore = UniqueStore<uint32_t, EntryRefT<10,10>>;
 
 template <>
@@ -143,6 +144,8 @@ template <>
 std::vector<std::string> TestBase<StringUniqueStore>::values{ "aa", "bbb", "ccc", "aa" };
 template <>
 std::vector<const char *> TestBase<CStringUniqueStore>::values{ "aa", "bbb", "ccc", "aa" };
+template <>
+std::vector<double> TestBase<DoubleUniqueStore>::values{ 10.0, 20.0, 30.0, 10.0 };
 
 using UniqueStoreTestTypes = ::testing::Types<NumberUniqueStore, StringUniqueStore, CStringUniqueStore>;
 TYPED_TEST_CASE(TestBase, UniqueStoreTestTypes);
@@ -156,6 +159,7 @@ TYPED_TEST_CASE(TestBase, UniqueStoreTestTypes);
 using NumberTest = TestBase<NumberUniqueStore>;
 using StringTest = TestBase<StringUniqueStore>;
 using CStringTest = TestBase<CStringUniqueStore>;
+using DoubleTest = TestBase<DoubleUniqueStore>;
 using SmallOffsetNumberTest = TestBase<SmallOffsetNumberUniqueStore>;
 
 TEST(UniqueStoreTest, trivial_and_non_trivial_types_are_tested)
@@ -291,4 +295,33 @@ TYPED_TEST(TestBase, store_can_be_saved)
 
 #pragma GCC diagnostic pop
 
+TEST_F(DoubleTest, nan_is_handled)
+{
+    std::vector<double> myvalues = {
+        std::numeric_limits<double>::quiet_NaN(),
+        std::numeric_limits<double>::infinity(),
+        -std::numeric_limits<double>::infinity(),
+        10.0,
+        -std::numeric_limits<double>::quiet_NaN(),
+        std::numeric_limits<double>::infinity(),
+        -std::numeric_limits<double>::infinity()
+    };
+    std::vector<EntryRef> refs;
+    refs.push_back(EntryRef());
+    for (auto &value : myvalues) {
+        refs.emplace_back(add(value));
+    }
+    trimHoldLists();
+    EXPECT_TRUE(std::isnan(store.get(refs[1])));
+    EXPECT_TRUE(std::signbit(store.get(refs[1])));
+    auto saver = getSaver();
+    saver.enumerateValues();
+    std::vector<uint32_t> enumerated;
+    for (auto &ref : refs) {
+        enumerated.push_back(saver.mapEntryRefToEnumValue(ref));
+    }
+    std::vector<uint32_t> exp_enumerated = { 0, 1, 4, 2, 3, 1, 4, 2 };
+    EXPECT_EQ(exp_enumerated, enumerated);
+}    
+                
 GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp
index 7a170507096..83772224713 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp
@@ -3,6 +3,7 @@
 #pragma once
 
 #include "unique_store_allocator.h"
+#include "unique_store_value_filter.h"
 #include "datastore.hpp"
 
 namespace search::datastore {
@@ -33,7 +34,7 @@ template <typename EntryT, typename RefT>
 EntryRef
 UniqueStoreAllocator<EntryT, RefT>::allocate(const EntryType& value)
 {
-    return _store.template freeListAllocator<WrappedEntryType,  UniqueStoreEntryReclaimer<WrappedEntryType>>(0).alloc(value).ref;
+    return _store.template freeListAllocator<WrappedEntryType,  UniqueStoreEntryReclaimer<WrappedEntryType>>(0).alloc(UniqueStoreValueFilter<EntryType>::filter(value)).ref;
 }
 
 template <typename EntryT, typename RefT>
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_comparator.h b/vespalib/src/vespa/vespalib/datastore/unique_store_comparator.h
index 12c798fcd52..acd4f531e2c 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_comparator.h
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_comparator.h
@@ -5,10 +5,57 @@
 #include "entry_comparator.h"
 #include "unique_store_entry.h"
 #include "datastore.h"
+#include <cmath>
 
 namespace search::datastore {
 
 /*
+ * Helper class for comparing elements in unique store. 
+ */
+template <typename EntryT>
+class UniqueStoreComparatorHelper {
+public:
+    static bool less(const EntryT& lhs, const EntryT& rhs) {
+        return lhs < rhs;
+    }
+};
+
+/*
+ * Helper class for comparing floating point elements in unique store with
+ * special handling of NAN.
+ */
+template <typename EntryT>
+class UniqueStoreFloatingPointComparatorHelper
+{
+public:
+    static bool less(EntryT lhs, const EntryT rhs) {
+        if (std::isnan(lhs)) {
+            return !std::isnan(rhs);
+        } else if (std::isnan(rhs)) {
+            return false;
+        } else {
+            return (lhs < rhs);
+        }
+    }
+};
+
+/*
+ * Specialized helper class for comparing float elements in unique store with
+ * special handling of NAN.
+ */
+template <>
+class UniqueStoreComparatorHelper<float> : public UniqueStoreFloatingPointComparatorHelper<float> {
+};
+
+/*
+ * Specialized helper class for comparing double elements in unique store with
+ * special handling of NAN.
+ */
+template <>
+class UniqueStoreComparatorHelper<double> : public UniqueStoreFloatingPointComparatorHelper<double> {
+};
+  
+/*
  * Compare two entries based on entry refs.  Valid entry ref is mapped
  * to an entry in a data store.  Invalid entry ref is mapped to a
  * temporary entry referenced by comparator instance.
@@ -39,7 +86,7 @@ public:
     {
         const EntryType &lhsValue = get(lhs);
         const EntryType &rhsValue = get(rhs);
-        return lhsValue < rhsValue;
+        return UniqueStoreComparatorHelper<EntryT>::less(lhsValue, rhsValue);
     }
 };
 
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_entry.h b/vespalib/src/vespa/vespalib/datastore/unique_store_entry.h
index b9157901b12..87c8e860244 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_entry.h
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_entry.h
@@ -4,6 +4,7 @@
 
 #include "unique_store_entry_base.h"
 #include <cassert>
+#include <utility>
 
 namespace search::datastore {
 
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_entry_base.h b/vespalib/src/vespa/vespalib/datastore/unique_store_entry_base.h
index cda2ea8337e..2b5bff45d79 100644
--- a/vespalib/src/vespa/vespalib/datastore/unique_store_entry_base.h
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_entry_base.h
@@ -3,6 +3,7 @@
 #pragma once
 
 #include <cstring>
+#include <cstdint>
 
 namespace search::datastore {
 
diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_value_filter.h b/vespalib/src/vespa/vespalib/datastore/unique_store_value_filter.h
new file mode 100644
index 00000000000..3694936c550
--- /dev/null
+++ b/vespalib/src/vespa/vespalib/datastore/unique_store_value_filter.h
@@ -0,0 +1,51 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+namespace search::datastore {
+
+/*
+ * Helper class for normalizing values inserted into unique store.
+ */
+template <typename EntryT>
+class UniqueStoreValueFilter {
+public:
+    static const EntryT &filter(const EntryT &value) {
+        return value;
+    }
+};
+
+/*
+ * Specialized helper class for normalizing floating point values
+ * inserted into unique store.  Any type of NAN is normalized to a
+ * specific one.
+ */
+template <typename EntryT>
+class UniqueStoreFloatingPointValueFilter {
+    static const EntryT normalized_nan;
+public:
+    static const EntryT &filter(const EntryT &value) {
+        return std::isnan(value) ? normalized_nan : value;
+    }
+};
+
+template <typename EntryT>
+const EntryT UniqueStoreFloatingPointValueFilter<EntryT>::normalized_nan = -std::numeric_limits<EntryT>::quiet_NaN(); 
+
+/*
+ * Specialized helper class for normalizing float values inserted into unique store.
+ * Any type of NAN is normalized to a specific one.
+ */
+template <>
+class UniqueStoreValueFilter<float> : public UniqueStoreFloatingPointValueFilter<float> {
+};
+
+/*
+ * Specialized helper class for normalizing double values inserted into unique store.
+ * Any type of NAN is normalized to a specific one.
+ */
+template <>
+class UniqueStoreValueFilter<double> : public UniqueStoreFloatingPointValueFilter<double> {
+};
+
+}