add parsing of special strings for inf/nan cell values

3 files changed, 62 insertions, 6 deletions

diff --git a/eval/src/tests/eval/value_cache/dense-special.json b/eval/src/tests/eval/value_cache/dense-special.json
new file mode 100644
index 00000000000..644f0f436e7
--- /dev/null
+++ b/eval/src/tests/eval/value_cache/dense-special.json
@@ -0,0 +1,7 @@
+[
+    "Infinity", "+Infinity",
+    "INF", "+INF",
+    "-Infinity", "-INF",
+    "NAN", "+NAN",
+    "-nan", "-NAN"
+]
diff --git a/eval/src/tests/eval/value_cache/tensor_loader_test.cpp b/eval/src/tests/eval/value_cache/tensor_loader_test.cpp
index 22847a1d08e..24d6aaab007 100644
--- a/eval/src/tests/eval/value_cache/tensor_loader_test.cpp
+++ b/eval/src/tests/eval/value_cache/tensor_loader_test.cpp
@@ -130,4 +130,27 @@ TEST_F("require that bad lz4 file fails to load creating empty result", Constant
     TEST_DO(verify_tensor(sparse_tensor_nocells(), f1.create(TEST_PATH("bad_lz4.json.lz4"), "tensor(x{},y{})")));
 }
 
+void checkBitEq(double a, double b) {
+    size_t aa, bb;
+    memcpy(&aa, &a, sizeof(aa));
+    memcpy(&bb, &b, sizeof(bb));
+    EXPECT_EQUAL(aa, bb);
+}
+
+TEST_F("require that special string-encoded values work", ConstantTensorLoader(factory)) {
+    auto c = f1.create(TEST_PATH("dense-special.json"), "tensor<float>(z[10])");
+    const auto &v = c->value();
+    auto cells = v.cells().template typify<float>();
+    EXPECT_EQUAL(std::numeric_limits<float>::infinity(), cells[0]);
+    EXPECT_EQUAL(std::numeric_limits<float>::infinity(), cells[1]);
+    EXPECT_EQUAL(std::numeric_limits<float>::infinity(), cells[2]);
+    EXPECT_EQUAL(std::numeric_limits<float>::infinity(), cells[3]);
+    EXPECT_EQUAL(-std::numeric_limits<float>::infinity(), cells[4]);
+    EXPECT_EQUAL(-std::numeric_limits<float>::infinity(), cells[5]);
+    TEST_DO(checkBitEq(std::numeric_limits<float>::quiet_NaN(), cells[6]));
+    TEST_DO(checkBitEq(std::numeric_limits<float>::quiet_NaN(), cells[7]));
+    TEST_DO(checkBitEq(-std::numeric_limits<float>::quiet_NaN(), cells[8]));
+    TEST_DO(checkBitEq(-std::numeric_limits<float>::quiet_NaN(), cells[9]));
+}
+
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp b/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp
index 189f7aa14ce..aea3e8a76fa 100644
--- a/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp
+++ b/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp
@@ -3,10 +3,11 @@
 #include "constant_tensor_loader.h"
 #include <vespa/eval/eval/tensor_spec.h>
 #include <vespa/eval/eval/value_codec.h>
-#include <vespa/vespalib/objects/nbostream.h>
-#include <vespa/vespalib/io/mapped_file_input.h>
 #include <vespa/vespalib/data/lz4_input_decoder.h>
 #include <vespa/vespalib/data/slime/slime.h>
+#include <vespa/vespalib/io/mapped_file_input.h>
+#include <vespa/vespalib/objects/nbostream.h>
+#include <vespa/vespalib/text/lowercase.h>
 #include <vespa/vespalib/util/size_literals.h>
 #include <set>
 
@@ -20,6 +21,31 @@ using ObjectTraverser = slime::ObjectTraverser;
 
 namespace {
 
+double decodeDouble(const Inspector &inspector) {
+    if (inspector.type().getId() == vespalib::slime::STRING::ID) {
+        auto orig = inspector.asString().make_stringref();
+        auto lower = vespalib::LowerCase::convert(orig);
+        if (lower == "infinity" || lower == "+infinity" || lower == "inf" || lower == "+inf") {
+            double d = std::numeric_limits<double>::infinity();
+            return d;
+        }
+        if (lower == "-infinity" || lower == "-inf") {
+            double d = -std::numeric_limits<double>::infinity();
+            return d;
+        }
+        if (lower == "nan" || lower == "+nan") {
+            double d = std::numeric_limits<double>::quiet_NaN();
+            return d;
+        }
+        if (lower == "-nan") {
+            double d = -std::numeric_limits<double>::quiet_NaN();
+            return d;
+        }
+        LOG(warning, "bad string-encoded numeric value '%.*s'", (int)orig.size(), orig.data());
+    }
+    return inspector.asDouble();
+}
+
 struct Target {
     const ValueType tensor_type;
     TensorSpec spec;
@@ -110,7 +136,7 @@ struct SingleMappedExtractor : ObjectTraverser {
     {}
     void field(const Memory &symbol, const Inspector &inspector) override {
         vespalib::string label = symbol.make_string();
-        double value = inspector.asDouble();
+        double value = decodeDouble(inspector);
         TensorSpec::Address address;
         address.emplace(dimension, label);
         target.check_add(address, value);
@@ -128,7 +154,7 @@ void decodeSingleDenseForm(const Inspector &values, const ValueType &value_type,
     for (size_t i = 0; i < values.entries(); ++i) {
         TensorSpec::Address address;
         address.emplace(dimension, TensorSpec::Label(i));
-        target.check_add(address, values[i].asDouble());
+        target.check_add(address, decodeDouble(values[i]));
     }
 }
 
@@ -137,7 +163,7 @@ struct DenseValuesDecoder {
     Target &_target;
     void decode(const Inspector &input, const TensorSpec::Address &address, size_t dim_idx) {
         if (dim_idx == _idims.size()) {
-            _target.check_add(address, input.asDouble());
+            _target.check_add(address, decodeDouble(input));
         } else {
             const auto &dimension = _idims[dim_idx];
             if (input.entries() != dimension.size) {
@@ -209,7 +235,7 @@ void decodeLiteralForm(const Inspector &cells, const ValueType &value_type, Targ
         TensorSpec::Address address;
         AddressExtractor extractor(indexed, address);
         cells[i]["address"].traverse(extractor);
-        target.check_add(address, cells[i]["value"].asDouble());
+        target.check_add(address, decodeDouble(cells[i]["value"]));
     }
 }