diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2023-12-13 21:25:04 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-12-13 21:25:04 +0100 |
commit | d976f82207c09b3215661e1d034ae9a42f28a63d (patch) | |
tree | 2d301a94e4326e1b9493f77a8b4419073de6a379 /eval/src | |
parent | 4f48e420144ab7288fe45406bd4d1ea69de6eecb (diff) |
Revert "add parsing of special strings for inf/nan cell values"
Diffstat (limited to 'eval/src')
3 files changed, 6 insertions, 67 deletions
diff --git a/eval/src/tests/eval/value_cache/dense-special.json b/eval/src/tests/eval/value_cache/dense-special.json deleted file mode 100644 index a3bee19a573..00000000000 --- a/eval/src/tests/eval/value_cache/dense-special.json +++ /dev/null @@ -1,8 +0,0 @@ -[ - "Infinity", "+Infinity", - "INF", "+INF", - "-Infinity", "-INF", - "NAN", "+NAN", - null, - "-nan", "-NAN" -] diff --git a/eval/src/tests/eval/value_cache/tensor_loader_test.cpp b/eval/src/tests/eval/value_cache/tensor_loader_test.cpp index 82d3e2b0a55..22847a1d08e 100644 --- a/eval/src/tests/eval/value_cache/tensor_loader_test.cpp +++ b/eval/src/tests/eval/value_cache/tensor_loader_test.cpp @@ -130,28 +130,4 @@ TEST_F("require that bad lz4 file fails to load creating empty result", Constant TEST_DO(verify_tensor(sparse_tensor_nocells(), f1.create(TEST_PATH("bad_lz4.json.lz4"), "tensor(x{},y{})"))); } -void checkBitEq(double a, double b) { - size_t aa, bb; - memcpy(&aa, &a, sizeof(aa)); - memcpy(&bb, &b, sizeof(bb)); - EXPECT_EQUAL(aa, bb); -} - -TEST_F("require that special string-encoded values work", ConstantTensorLoader(factory)) { - auto c = f1.create(TEST_PATH("dense-special.json"), "tensor<float>(z[11])"); - const auto &v = c->value(); - auto cells = v.cells().template typify<float>(); - EXPECT_EQUAL(std::numeric_limits<float>::infinity(), cells[0]); - EXPECT_EQUAL(std::numeric_limits<float>::infinity(), cells[1]); - EXPECT_EQUAL(std::numeric_limits<float>::infinity(), cells[2]); - EXPECT_EQUAL(std::numeric_limits<float>::infinity(), cells[3]); - EXPECT_EQUAL(-std::numeric_limits<float>::infinity(), cells[4]); - EXPECT_EQUAL(-std::numeric_limits<float>::infinity(), cells[5]); - TEST_DO(checkBitEq(std::numeric_limits<float>::quiet_NaN(), cells[6])); - TEST_DO(checkBitEq(std::numeric_limits<float>::quiet_NaN(), cells[7])); - TEST_DO(checkBitEq(std::numeric_limits<float>::quiet_NaN(), cells[8])); - TEST_DO(checkBitEq(-std::numeric_limits<float>::quiet_NaN(), cells[9])); - TEST_DO(checkBitEq(-std::numeric_limits<float>::quiet_NaN(), cells[10])); -} - TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp b/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp index 7cdc55fc37f..189f7aa14ce 100644 --- a/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp +++ b/eval/src/vespa/eval/eval/value_cache/constant_tensor_loader.cpp @@ -3,11 +3,10 @@ #include "constant_tensor_loader.h" #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value_codec.h> +#include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/io/mapped_file_input.h> #include <vespa/vespalib/data/lz4_input_decoder.h> #include <vespa/vespalib/data/slime/slime.h> -#include <vespa/vespalib/io/mapped_file_input.h> -#include <vespa/vespalib/objects/nbostream.h> -#include <vespa/vespalib/text/lowercase.h> #include <vespa/vespalib/util/size_literals.h> #include <set> @@ -21,34 +20,6 @@ using ObjectTraverser = slime::ObjectTraverser; namespace { -double decodeDouble(const Inspector &inspector) { - if (inspector.type().getId() == vespalib::slime::STRING::ID) { - auto orig = inspector.asString().make_stringref(); - auto lower = vespalib::LowerCase::convert(orig); - if (lower == "infinity" || lower == "+infinity" || lower == "inf" || lower == "+inf") { - double d = std::numeric_limits<double>::infinity(); - return d; - } - if (lower == "-infinity" || lower == "-inf") { - double d = -std::numeric_limits<double>::infinity(); - return d; - } - if (lower == "nan" || lower == "+nan") { - double d = std::numeric_limits<double>::quiet_NaN(); - return d; - } - if (lower == "-nan") { - double d = -std::numeric_limits<double>::quiet_NaN(); - return d; - } - LOG(warning, "bad string-encoded numeric value '%.*s'", (int)orig.size(), orig.data()); - } - if (inspector.valid() && inspector.type().getId() == vespalib::slime::NIX::ID) { - return std::numeric_limits<double>::quiet_NaN(); - } - return inspector.asDouble(); -} - struct Target { const ValueType tensor_type; TensorSpec spec; @@ -139,7 +110,7 @@ struct SingleMappedExtractor : ObjectTraverser { {} void field(const Memory &symbol, const Inspector &inspector) override { vespalib::string label = symbol.make_string(); - double value = decodeDouble(inspector); + double value = inspector.asDouble(); TensorSpec::Address address; address.emplace(dimension, label); target.check_add(address, value); @@ -157,7 +128,7 @@ void decodeSingleDenseForm(const Inspector &values, const ValueType &value_type, for (size_t i = 0; i < values.entries(); ++i) { TensorSpec::Address address; address.emplace(dimension, TensorSpec::Label(i)); - target.check_add(address, decodeDouble(values[i])); + target.check_add(address, values[i].asDouble()); } } @@ -166,7 +137,7 @@ struct DenseValuesDecoder { Target &_target; void decode(const Inspector &input, const TensorSpec::Address &address, size_t dim_idx) { if (dim_idx == _idims.size()) { - _target.check_add(address, decodeDouble(input)); + _target.check_add(address, input.asDouble()); } else { const auto &dimension = _idims[dim_idx]; if (input.entries() != dimension.size) { @@ -238,7 +209,7 @@ void decodeLiteralForm(const Inspector &cells, const ValueType &value_type, Targ TensorSpec::Address address; AddressExtractor extractor(indexed, address); cells[i]["address"].traverse(extractor); - target.check_add(address, decodeDouble(cells[i]["value"])); + target.check_add(address, cells[i]["value"].asDouble()); } } |