diff options
Diffstat (limited to 'document/src')
4 files changed, 94 insertions, 134 deletions
diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp index 88795041951..031cf1e400a 100644 --- a/document/src/tests/documentupdatetestcase.cpp +++ b/document/src/tests/documentupdatetestcase.cpp @@ -549,7 +549,7 @@ TEST(DocumentUpdateTest, testGenerateSerializedFile) ArithmeticValueUpdate(ArithmeticValueUpdate::Add, 2))) .addUpdate(MapValueUpdate(StringFieldValue("foo"), ArithmeticValueUpdate(ArithmeticValueUpdate::Mul, 2)))); - ByteBuffer::UP buf(serialize42(upd)); + ByteBuffer::UP buf(serializeHEAD(upd)); writeBufferToFile(*buf, "data/serializeupdatecpp.dat"); } diff --git a/document/src/tests/serialization/vespadocumentserializer_test.cpp b/document/src/tests/serialization/vespadocumentserializer_test.cpp index 98a03b0ac89..7feea4e51d2 100644 --- a/document/src/tests/serialization/vespadocumentserializer_test.cpp +++ b/document/src/tests/serialization/vespadocumentserializer_test.cpp @@ -367,31 +367,6 @@ TEST("requireThatArrayFieldValueCanBeSerialized") { TEST_DO(checkArrayFieldValue<uint32_t>(0x4000)); } -TEST("requireThatOldVersionArrayFieldValueCanBeDeserialized") { - uint16_t old_version = 6; - - nbostream stream; - uint32_t type_id = 3; - uint32_t size = 2; - uint32_t element_size = 4; - uint32_t element1 = 21, element2 = 42; - stream << type_id << size - << element_size << element1 - << element_size << element2; - - ArrayDataType array_type(*DataType::INT); - ArrayFieldValue value(array_type); - VespaDocumentDeserializer deserializer(repo, stream, old_version); - deserializer.read(value); - ASSERT_TRUE(EXPECT_EQUAL(size, value.size())); - IntFieldValue *int_value = dynamic_cast<IntFieldValue *>(&value[0]); - ASSERT_TRUE(int_value); - EXPECT_EQUAL(element1, static_cast<uint32_t>(int_value->getValue())); - int_value = dynamic_cast<IntFieldValue *>(&value[1]); - ASSERT_TRUE(int_value); - EXPECT_EQUAL(element2, static_cast<uint32_t>(int_value->getValue())); -} - template <typename SizeType> void checkMapFieldValue(SizeType value_count, bool check_equal) { MapDataType map_type(*DataType::LONG, *DataType::BYTE); @@ -422,33 +397,6 @@ TEST("requireThatMapFieldValueCanBeSerialized") { TEST_DO(checkMapFieldValue<uint32_t>(0x4000, false)); } -TEST("requireThatOldVersionMapFieldValueCanBeDeserialized") { - uint16_t old_version = 6; - - nbostream stream; - uint32_t type_id = 4; - uint32_t size = 2; - uint32_t element_size = 9; - uint64_t key1 = 21, key2 = 42; - uint8_t val1 = 1, val2 = 2; - stream << type_id << size - << element_size << key1 << val1 - << element_size << key2 << val2; - - MapDataType map_type(*DataType::LONG, *DataType::BYTE); - MapFieldValue value(map_type); - VespaDocumentDeserializer deserializer(repo, stream, old_version); - deserializer.read(value); - ASSERT_TRUE(EXPECT_EQUAL(size, value.size())); - - ASSERT_TRUE(value.contains(LongFieldValue(key1))); - ASSERT_TRUE(value.contains(LongFieldValue(key2))); - EXPECT_EQUAL(ByteFieldValue(val1), - *value.find(LongFieldValue(key1))->second); - EXPECT_EQUAL(ByteFieldValue(val2), - *value.find(LongFieldValue(key2))->second); -} - TEST("requireThatWeightedSetFieldValueCanBeSerialized") { WeightedSetDataType ws_type(*DataType::DOUBLE, false, false); WeightedSetFieldValue value(ws_type); @@ -599,7 +547,7 @@ TEST("requireThatDocumentCanBeSerialized") { EXPECT_EQUAL(0, read_version); } -TEST("requireThatOldVersionDocumentCanBeDeserialized") { +TEST("requireThatOldVersionDocumentCanNotBeDeserialized") { uint16_t old_version = 6; uint16_t data_size = 432; string doc_id = "id:ns:my_doctype::"; @@ -615,11 +563,12 @@ TEST("requireThatOldVersionDocumentCanBeDeserialized") { Document value; VespaDocumentDeserializer deserializer(repo, stream, old_version); - deserializer.read(value); - - EXPECT_EQUAL(doc_id, value.getId().getScheme().toString()); - EXPECT_EQUAL(doc_name, value.getType().getName()); - EXPECT_TRUE(value.getFields().empty()); + try { + deserializer.read(value); + ASSERT_FALSE(true); + } catch (vespalib::Exception & e) { + EXPECT_EQUAL("CORRUPT DATA: Unrecognized serialization version 6", e.getMessage()); + } } TEST("requireThatUnmodifiedDocumentRetainsUnknownFieldOnSerialization") { diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.cpp b/document/src/vespa/document/serialization/vespadocumentserializer.cpp index 9a2e6a89f80..6e8c7a811de 100644 --- a/document/src/vespa/document/serialization/vespadocumentserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentserializer.cpp @@ -47,27 +47,33 @@ VespaDocumentSerializer::VespaDocumentSerializer(nbostream &stream) : _stream(stream) { } -void VespaDocumentSerializer::writeFieldValue(const FieldValue &value) { +void +VespaDocumentSerializer::writeFieldValue(const FieldValue &value) { write(value); } -void VespaDocumentSerializer::writeSerializedData(const void *buf, size_t length) { +void +VespaDocumentSerializer::writeSerializedData(const void *buf, size_t length) { _stream.write(buf, length); } -void VespaDocumentSerializer::write(const ValueUpdate &value) { +void +VespaDocumentSerializer::write(const ValueUpdate &value) { value.accept(*this); } -void VespaDocumentSerializer::write(const FieldPathUpdate &value) { +void +VespaDocumentSerializer::write(const FieldPathUpdate &value) { value.accept(*this); } -void VespaDocumentSerializer::write(const FieldValue &value) { +void +VespaDocumentSerializer::write(const FieldValue &value) { value.accept(*this); } -void VespaDocumentSerializer::write(const DocumentId &value) { +void +VespaDocumentSerializer::write(const DocumentId &value) { string id_string = value.getScheme().toString(); _stream.write(id_string.data(), id_string.size()); _stream << static_cast<uint8_t>(0); @@ -100,7 +106,8 @@ static inline size_t wantChunks(bool hasHeader, bool hasBody) { return res; } -void VespaDocumentSerializer::write(const Document &value, DocSerializationMode mode) { +void +VespaDocumentSerializer::write(const Document &value, DocSerializationMode mode) { nbostream doc_stream; VespaDocumentSerializer doc_serializer(doc_stream); doc_serializer.write(value.getId()); @@ -133,7 +140,7 @@ void VespaDocumentSerializer::write(const Document &value, DocSerializationMode // chunks contain, so we just send them as-is, even if some fields // may have moved from header to body or vice versa. if (hasHeader || hasBody) { - assert(chunks.size() >= 1); + assert( ! chunks.empty()); doc_serializer.writeUnchanged(chunks[0]); } if (hasHeader && hasBody) { @@ -155,10 +162,11 @@ void VespaDocumentSerializer::write(const Document &value, DocSerializationMode _stream.write(doc_stream.peek(), doc_stream.size()); } -void VespaDocumentSerializer::visit(const StructFieldValue &value) +void +VespaDocumentSerializer::visit(const StructFieldValue &value) { const StructFieldValue::Chunks & chunks = value.getChunks(); - if (!structNeedsReserialization(value) && chunks.size() > 0) { + if (!structNeedsReserialization(value) && ! chunks.empty()) { assert(chunks.size() == 1); writeUnchanged(chunks[0]); } else { @@ -166,19 +174,21 @@ void VespaDocumentSerializer::visit(const StructFieldValue &value) } } -void VespaDocumentSerializer::write(const AnnotationReferenceFieldValue &value) +void +VespaDocumentSerializer::write(const AnnotationReferenceFieldValue &value) { putInt1_2_4Bytes(_stream, value.getAnnotationIndex()); } void VespaDocumentSerializer::write(const ArrayFieldValue &value) { putInt1_2_4Bytes(_stream, value.size()); - for (size_t i(0), m(value.size()); i < m; ++i) { - value[i].accept(*this); + for (auto & item : value) { + item.accept(*this); } } -void VespaDocumentSerializer::write(const MapFieldValue &value) { +void +VespaDocumentSerializer::write(const MapFieldValue &value) { putInt1_2_4Bytes(_stream, value.size()); for (const auto & entry : value) { (*entry.first).accept(*this); @@ -186,56 +196,57 @@ void VespaDocumentSerializer::write(const MapFieldValue &value) { } } -void VespaDocumentSerializer::write(const BoolFieldValue &value) { +void +VespaDocumentSerializer::write(const BoolFieldValue &value) { _stream << value.getValue(); } -void VespaDocumentSerializer::write(const ByteFieldValue &value) { +void +VespaDocumentSerializer::write(const ByteFieldValue &value) { _stream << value.getValue(); } -void VespaDocumentSerializer::write(const DoubleFieldValue &value) { +void +VespaDocumentSerializer::write(const DoubleFieldValue &value) { _stream << value.getValue(); } -void VespaDocumentSerializer::write(const FloatFieldValue &value) { +void +VespaDocumentSerializer::write(const FloatFieldValue &value) { _stream << value.getValue(); } -void VespaDocumentSerializer::write(const IntFieldValue &value) { +void +VespaDocumentSerializer::write(const IntFieldValue &value) { _stream << static_cast<uint32_t>(value.getValue()); } -void VespaDocumentSerializer::write(const LongFieldValue &value) { +void +VespaDocumentSerializer::write(const LongFieldValue &value) { _stream << static_cast<uint64_t>(value.getValue()); } -void VespaDocumentSerializer::write(const PredicateFieldValue &value) { +void +VespaDocumentSerializer::write(const PredicateFieldValue &value) { SlimeOutputToVector output; vespalib::slime::BinaryFormat::encode(value.getSlime(), output); _stream << static_cast<uint32_t>(output.size()); _stream.write(output.data(), output.size()); } -void VespaDocumentSerializer::write(const RawFieldValue &value) { +void +VespaDocumentSerializer::write(const RawFieldValue &value) { _stream << static_cast<uint32_t>(value.getValueRef().size()); _stream.write(value.getValueRef().data(), value.getValueRef().size()); } -void VespaDocumentSerializer::write(const ShortFieldValue &value) { +void +VespaDocumentSerializer::write(const ShortFieldValue &value) { _stream << static_cast<uint16_t>(value.getValue()); } -namespace { -template <typename Map> -void writeAnnotations(AnnotationSerializer &serializer, const Map &m) { - for (const auto & annotation : m) { - serializer.write(*annotation.second); - } -} -} // namespace - -void VespaDocumentSerializer::write(const StringFieldValue &value) { +void +VespaDocumentSerializer::write(const StringFieldValue &value) { uint8_t coding = (value.hasSpanTrees() << 6); _stream << coding; putInt1_4Bytes(_stream, value.getValueRef().size() + 1); @@ -249,8 +260,9 @@ void VespaDocumentSerializer::write(const StringFieldValue &value) { } namespace { -void serializeFields(const StructFieldValue &value, nbostream &stream, - vector<pair<uint32_t, uint32_t> > &field_info, const FieldSet& fieldSet) +void +serializeFields(const StructFieldValue &value, nbostream &stream, + vector<pair<uint32_t, uint32_t> > &field_info, const FieldSet& fieldSet) { VespaDocumentSerializer serializer(stream); for (StructuredFieldValue::const_iterator it(value.begin()), e(value.end()); it != e; ++it) { @@ -263,7 +275,7 @@ void serializeFields(const StructFieldValue &value, nbostream &stream, continue; } size_t field_size = stream.size() - original_size; - field_info.push_back(make_pair(it.field().getId(), field_size)); + field_info.emplace_back(it.field().getId(), field_size); } } @@ -297,11 +309,12 @@ compressStream(const CompressionConfig &config, nbostream &stream, vespalib::Dat return buf; } -void putFieldInfo(nbostream &output, const vector<pair<uint32_t, uint32_t> > &field_info) { +void +putFieldInfo(nbostream &output, const vector<pair<uint32_t, uint32_t> > &field_info) { putInt1_4Bytes(output, field_info.size()); - for (size_t i = 0; i < field_info.size(); ++i) { - putInt1_4Bytes(output, field_info[i].first); - putInt2_4_8Bytes(output, field_info[i].second); + for (const auto & field : field_info) { + putInt1_4Bytes(output, field.first); + putInt2_4_8Bytes(output, field.second); } } } // namespace @@ -313,8 +326,8 @@ void putFieldInfo(nbostream &output, const vector<pair<uint32_t, uint32_t> > &fi * The last and is to make sure that we do not decompress a document * unintentionally. */ -bool VespaDocumentSerializer::structNeedsReserialization( - const StructFieldValue &value) +bool +VespaDocumentSerializer::structNeedsReserialization(const StructFieldValue &value) { if (value.hasChanged()) { return true; @@ -351,7 +364,7 @@ void VespaDocumentSerializer::writeUnchanged(const SerializableArray &value) { } const ByteBuffer* buffer = value.getSerializedBuffer(); - uint32_t sz = (buffer != NULL) ? buffer->getLength() : 0; + uint32_t sz = (buffer != nullptr) ? buffer->getLength() : 0; size_t estimatedRequiredSpace = sz + 4 + 1 + 8 + 4 + field_info.size()*12; _stream.reserve(_stream.size() + estimatedRequiredSpace); _stream << sz; @@ -390,8 +403,9 @@ void VespaDocumentSerializer::write(const StructFieldValue &value, _stream.write(toSerialize.c_str(), toSerialize.size()); } -void VespaDocumentSerializer::write(const WeightedSetFieldValue &value) { - const WeightedSetDataType *type = static_cast<const WeightedSetDataType *>(value.getDataType()); +void +VespaDocumentSerializer::write(const WeightedSetFieldValue &value) { + auto type = static_cast<const WeightedSetDataType *>(value.getDataType()); _stream << static_cast<uint32_t>(type->getNestedType().getId()); _stream << static_cast<uint32_t>(value.size()); for (const auto & entry : value) { @@ -411,7 +425,7 @@ VespaDocumentSerializer::write(const TensorFieldValue &value) { auto &tensor = value.getAsTensorPtr(); if (tensor) { vespalib::tensor::TypedBinaryFormat::serialize(tmpStream, *tensor); - assert(tmpStream.size() != 0); + assert( ! tmpStream.empty()); _stream.putInt1_4Bytes(tmpStream.size()); _stream.write(tmpStream.peek(), tmpStream.size()); } else { @@ -419,7 +433,8 @@ VespaDocumentSerializer::write(const TensorFieldValue &value) { } } -void VespaDocumentSerializer::write(const ReferenceFieldValue& value) { +void +VespaDocumentSerializer::write(const ReferenceFieldValue& value) { _stream << static_cast<uint8_t>(value.hasValidDocumentId() ? 1 : 0); if (value.hasValidDocumentId()) { write(value.getDocumentId()); @@ -427,25 +442,11 @@ void VespaDocumentSerializer::write(const ReferenceFieldValue& value) { } namespace { - const uint8_t CONTENT_HASTYPE(0x01); const uint8_t CONTENT_HASVALUE(0x01); } -void VespaDocumentSerializer::write42(const DocumentUpdate &value) -{ - _stream << static_cast<uint16_t>(Document::getNewestSerializationVersion()); - write(value.getId()); - _stream << static_cast<uint8_t>(CONTENT_HASTYPE); - _stream.write(value.getType().getName().c_str(), value.getType().getName().size() + 1); - _stream << static_cast<uint16_t>(0); - const DocumentUpdate::FieldUpdateV & updates(value.getUpdates()); - _stream << static_cast<uint32_t>(value.serializeFlags(updates.size())); - for (const auto & update : updates) { - write(update); - } -} - -void VespaDocumentSerializer::writeHEAD(const DocumentUpdate &value) +void +VespaDocumentSerializer::writeHEAD(const DocumentUpdate &value) { if (!value._needHardReserialize) { _stream.write(value._backing.peek(), value._backing.size()); @@ -465,7 +466,8 @@ void VespaDocumentSerializer::writeHEAD(const DocumentUpdate &value) } } -void VespaDocumentSerializer::write(const FieldUpdate &value) +void +VespaDocumentSerializer::write(const FieldUpdate &value) { _stream << static_cast<int32_t>(value.getField().getId()); _stream << static_cast<int32_t>(value.size()); @@ -474,28 +476,32 @@ void VespaDocumentSerializer::write(const FieldUpdate &value) } } -void VespaDocumentSerializer::write(const RemoveValueUpdate &value) +void +VespaDocumentSerializer::write(const RemoveValueUpdate &value) { _stream << RemoveValueUpdate::classId; write(value.getKey()); } -void VespaDocumentSerializer::write(const AddValueUpdate &value) +void +VespaDocumentSerializer::write(const AddValueUpdate &value) { _stream << AddValueUpdate::classId; write(value.getValue()); _stream << static_cast<int32_t>(value.getWeight()); } -void VespaDocumentSerializer::write(const ArithmeticValueUpdate &value) +void +VespaDocumentSerializer::write(const ArithmeticValueUpdate &value) { _stream << ArithmeticValueUpdate::classId; _stream << static_cast<uint32_t>(value.getOperator()); _stream << static_cast<double>(value.getOperand()); } -void VespaDocumentSerializer::write(const AssignValueUpdate &value) +void +VespaDocumentSerializer::write(const AssignValueUpdate &value) { _stream << AssignValueUpdate::classId; if (value.hasValue()) { @@ -506,7 +512,8 @@ void VespaDocumentSerializer::write(const AssignValueUpdate &value) } } -void VespaDocumentSerializer::write(const ClearValueUpdate &value) +void +VespaDocumentSerializer::write(const ClearValueUpdate &value) { (void) value; _stream << ClearValueUpdate::classId; @@ -524,14 +531,16 @@ namespace { // We must ensure that string passed is always zero-terminated, so take in // string instead of stringref. No extra allocs; function only ever called with // string arguments. -void writeStringWithZeroTermination(nbostream & os, const vespalib::string& s) +void +writeStringWithZeroTermination(nbostream & os, const vespalib::string& s) { uint32_t sz(s.size() + 1); os << sz; os.write(s.c_str(), sz); } -void writeFieldPath(nbostream & os, const FieldPathUpdate & value) +void +writeFieldPath(nbostream & os, const FieldPathUpdate & value) { writeStringWithZeroTermination(os, value.getOriginalFieldPath()); writeStringWithZeroTermination(os, value.getOriginalWhereClause()); @@ -539,13 +548,15 @@ void writeFieldPath(nbostream & os, const FieldPathUpdate & value) } -void VespaDocumentSerializer::write(const AddFieldPathUpdate &value) +void +VespaDocumentSerializer::write(const AddFieldPathUpdate &value) { writeFieldPath(_stream, value); write(value.getValues()); } -void VespaDocumentSerializer::write(const AssignFieldPathUpdate &value) +void +VespaDocumentSerializer::write(const AssignFieldPathUpdate &value) { writeFieldPath(_stream, value); uint8_t flags = 0; @@ -561,7 +572,8 @@ void VespaDocumentSerializer::write(const AssignFieldPathUpdate &value) } -void VespaDocumentSerializer::write(const RemoveFieldPathUpdate &value) +void +VespaDocumentSerializer::write(const RemoveFieldPathUpdate &value) { writeFieldPath(_stream, value); } diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.h b/document/src/vespa/document/serialization/vespadocumentserializer.h index ba3bf63afa7..d83532771e8 100644 --- a/document/src/vespa/document/serialization/vespadocumentserializer.h +++ b/document/src/vespa/document/serialization/vespadocumentserializer.h @@ -53,7 +53,6 @@ public: void write(const TensorFieldValue &value); void write(const ReferenceFieldValue& value); - void write42(const DocumentUpdate &value); void writeHEAD(const DocumentUpdate &value); void write(const FieldUpdate &value); void write(const ValueUpdate &value); |