Merge pull request #25871 from vespa-engine/balder/drop-boost-crc-random-tokenizer

Drop boost crc, random and tokenizer

7 files changed, 25 insertions, 86 deletions

diff --git a/document/src/tests/documenttestcase.cpp b/document/src/tests/documenttestcase.cpp
index 4c97e99459e..bc4065438e6 100644
--- a/document/src/tests/documenttestcase.cpp
+++ b/document/src/tests/documenttestcase.cpp
@@ -900,37 +900,6 @@ TEST(DocumentTest, testBogusserialize)
     }
 }
 
-TEST(DocumentTest, testCRC32)
-{
-    TestDocRepo test_repo;
-    Document doc(*test_repo.getDocumentType("testdoctype1"), DocumentId("id:ns:testdoctype1::crawler:http://www.ntnu.no/"));
-
-    doc.setValue(doc.getField("hstringval"), StringFieldValue("bla bla bla bla bla"));
-
-    uint32_t crc = doc.calculateChecksum();
-    EXPECT_EQ(3987392271u, crc);
-
-    nbostream buf = doc.serialize();
-
-    int pos = 30;
-
-    // Corrupt serialization.
-    const_cast<char *>(buf.peek())[pos] ^= 72;
-        // Create document. Byte corrupted above is in data area and
-        // shouldn't fail deserialization.
-    try {
-        Document doc2(test_repo.getTypeRepo(), buf);
-        buf.rp(0);
-        EXPECT_TRUE(crc != doc2.calculateChecksum());
-    } catch (document::DeserializeException& e) {
-        EXPECT_TRUE(false);
-    }
-        // Return original value and retry
-    const_cast<char *>(buf.peek())[pos] ^= 72;
-
-    /// \todo TODO (was warning):  Cannot test for in memory representation altered, as there is no syntax for getting internal refs to data from document. Add test when this is added.
-}
-
 TEST(DocumentTest, testSliceSerialize)
 {
         // Test that document doesn't need its own bytebuffer, such that we
diff --git a/document/src/vespa/document/base/testdocman.cpp b/document/src/vespa/document/base/testdocman.cpp
index 2994de9c9c1..59e4d4d8386 100644
--- a/document/src/vespa/document/base/testdocman.cpp
+++ b/document/src/vespa/document/base/testdocman.cpp
@@ -5,7 +5,7 @@
 #include <vespa/document/repo/documenttyperepo.h>
 #include <vespa/document/fieldvalue/document.h>
 #include <vespa/document/fieldvalue/stringfieldvalue.h>
-#include <boost/random.hpp>
+#include <vespa/vespalib/util/rand48.h>
 #include <sstream>
 
 namespace document {
@@ -62,13 +62,13 @@ TestDocMan::TestDocMan()
       _typeCfg(&_test_repo.getTypeConfig())
 { }
 
-TestDocMan::~TestDocMan() { }
+TestDocMan::~TestDocMan() = default;
 
 void
 TestDocMan::setTypeRepo(const std::shared_ptr<const DocumentTypeRepo> &repo)
 {
     _repo = repo;
-    _typeCfg = NULL;
+    _typeCfg = nullptr;
 }
 
 Document::UP
@@ -77,8 +77,7 @@ TestDocMan::createDocument(
         const std::string& id,
         const std::string& type) const
 {
-    const DocumentType *type_ptr = 0;
-    type_ptr = _repo->getDocumentType(type);
+    const DocumentType *type_ptr = _repo->getDocumentType(type);
     assert(type_ptr);
     Document::UP doc(new Document(*type_ptr, DocumentId(id)));
     doc->setValue(doc->getField("content"), StringFieldValue(content.c_str()));
@@ -96,11 +95,11 @@ Document::UP
 TestDocMan::createRandomDocumentAtLocation(
                 int location, int seed, int maxContentSize) const
 {
-    boost::rand48 rnd(seed);
+    vespalib::Rand48 rnd(seed);
     std::ostringstream id;
-    id << "id:mail:testdoctype1:n=" << location << ":" << (rnd() % 0x10000)
+    id << "id:mail:testdoctype1:n=" << location << ":" << (rnd.lrand48() % 0x10000)
        << ".html";
-    return createDocument(generateRandomContent(rnd() % maxContentSize),
+    return createDocument(generateRandomContent(rnd.lrand48() % maxContentSize),
                           id.str(), "testdoctype1");
 }
 
@@ -108,27 +107,26 @@ Document::UP
 TestDocMan::createRandomDocumentAtLocation(
         int location, int seed, int minContentSize, int maxContentSize) const
 {
-    boost::rand48 rnd(seed);
+    vespalib::Rand48 rnd(seed);
     std::ostringstream id;
-    id << "id:mail:testdoctype1:n=" << location << ":" << (rnd() % 0x10000)
+    id << "id:mail:testdoctype1:n=" << location << ":" << (rnd.lrand48() % 0x10000)
        << ".html";
 
-    int size = maxContentSize > minContentSize ?
-               rnd() % (maxContentSize - minContentSize) + minContentSize :
-               minContentSize;
+    int size = maxContentSize > minContentSize
+            ? rnd.lrand48() % (maxContentSize - minContentSize) + minContentSize
+            : minContentSize;
     return
         createDocument(generateRandomContent(size), id.str(), "testdoctype1");
 }
 
 Document::UP
-TestDocMan::createRandomDocument(
-        const std::string& type, int seed, int maxContentSize) const
+TestDocMan::createRandomDocument(const std::string& type, int seed, int maxContentSize) const
 {
-    boost::rand48 rnd(seed);
+    vespalib::Rand48 rnd(seed);
     std::ostringstream id;
-    id << "id:mail:" << type << ":n=" << (rnd() % 0xFFFF);
-    id << ":" << (rnd() % 256) << ".html";
-    return createDocument(generateRandomContent(rnd() % maxContentSize), id.str(), type);
+    id << "id:mail:" << type << ":n=" << (rnd.lrand48() % 0xFFFF);
+    id << ":" << (rnd.lrand48() % 256) << ".html";
+    return createDocument(generateRandomContent(rnd.lrand48() % maxContentSize), id.str(), type);
 }
 
 std::string
diff --git a/document/src/vespa/document/fieldvalue/document.cpp b/document/src/vespa/document/fieldvalue/document.cpp
index 0cd1faa5d62..33b1bbe8d2c 100644
--- a/document/src/vespa/document/fieldvalue/document.cpp
+++ b/document/src/vespa/document/fieldvalue/document.cpp
@@ -3,7 +3,6 @@
 #include "document.h"
 #include "structuredcache.h"
 #include <vespa/document/datatype/documenttype.h>
-#include <vespa/vespalib/util/crc.h>
 #include <vespa/document/repo/documenttyperepo.h>
 #include <vespa/document/serialization/vespadocumentdeserializer.h>
 #include <vespa/document/serialization/vespadocumentserializer.h>
@@ -238,20 +237,6 @@ Document::toXml(const std::string& indent) const
     return ost.str();
 }
 
-uint32_t
-Document::calculateChecksum() const
-{
-    vespalib::string docId(_id.toString());
-    const vespalib::string & typeName(getType().getName());
-    uint16_t typeVersion(0);  // Hardcode version 0 (version not supported)
-
-    vespalib::crc_32_type calculator;
-    calculator.process_bytes(docId.c_str(), docId.size());
-    calculator.process_bytes(typeName.c_str(), typeName.size());
-    calculator.process_bytes(&typeVersion, sizeof(typeVersion));
-    return calculator.checksum() ^ _fields.calculateChecksum();
-}
-
 void Document::serializeHeader(nbostream& stream) const {
     VespaDocumentSerializer serializer(stream);
     serializer.write(*this);
diff --git a/document/src/vespa/document/fieldvalue/document.h b/document/src/vespa/document/fieldvalue/document.h
index efdc61c93b0..c871e324f05 100644
--- a/document/src/vespa/document/fieldvalue/document.h
+++ b/document/src/vespa/document/fieldvalue/document.h
@@ -106,7 +106,6 @@ public:
 
     bool empty() const override { return _fields.empty(); }
 
-    uint32_t calculateChecksum() const;
     void setFieldValue(const Field& field, FieldValue::UP data) override;
 private:
     friend TransactionGuard;
@@ -125,7 +124,7 @@ private:
 
 class TransactionGuard {
 public:
-    TransactionGuard(Document & value)
+    explicit TransactionGuard(Document & value)
         : _value(value)
     {
         _value.beginTransaction();
diff --git a/document/src/vespa/document/fieldvalue/stringfieldvalue.h b/document/src/vespa/document/fieldvalue/stringfieldvalue.h
index a9d1267d21c..1f364acc760 100644
--- a/document/src/vespa/document/fieldvalue/stringfieldvalue.h
+++ b/document/src/vespa/document/fieldvalue/stringfieldvalue.h
@@ -30,7 +30,7 @@ public:
 
     StringFieldValue &operator=(const StringFieldValue &rhs);
     StringFieldValue &operator=(vespalib::stringref value) override;
-    ~StringFieldValue();
+    ~StringFieldValue() override;
 
     FieldValue &assign(const FieldValue &) override;
 
@@ -69,9 +69,9 @@ private:
         VESPA_DLL_LOCAL AnnotationData(vespalib::ConstBufferRef serialized, const FixedTypeRepo &repo,
                                        uint8_t version, bool isSerializedDataLongLived);
 
-        bool hasSpanTrees() const { return _serialized.size() > 0u; }
-        vespalib::ConstBufferRef getSerializedAnnotations() const { return _serialized; }
-        VESPA_DLL_LOCAL SpanTrees getSpanTrees() const;
+        [[nodiscard]] bool hasSpanTrees() const { return _serialized.size() > 0u; }
+        [[nodiscard]] vespalib::ConstBufferRef getSerializedAnnotations() const { return _serialized; }
+        [[nodiscard]] VESPA_DLL_LOCAL SpanTrees getSpanTrees() const;
     private:
         vespalib::ConstBufferRef _serialized;
         BackingBlob              _backingBlob;
diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp
index 7975dd3a327..444e763c52b 100644
--- a/document/src/vespa/document/fieldvalue/structfieldvalue.cpp
+++ b/document/src/vespa/document/fieldvalue/structfieldvalue.cpp
@@ -7,7 +7,6 @@
 #include <vespa/document/repo/documenttyperepo.h>
 #include <vespa/document/serialization/vespadocumentdeserializer.h>
 #include <vespa/vespalib/objects/nbostream.h>
-#include <vespa/vespalib/util/crc.h>
 #include <vespa/document/datatype/positiondatatype.h>
 #include <vespa/document/util/serializableexceptions.h>
 #include <vespa/document/base/exceptions.h>
@@ -148,7 +147,7 @@ StructFieldValue::getFieldValue(const Field& field) const
         }
         return value;
     }
-    return FieldValue::UP();
+    return {};
 }
 
 vespalib::ConstBufferRef
@@ -159,7 +158,7 @@ StructFieldValue::getRawField(uint32_t id) const
         return buf;
     }
 
-    return vespalib::ConstBufferRef();
+    return {};
 }
 
 bool
@@ -265,15 +264,6 @@ StructFieldValue::compare(const FieldValue& otherOrg) const
     return 0;
 }
 
-uint32_t
-StructFieldValue::calculateChecksum() const
-{
-    nbostream buffer(serialize());
-    vespalib::crc_32_type calculator;
-    calculator.process_bytes(buffer.peek(), buffer.size());
-    return calculator.checksum();
-}
-
 void
 StructFieldValue::printXml(XmlOutputStream& xos) const
 {
diff --git a/document/src/vespa/document/fieldvalue/structfieldvalue.h b/document/src/vespa/document/fieldvalue/structfieldvalue.h
index 564c48273e6..360aa6dca63 100644
--- a/document/src/vespa/document/fieldvalue/structfieldvalue.h
+++ b/document/src/vespa/document/fieldvalue/structfieldvalue.h
@@ -35,7 +35,7 @@ private:
 public:
     using UP = std::unique_ptr<StructFieldValue>;
 
-    StructFieldValue(const DataType &type);
+    explicit StructFieldValue(const DataType &type);
     StructFieldValue(const StructFieldValue & rhs);
     StructFieldValue & operator = (const StructFieldValue & rhs);
     StructFieldValue(StructFieldValue && rhs) noexcept = default;
@@ -83,8 +83,6 @@ public:
      */
     bool hasChanged() const { return _hasChanged; }
 
-    uint32_t calculateChecksum() const;
-
     /**
      * Called by document to reset struct when deserializing where this struct
      * has no content. This clears content and sets changed to false.